From: Linus Torvalds Date: Wed, 27 Jul 2016 21:03:52 +0000 (-0700) Subject: Merge tag 'leds_for_4.8' of git://git.kernel.org/pub/scm/linux/kernel/git/j.anaszewsk... X-Git-Tag: v4.8-rc1~136 X-Git-Url: http://git.samba.org/samba.git/?p=sfrench%2Fcifs-2.6.git;a=commitdiff_plain;h=607e11ab6654e167b1b0ec132cedc73e220f63c6;hp=5706c01fcfb313cef7aed6a283b43c5452cf8336 Merge tag 'leds_for_4.8' of git://git./linux/kernel/git/j.anaszewski/linux-leds Pull LED updates from Jacek Anaszewski: "New LED class driver: - LED driver for TI LP3952 6-Channel Color LED LED core improvements: - Only descend into leds directory when CONFIG_NEW_LEDS is set - Add no-op gpio_led_register_device when LED subsystem is disabled - MAINTAINERS: Add file patterns for led device tree bindings LED Trigger core improvements: - return error if invalid trigger name is provided via sysfs LED class drivers improvements - is31fl32xx: define complete i2c_device_id table - is31fl32xx: fix typo in id and match table names - leds-gpio: Set of_node for created LED devices - pca9532: Add device tree support Conversion of IDE trigger to common disk trigger: - leds: convert IDE trigger to common disk trigger - leds: documentation: 'ide-disk' to 'disk-activity' - unicore32: use the new LED disk activity trigger - parisc: use the new LED disk activity trigger - mips: use the new LED disk activity trigger - arm: use the new LED disk activity trigger - powerpc: use the new LED disk activity trigger" * tag 'leds_for_4.8' of git://git.kernel.org/pub/scm/linux/kernel/git/j.anaszewski/linux-leds: leds: is31fl32xx: define complete i2c_device_id table leds: is31fl32xx: fix typo in id and match table names leds: LED driver for TI LP3952 6-Channel Color LED leds: leds-gpio: Set of_node for created LED devices leds: triggers: return error if invalid trigger name is provided via sysfs leds: Only descend into leds directory when CONFIG_NEW_LEDS is set leds: Add no-op gpio_led_register_device when LED subsystem is disabled unicore32: use the new LED disk activity trigger parisc: use the new LED disk activity trigger mips: use the new LED disk activity trigger arm: use the new LED disk activity trigger powerpc: use the new LED disk activity trigger leds: documentation: 'ide-disk' to 'disk-activity' leds: convert IDE trigger to common disk trigger leds: pca9532: Add device tree support MAINTAINERS: Add file patterns for led device tree bindings --- diff --git a/.mailmap b/.mailmap index 779a9caadc7f..d2acafb09e60 100644 --- a/.mailmap +++ b/.mailmap @@ -11,6 +11,7 @@ Aaron Durbin Adam Oldham Adam Radford Adrian Bunk +Adriana Reus Alan Cox Alan Cox Aleksey Gorelov @@ -21,6 +22,7 @@ Andrey Ryabinin Andrew Morton Andrew Vasquez Andy Adamson +Antoine Tenart Antonio Ospite Archit Taneja Arnaud Patard @@ -30,6 +32,9 @@ Axel Lin Ben Gardner Ben M Cahill Björn Steinbrink +Boris Brezillon +Boris Brezillon +Boris Brezillon Brian Avery Brian King Christoph Hellwig @@ -90,6 +95,8 @@ Linas Vepstas Mark Brown Matthieu CASTET Mauro Carvalho Chehab +Matt Ranostay Matthew Ranostay +Matt Ranostay Mayuresh Janorkar Michael Buesch Michel Dänzer diff --git a/Documentation/.gitignore b/Documentation/.gitignore new file mode 100644 index 000000000000..e74fec8693b2 --- /dev/null +++ b/Documentation/.gitignore @@ -0,0 +1,2 @@ +output +*.pyc diff --git a/Documentation/00-INDEX b/Documentation/00-INDEX index cd077ca0e1b8..cb9a6c6fa83b 100644 --- a/Documentation/00-INDEX +++ b/Documentation/00-INDEX @@ -255,10 +255,10 @@ kbuild/ - directory with info about the kernel build process. kdump/ - directory with mini HowTo on getting the crash dump code to work. -kernel-doc-nano-HOWTO.txt - - mini HowTo on generation and location of kernel documentation files. kernel-docs.txt - listing of various WWW + books that document kernel internals. +kernel-documentation.rst + - how to write and format reStructuredText kernel documentation kernel-parameters.txt - summary listing of command line / boot prompt args for the kernel. kernel-per-CPU-kthreads.txt diff --git a/Documentation/ABI/testing/configfs-acpi b/Documentation/ABI/testing/configfs-acpi new file mode 100644 index 000000000000..4ab4e99aa863 --- /dev/null +++ b/Documentation/ABI/testing/configfs-acpi @@ -0,0 +1,36 @@ +What: /config/acpi +Date: July 2016 +KernelVersion: 4.8 +Contact: linux-acpi@vger.kernel.org +Description: + This represents the ACPI subsystem entry point directory. It + contains sub-groups corresponding to ACPI configurable options. + +What: /config/acpi/table +Date: July 2016 +KernelVersion: 4.8 +Description: + + This group contains the configuration for user defined ACPI + tables. The attributes of a user define table are: + + aml - a binary attribute that the user can use to + fill in the ACPI aml definitions. Once the aml + data is written to this file and the file is + closed the table will be loaded and ACPI devices + will be enumerated. To check if the operation is + successful the user must check the error code + for close(). If the operation is successful, + subsequent writes to this attribute will fail. + + The rest of the attributes are read-only and are valid only + after the table has been loaded by filling the aml entry: + + signature - ASCII table signature + length - length of table in bytes, including the header + revision - ACPI Specification minor version number + oem_id - ASCII OEM identification + oem_table_id - ASCII OEM table identification + oem_revision - OEM revision number + asl_compiler_id - ASCII ASL compiler vendor ID + asl_compiler_revision - ASL compiler version diff --git a/Documentation/ABI/testing/configfs-iio b/Documentation/ABI/testing/configfs-iio index 2483756fccf5..aebda53ec0f7 100644 --- a/Documentation/ABI/testing/configfs-iio +++ b/Documentation/ABI/testing/configfs-iio @@ -19,3 +19,16 @@ KernelVersion: 4.4 Description: High resolution timers directory. Creating a directory here will result in creating a hrtimer trigger in the IIO subsystem. + +What: /config/iio/devices +Date: April 2016 +KernelVersion: 4.7 +Description: + Industrial IO software devices directory. + +What: /config/iio/devices/dummy +Date: April 2016 +KernelVersion: 4.7 +Description: + Dummy IIO devices directory. Creating a directory here will result + in creating a dummy IIO device in the IIO subystem. diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio index df44998e7506..fee35c00cc4e 100644 --- a/Documentation/ABI/testing/sysfs-bus-iio +++ b/Documentation/ABI/testing/sysfs-bus-iio @@ -32,6 +32,13 @@ Description: Description of the physical chip / device for device X. Typically a part number. +What: /sys/bus/iio/devices/iio:deviceX/timestamp_clock +KernelVersion: 4.5 +Contact: linux-iio@vger.kernel.org +Description: + String identifying current posix clock used to timestamp + buffered samples and events for device X. + What: /sys/bus/iio/devices/iio:deviceX/sampling_frequency What: /sys/bus/iio/devices/iio:deviceX/buffer/sampling_frequency What: /sys/bus/iio/devices/triggerX/sampling_frequency @@ -1565,3 +1572,10 @@ Description: * X is in the plane of the propellers, perpendicular to Y axis, and positive towards the starboard side of the UAV ; * Z is perpendicular to propellers plane and positive upwards. + +What: /sys/bus/iio/devices/iio:deviceX/in_electricalconductivity_raw +KernelVersion: 4.8 +Contact: linux-iio@vger.kernel.org +Description: + Raw (unscaled no offset etc.) electric conductivity reading that + can be processed to siemens per meter. diff --git a/Documentation/ABI/testing/sysfs-bus-iio-health-afe440x b/Documentation/ABI/testing/sysfs-bus-iio-health-afe440x index 3740f253d406..6adba9058b22 100644 --- a/Documentation/ABI/testing/sysfs-bus-iio-health-afe440x +++ b/Documentation/ABI/testing/sysfs-bus-iio-health-afe440x @@ -1,54 +1,41 @@ -What: /sys/bus/iio/devices/iio:deviceX/tia_resistanceY - /sys/bus/iio/devices/iio:deviceX/tia_capacitanceY -Date: December 2015 -KernelVersion: -Contact: Andrew F. Davis -Description: - Get and set the resistance and the capacitance settings for the - Transimpedance Amplifier. Y is 1 for Rf1 and Cf1, Y is 2 for - Rf2 and Cf2 values. - -What: /sys/bus/iio/devices/iio:deviceX/tia_separate_en -Date: December 2015 -KernelVersion: -Contact: Andrew F. Davis -Description: - Enable or disable separate settings for the TransImpedance - Amplifier above, when disabled both values are set by the - first channel. - -What: /sys/bus/iio/devices/iio:deviceX/in_intensity_ledY_raw - /sys/bus/iio/devices/iio:deviceX/in_intensity_ledY_ambient_raw -Date: December 2015 +What: /sys/bus/iio/devices/iio:deviceX/in_intensityY_raw +Date: May 2016 KernelVersion: Contact: Andrew F. Davis Description: Get measured values from the ADC for these stages. Y is the - specific LED number. The values are expressed in 24-bit twos - complement. + specific stage number corresponding to datasheet stage names + as follows: + 1 -> LED2 + 2 -> ALED2/LED3 + 3 -> LED1 + 4 -> ALED1/LED4 + Note that channels 5 and 6 represent LED2-ALED2 and LED1-ALED1 + respectively which simply helper channels containing the + calculated difference in the value of stage 1 - 2 and 3 - 4. + The values are expressed in 24-bit twos complement. -What: /sys/bus/iio/devices/iio:deviceX/in_intensity_ledY-ledY_ambient_raw -Date: December 2015 +What: /sys/bus/iio/devices/iio:deviceX/in_intensityY_offset +Date: May 2016 KernelVersion: Contact: Andrew F. Davis Description: - Get differential values from the ADC for these stages. Y is the - specific LED number. The values are expressed in 24-bit twos - complement for the specified LEDs. + Get and set the offset cancellation DAC setting for these + stages. The values are expressed in 5-bit sign-magnitude. -What: /sys/bus/iio/devices/iio:deviceX/out_current_ledY_offset - /sys/bus/iio/devices/iio:deviceX/out_current_ledY_ambient_offset -Date: December 2015 +What: /sys/bus/iio/devices/iio:deviceX/in_intensityY_resistance +What: /sys/bus/iio/devices/iio:deviceX/in_intensityY_capacitance +Date: May 2016 KernelVersion: Contact: Andrew F. Davis Description: - Get and set the offset cancellation DAC setting for these - stages. The values are expressed in 5-bit sign-magnitude. + Get and set the resistance and the capacitance settings for the + Transimpedance Amplifier during the associated stage. -What: /sys/bus/iio/devices/iio:deviceX/out_current_ledY_raw -Date: December 2015 +What: /sys/bus/iio/devices/iio:deviceX/out_currentY_raw +Date: May 2016 KernelVersion: Contact: Andrew F. Davis Description: - Get and set the LED current for the specified LED. Y is the - specific LED number. + Get and set the LED current for the specified LED active during + this stage. Y is the specific stage number. diff --git a/Documentation/ABI/testing/sysfs-class-net-batman-adv b/Documentation/ABI/testing/sysfs-class-net-batman-adv index 518f6a1dbc0c..898106849e27 100644 --- a/Documentation/ABI/testing/sysfs-class-net-batman-adv +++ b/Documentation/ABI/testing/sysfs-class-net-batman-adv @@ -1,19 +1,10 @@ -What: /sys/class/net//batman-adv/throughput_override -Date: Feb 2014 -Contact: Antonio Quartulli -description: - Defines the throughput value to be used by B.A.T.M.A.N. V - when estimating the link throughput using this interface. - If the value is set to 0 then batman-adv will try to - estimate the throughput by itself. - What: /sys/class/net//batman-adv/elp_interval Date: Feb 2014 Contact: Linus Lüssing Description: Defines the interval in milliseconds in which batman - sends its probing packets for link quality measurements. + emits probing packets for neighbor sensing (ELP). What: /sys/class/net//batman-adv/iface_status Date: May 2010 @@ -28,3 +19,12 @@ Description: The /sys/class/net//batman-adv/mesh_iface file displays the batman mesh interface this currently is associated with. + +What: /sys/class/net//batman-adv/throughput_override +Date: Feb 2014 +Contact: Antonio Quartulli +description: + Defines the throughput value to be used by B.A.T.M.A.N. V + when estimating the link throughput using this interface. + If the value is set to 0 then batman-adv will try to + estimate the throughput by itself. diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 16501334b99f..498741737055 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -340,3 +340,13 @@ Description: POWERNV CPUFreq driver's frequency throttle stats directory and 'policyX/throttle_stats' directory and all the attributes are same as the /sys/devices/system/cpu/cpuX/cpufreq/throttle_stats directory and attributes which give the frequency throttle information of the chip. + +What: /sys/devices/system/cpu/cpuX/regs/ + /sys/devices/system/cpu/cpuX/regs/identification/ + /sys/devices/system/cpu/cpuX/regs/identification/midr_el1 + /sys/devices/system/cpu/cpuX/regs/identification/revidr_el1 +Date: June 2016 +Contact: Linux ARM Kernel Mailing list +Description: AArch64 CPU registers + 'identification' directory exposes the CPU ID registers for + identifying model and revision of the CPU. diff --git a/Documentation/CodingStyle b/Documentation/CodingStyle index 9a70ddd16584..a096836723ca 100644 --- a/Documentation/CodingStyle +++ b/Documentation/CodingStyle @@ -458,7 +458,7 @@ of the function, telling people what it does, and possibly WHY it does it. When commenting the kernel API functions, please use the kernel-doc format. -See the files Documentation/kernel-doc-nano-HOWTO.txt and scripts/kernel-doc +See the files Documentation/kernel-documentation.rst and scripts/kernel-doc for details. Linux style for comments is the C89 "/* ... */" style. diff --git a/Documentation/DocBook/80211.tmpl b/Documentation/DocBook/80211.tmpl index 5f7c55999c77..800fe7a9024c 100644 --- a/Documentation/DocBook/80211.tmpl +++ b/Documentation/DocBook/80211.tmpl @@ -136,6 +136,7 @@ !Finclude/net/cfg80211.h cfg80211_ibss_joined !Finclude/net/cfg80211.h cfg80211_connect_result !Finclude/net/cfg80211.h cfg80211_connect_bss +!Finclude/net/cfg80211.h cfg80211_connect_timeout !Finclude/net/cfg80211.h cfg80211_roamed !Finclude/net/cfg80211.h cfg80211_disconnected !Finclude/net/cfg80211.h cfg80211_ready_on_channel diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile index d70f9b68174e..01bab5014a4a 100644 --- a/Documentation/DocBook/Makefile +++ b/Documentation/DocBook/Makefile @@ -6,6 +6,8 @@ # To add a new book the only step required is to add the book to the # list of DOCBOOKS. +ifeq ($(IGNORE_DOCBOOKS),) + DOCBOOKS := z8530book.xml device-drivers.xml \ kernel-hacking.xml kernel-locking.xml deviceiobook.xml \ writing_usb_driver.xml networking.xml \ @@ -33,10 +35,6 @@ PDF_METHOD = $(prefer-db2x) PS_METHOD = $(prefer-db2x) -### -# The targets that may be used. -PHONY += xmldocs sgmldocs psdocs pdfdocs htmldocs mandocs installmandocs cleandocs - targets += $(DOCBOOKS) BOOKS := $(addprefix $(obj)/,$(DOCBOOKS)) xmldocs: $(BOOKS) @@ -63,6 +61,9 @@ installmandocs: mandocs sort -k 2 -k 1 | uniq -f 1 | sed -e 's: :/:' | \ xargs install -m 644 -t /usr/local/man/man9/ +# no-op for the DocBook toolchain +epubdocs: + ### #External programs used KERNELDOCXMLREF = $(srctree)/scripts/kernel-doc-xml-ref @@ -216,10 +217,24 @@ silent_gen_xml = : -e "s/>/\\>/g"; \ echo "") > $@ +else + +# Needed, due to cleanmediadocs +include Documentation/DocBook/media/Makefile + +htmldocs: +pdfdocs: +psdocs: +xmldocs: +installmandocs: + +endif # IGNORE_DOCBOOKS + + ### # Help targets as used by the top-level makefile dochelp: - @echo ' Linux kernel internal documentation in different formats:' + @echo ' Linux kernel internal documentation in different formats (DocBook):' @echo ' htmldocs - HTML' @echo ' pdfdocs - PDF' @echo ' psdocs - Postscript' @@ -228,8 +243,11 @@ dochelp: @echo ' installmandocs - install man pages generated by mandocs' @echo ' cleandocs - clean all generated DocBook files' @echo - @echo 'make DOCBOOKS="s1.xml s2.xml" [target] Generate only docs s1.xml s2.xml' + @echo ' make DOCBOOKS="s1.xml s2.xml" [target] Generate only docs s1.xml s2.xml' @echo ' valid values for DOCBOOKS are: $(DOCBOOKS)' + @echo + @echo " make IGNORE_DOCBOOKS=1 [target] Don't generate docs from Docbook" + @echo ' This is useful to generate only the ReST docs (Sphinx)' ### diff --git a/Documentation/DocBook/crypto-API.tmpl b/Documentation/DocBook/crypto-API.tmpl index d55dc5a39bad..fb2a1526f6ec 100644 --- a/Documentation/DocBook/crypto-API.tmpl +++ b/Documentation/DocBook/crypto-API.tmpl @@ -440,8 +440,8 @@ The type flag specifies the type of the cipher algorithm. The caller usually provides a 0 when the caller wants the default handling. Otherwise, the caller may provide the - following selections which match the the aforementioned - cipher types: + following selections which match the aforementioned cipher + types: diff --git a/Documentation/DocBook/device-drivers.tmpl b/Documentation/DocBook/device-drivers.tmpl index 8c68768ebee5..58af32b01b90 100644 --- a/Documentation/DocBook/device-drivers.tmpl +++ b/Documentation/DocBook/device-drivers.tmpl @@ -300,6 +300,9 @@ X!Isound/sound_firmware.c !Iinclude/media/media-devnode.h !Iinclude/media/media-entity.h + Consumer Electronics Control devices +!Iinclude/media/cec-edid.h + diff --git a/Documentation/DocBook/iio.tmpl b/Documentation/DocBook/iio.tmpl index f525bf56d1dd..e2ab6a1f223e 100644 --- a/Documentation/DocBook/iio.tmpl +++ b/Documentation/DocBook/iio.tmpl @@ -594,7 +594,7 @@ irqreturn_t sensor_iio_pollfunc(int irq, void *p) { - pf->timestamp = iio_get_time_ns(); + pf->timestamp = iio_get_time_ns((struct indio_dev *)p); return IRQ_WAKE_THREAD; } diff --git a/Documentation/DocBook/media/Makefile b/Documentation/DocBook/media/Makefile index 2840ff483d5a..fdc138624800 100644 --- a/Documentation/DocBook/media/Makefile +++ b/Documentation/DocBook/media/Makefile @@ -64,6 +64,7 @@ IOCTLS = \ $(shell perl -ne 'print "$$1 " if /\#define\s+([A-Z][^\s]+)\s+_IO/' $(srctree)/include/uapi/linux/dvb/net.h) \ $(shell perl -ne 'print "$$1 " if /\#define\s+([^\s]+)\s+_IO/' $(srctree)/include/uapi/linux/dvb/video.h) \ $(shell perl -ne 'print "$$1 " if /\#define\s+([^\s]+)\s+_IO/' $(srctree)/include/uapi/linux/media.h) \ + $(shell perl -ne 'print "$$1 " if /\#define\s+([^\s]+)\s+_IO/' $(srctree)/include/linux/cec.h) \ $(shell perl -ne 'print "$$1 " if /\#define\s+([^\s]+)\s+_IO/' $(srctree)/include/uapi/linux/v4l2-subdev.h) \ DEFINES = \ @@ -100,6 +101,7 @@ STRUCTS = \ $(shell perl -ne 'print "$$1 " if (/^struct\s+([^\s]+)\s+/ && !/_old/)' $(srctree)/include/uapi/linux/dvb/net.h) \ $(shell perl -ne 'print "$$1 " if (/^struct\s+([^\s]+)\s+/)' $(srctree)/include/uapi/linux/dvb/video.h) \ $(shell perl -ne 'print "$$1 " if /^struct\s+([^\s]+)\s+/' $(srctree)/include/uapi/linux/media.h) \ + $(shell perl -ne 'print "$$1 " if /^struct\s+([^\s]+)\s+/' $(srctree)/include/linux/cec.h) \ $(shell perl -ne 'print "$$1 " if /^struct\s+([^\s]+)\s+/' $(srctree)/include/uapi/linux/v4l2-subdev.h) \ $(shell perl -ne 'print "$$1 " if /^struct\s+([^\s]+)\s+/' $(srctree)/include/uapi/linux/v4l2-mediabus.h) diff --git a/Documentation/DocBook/media/v4l/biblio.xml b/Documentation/DocBook/media/v4l/biblio.xml index 9beb30f0071b..87f1d24958aa 100644 --- a/Documentation/DocBook/media/v4l/biblio.xml +++ b/Documentation/DocBook/media/v4l/biblio.xml @@ -342,6 +342,16 @@ in the frequency range from 87,5 to 108,0 MHz Specification Version 1.4a + + HDMI2 + + HDMI Licensing LLC +(http://www.hdmi.org) + + High-Definition Multimedia Interface + Specification Version 2.0 + + DP diff --git a/Documentation/DocBook/media/v4l/cec-api.xml b/Documentation/DocBook/media/v4l/cec-api.xml new file mode 100644 index 000000000000..7062c1fa4904 --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-api.xml @@ -0,0 +1,75 @@ + + + + Hans + Verkuil +
hans.verkuil@cisco.com
 + Initial version. + + + + 2016 + Hans Verkuil + + + + + + 1.0.0 + 2016-03-17 + hv + Initial revision + + + + +CEC API + + + CEC: Consumer Electronics Control + +
+ Introduction + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + HDMI connectors provide a single pin for use by the Consumer Electronics + Control protocol. This protocol allows different devices connected by an HDMI cable + to communicate. The protocol for CEC version 1.4 is defined in supplements 1 (CEC) + and 2 (HEAC or HDMI Ethernet and Audio Return Channel) of the HDMI 1.4a + () specification and the extensions added to CEC version 2.0 + are defined in chapter 11 of the HDMI 2.0 () specification. + + + The bitrate is very slow (effectively no more than 36 bytes per second) and + is based on the ancient AV.link protocol used in old SCART connectors. The protocol + closely resembles a crazy Rube Goldberg contraption and is an unholy mix of low and + high level messages. Some messages, especially those part of the HEAC protocol layered + on top of CEC, need to be handled by the kernel, others can be handled either by the + kernel or by userspace. + + In addition, CEC can be implemented in HDMI receivers, transmitters and in USB + devices that have an HDMI input and an HDMI output and that control just the CEC pin. + + Drivers that support CEC will create a CEC device node (/dev/cecX) + to give userspace access to the CEC adapter. The &CEC-ADAP-G-CAPS; ioctl will tell userspace + what it is allowed to do. +
+ + + + Function Reference + + &sub-cec-func-open; + &sub-cec-func-close; + &sub-cec-func-ioctl; + &sub-cec-func-poll; + + &sub-cec-ioc-adap-g-caps; + &sub-cec-ioc-adap-g-log-addrs; + &sub-cec-ioc-adap-g-phys-addr; + &sub-cec-ioc-dqevent; + &sub-cec-ioc-g-mode; + &sub-cec-ioc-receive; + diff --git a/Documentation/DocBook/media/v4l/cec-func-close.xml b/Documentation/DocBook/media/v4l/cec-func-close.xml new file mode 100644 index 000000000000..0812c8cd9634 --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-func-close.xml @@ -0,0 +1,64 @@ + + + cec close() + &manvol; + + + + cec-close + Close a cec device + + + + + #include <unistd.h> + + int close + int fd + + + + + + Arguments + + + + fd + + &fd; + + + + + + + Description + + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + Closes the cec device. Resources associated with the file descriptor + are freed. The device configuration remain unchanged. + + + + Return Value + + close returns 0 on success. On error, -1 is + returned, and errno is set appropriately. Possible error + codes are: + + + + EBADF + + fd is not a valid open file descriptor. + + + + + + diff --git a/Documentation/DocBook/media/v4l/cec-func-ioctl.xml b/Documentation/DocBook/media/v4l/cec-func-ioctl.xml new file mode 100644 index 000000000000..f92817a2dc80 --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-func-ioctl.xml @@ -0,0 +1,78 @@ + + + cec ioctl() + &manvol; + + + + cec-ioctl + Control a cec device + + + + + #include <sys/ioctl.h> + + int ioctl + int fd + int request + void *argp + + + + + + Arguments + + + + fd + + &fd; + + + + request + + CEC ioctl request code as defined in the cec.h header file, + for example CEC_ADAP_G_CAPS. + + + + argp + + Pointer to a request-specific structure. + + + + + + + Description + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + The ioctl() function manipulates cec device + parameters. The argument fd must be an open file + descriptor. + The ioctl request code specifies the cec + function to be called. It has encoded in it whether the argument is an + input, output or read/write parameter, and the size of the argument + argp in bytes. + Macros and structures definitions specifying cec ioctl requests and + their parameters are located in the cec.h header file. All cec ioctl + requests, their respective function and parameters are specified in + . + + + + &return-value; + + Request-specific error codes are listed in the + individual requests descriptions. + When an ioctl that takes an output or read/write parameter fails, + the parameter remains unmodified. + + diff --git a/Documentation/DocBook/media/v4l/cec-func-open.xml b/Documentation/DocBook/media/v4l/cec-func-open.xml new file mode 100644 index 000000000000..2edc5555b81a --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-func-open.xml @@ -0,0 +1,104 @@ + + + cec open() + &manvol; + + + + cec-open + Open a cec device + + + + + #include <fcntl.h> + + int open + const char *device_name + int flags + + + + + + Arguments + + + + device_name + + Device to be opened. + + + + flags + + Open flags. Access mode must be O_RDWR. + + When the O_NONBLOCK flag is +given, the &CEC-RECEIVE; ioctl will return &EAGAIN; when no message is +available, and the &CEC-TRANSMIT;, &CEC-ADAP-S-PHYS-ADDR; and +&CEC-ADAP-S-LOG-ADDRS; ioctls all act in non-blocking mode. + Other flags have no effect. + + + + + + Description + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + To open a cec device applications call open() + with the desired device name. The function has no side effects; the device + configuration remain unchanged. + When the device is opened in read-only mode, attempts to modify its + configuration will result in an error, and errno will be + set to EBADF. + + + Return Value + + open returns the new file descriptor on success. + On error, -1 is returned, and errno is set appropriately. + Possible error codes include: + + + + EACCES + + The requested access to the file is not allowed. + + + + EMFILE + + The process already has the maximum number of files open. + + + + + ENFILE + + The system limit on the total number of open files has been + reached. + + + + ENOMEM + + Insufficient kernel memory was available. + + + + ENXIO + + No device corresponding to this device special file exists. + + + + + + diff --git a/Documentation/DocBook/media/v4l/cec-func-poll.xml b/Documentation/DocBook/media/v4l/cec-func-poll.xml new file mode 100644 index 000000000000..1bddbde0142d --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-func-poll.xml @@ -0,0 +1,94 @@ + + + cec poll() + &manvol; + + + + cec-poll + Wait for some event on a file descriptor + + + + + #include <sys/poll.h> + + int poll + struct pollfd *ufds + unsigned int nfds + int timeout + + + + + + Description + + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + With the poll() function applications +can wait for CEC events. + + On success poll() returns the number of +file descriptors that have been selected (that is, file descriptors +for which the revents field of the +respective pollfd structure is non-zero). +CEC devices set the POLLIN and +POLLRDNORM flags in the +revents field if there are messages in the +receive queue. If the transmit queue has room for new messages, the +POLLOUT and POLLWRNORM +flags are set. If there are events in the event queue, then the +POLLPRI flag is set. +When the function timed out it returns a value of zero, on +failure it returns -1 and the +errno variable is set appropriately. + + + For more details see the +poll() manual page. + + + + Return Value + + On success, poll() returns the number +structures which have non-zero revents +fields, or zero if the call timed out. On error +-1 is returned, and the +errno variable is set appropriately: + + + + EBADF + + One or more of the ufds members +specify an invalid file descriptor. + + + + EFAULT + + ufds references an inaccessible +memory area. + + + + EINTR + + The call was interrupted by a signal. + + + + EINVAL + + The nfds argument is greater +than OPEN_MAX. + + + + + diff --git a/Documentation/DocBook/media/v4l/cec-ioc-adap-g-caps.xml b/Documentation/DocBook/media/v4l/cec-ioc-adap-g-caps.xml new file mode 100644 index 000000000000..3523ef2259b1 --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-ioc-adap-g-caps.xml @@ -0,0 +1,151 @@ + + + ioctl CEC_ADAP_G_CAPS + &manvol; + + + + CEC_ADAP_G_CAPS + Query device capabilities + + + + + + int ioctl + int fd + int request + struct cec_caps *argp + + + + + + Arguments + + + + fd + + File descriptor returned by + open(). + + + + request + + CEC_ADAP_G_CAPS + + + + argp + + + + + + + + + Description + + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + All cec devices must support the CEC_ADAP_G_CAPS + ioctl. To query device information, applications call the ioctl with a + pointer to a &cec-caps;. The driver fills the structure and returns + the information to the application. + The ioctl never fails. + + + struct <structname>cec_caps</structname> + + &cs-str; + + + char + driver[32] + The name of the cec adapter driver. + + + char + name[32] + The name of this CEC adapter. The combination driver + and name must be unique. + + + __u32 + capabilities + The capabilities of the CEC adapter, see . + + + __u32 + version + CEC Framework API version, formatted with the + KERNEL_VERSION() macro. + + + +
+ + + CEC Capabilities Flags + + &cs-def; + + + CEC_CAP_PHYS_ADDR + 0x00000001 + Userspace has to configure the physical address by + calling &CEC-ADAP-S-PHYS-ADDR;. If this capability isn't set, + then setting the physical address is handled by the kernel + whenever the EDID is set (for an HDMI receiver) or read (for + an HDMI transmitter). + + + CEC_CAP_LOG_ADDRS + 0x00000002 + Userspace has to configure the logical addresses by + calling &CEC-ADAP-S-LOG-ADDRS;. If this capability isn't set, + then the kernel will have configured this. + + + CEC_CAP_TRANSMIT + 0x00000004 + Userspace can transmit CEC messages by calling &CEC-TRANSMIT;. This + implies that userspace can be a follower as well, since being able to + transmit messages is a prerequisite of becoming a follower. If this + capability isn't set, then the kernel will handle all CEC transmits + and process all CEC messages it receives. + + + + CEC_CAP_PASSTHROUGH + 0x00000008 + Userspace can use the passthrough mode by + calling &CEC-S-MODE;. + + + CEC_CAP_RC + 0x00000010 + This adapter supports the remote control protocol. + + + CEC_CAP_MONITOR_ALL + 0x00000020 + The CEC hardware can monitor all messages, not just directed and + broadcast messages. + + + +
+ + + + &return-value; + + diff --git a/Documentation/DocBook/media/v4l/cec-ioc-adap-g-log-addrs.xml b/Documentation/DocBook/media/v4l/cec-ioc-adap-g-log-addrs.xml new file mode 100644 index 000000000000..302b8294f7fc --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-ioc-adap-g-log-addrs.xml @@ -0,0 +1,329 @@ + + + ioctl CEC_ADAP_G_LOG_ADDRS, CEC_ADAP_S_LOG_ADDRS + &manvol; + + + + CEC_ADAP_G_LOG_ADDRS + CEC_ADAP_S_LOG_ADDRS + Get or set the logical addresses + + + + + + int ioctl + int fd + int request + struct cec_log_addrs *argp + + + + + + Arguments + + + + fd + + File descriptor returned by + open(). + + + + request + + CEC_ADAP_G_LOG_ADDRS, CEC_ADAP_S_LOG_ADDRS + + + + argp + + + + + + + + + Description + + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + To query the current CEC logical addresses, applications call the +CEC_ADAP_G_LOG_ADDRS ioctl with a pointer to a +cec_log_addrs structure where the drivers stores the +logical addresses. + + To set new logical addresses, applications fill in struct cec_log_addrs +and call the CEC_ADAP_S_LOG_ADDRS ioctl with a pointer to this struct. +The CEC_ADAP_S_LOG_ADDRS ioctl is only available if +CEC_CAP_LOG_ADDRS is set (&ENOTTY; is returned otherwise). This ioctl will block until all +requested logical addresses have been claimed. CEC_ADAP_S_LOG_ADDRS +can only be called by a file handle in initiator mode (see &CEC-S-MODE;). + + + struct <structname>cec_log_addrs</structname> + + &cs-str; + + + __u8 + log_addr[CEC_MAX_LOG_ADDRS] + The actual logical addresses that were claimed. This is set by the + driver. If no logical address could be claimed, then it is set to + CEC_LOG_ADDR_INVALID. If this adapter is Unregistered, + then log_addr[0] is set to 0xf and all others to + CEC_LOG_ADDR_INVALID. + + + __u16 + log_addr_mask + The bitmask of all logical addresses this adapter has claimed. + If this adapter is Unregistered then log_addr_mask + sets bit 15 and clears all other bits. If this adapter is not configured at all, then + log_addr_mask is set to 0. Set by the driver. + + + __u8 + cec_version + The CEC version that this adapter shall use. See + . + Used to implement the CEC_MSG_CEC_VERSION and + CEC_MSG_REPORT_FEATURES messages. Note that + CEC_OP_CEC_VERSION_1_3A is not allowed + by the CEC framework. + + + + __u8 + num_log_addrs + Number of logical addresses to set up. Must be ≤ + available_log_addrs as returned by + &CEC-ADAP-G-CAPS;. All arrays in this structure are only filled up to + index available_log_addrs-1. The remaining + array elements will be ignored. Note that the CEC 2.0 standard allows + for a maximum of 2 logical addresses, although some hardware has support + for more. CEC_MAX_LOG_ADDRS is 4. The driver will + return the actual number of logical addresses it could claim, which may + be less than what was requested. If this field is set to 0, then the + CEC adapter shall clear all claimed logical addresses and all other + fields will be ignored. + + + __u32 + vendor_id + The vendor ID is a 24-bit number that identifies the specific + vendor or entity. Based on this ID vendor specific commands may be + defined. If you do not want a vendor ID then set it to + CEC_VENDOR_ID_NONE. + + + __u32 + flags + Flags. No flags are defined yet, so set this to 0. + + + char + osd_name[15] + The On-Screen Display name as is returned by the + CEC_MSG_SET_OSD_NAME message. + + + __u8 + primary_device_type[CEC_MAX_LOG_ADDRS] + Primary device type for each logical address. See + for possible types. + + + __u8 + log_addr_type[CEC_MAX_LOG_ADDRS] + Logical address types. See for + possible types. The driver will update this with the actual logical address + type that it claimed (e.g. it may have to fallback to + CEC_LOG_ADDR_TYPE_UNREGISTERED). + + + __u8 + all_device_types[CEC_MAX_LOG_ADDRS] + CEC 2.0 specific: all device types. See . + Used to implement the CEC_MSG_REPORT_FEATURES message. + This field is ignored if cec_version < + CEC_OP_CEC_VERSION_2_0. + + + __u8 + features[CEC_MAX_LOG_ADDRS][12] + Features for each logical address. Used to implement the + CEC_MSG_REPORT_FEATURES message. The 12 bytes include + both the RC Profile and the Device Features. + This field is ignored if cec_version < + CEC_OP_CEC_VERSION_2_0. + + + +
+ + + CEC Versions + + &cs-def; + + + CEC_OP_CEC_VERSION_1_3A + 4 + CEC version according to the HDMI 1.3a standard. + + + CEC_OP_CEC_VERSION_1_4B + 5 + CEC version according to the HDMI 1.4b standard. + + + CEC_OP_CEC_VERSION_2_0 + 6 + CEC version according to the HDMI 2.0 standard. + + + +
+ + + CEC Primary Device Types + + &cs-def; + + + CEC_OP_PRIM_DEVTYPE_TV + 0 + Use for a TV. + + + CEC_OP_PRIM_DEVTYPE_RECORD + 1 + Use for a recording device. + + + CEC_OP_PRIM_DEVTYPE_TUNER + 3 + Use for a device with a tuner. + + + CEC_OP_PRIM_DEVTYPE_PLAYBACK + 4 + Use for a playback device. + + + CEC_OP_PRIM_DEVTYPE_AUDIOSYSTEM + 5 + Use for an audio system (e.g. an audio/video receiver). + + + CEC_OP_PRIM_DEVTYPE_SWITCH + 6 + Use for a CEC switch. + + + CEC_OP_PRIM_DEVTYPE_VIDEOPROC + 7 + Use for a video processor device. + + + +
+ + + CEC Logical Address Types + + &cs-def; + + + CEC_LOG_ADDR_TYPE_TV + 0 + Use for a TV. + + + CEC_LOG_ADDR_TYPE_RECORD + 1 + Use for a recording device. + + + CEC_LOG_ADDR_TYPE_TUNER + 2 + Use for a tuner device. + + + CEC_LOG_ADDR_TYPE_PLAYBACK + 3 + Use for a playback device. + + + CEC_LOG_ADDR_TYPE_AUDIOSYSTEM + 4 + Use for an audio system device. + + + CEC_LOG_ADDR_TYPE_SPECIFIC + 5 + Use for a second TV or for a video processor device. + + + CEC_LOG_ADDR_TYPE_UNREGISTERED + 6 + Use this if you just want to remain unregistered. + Used for pure CEC switches or CDC-only devices (CDC: + Capability Discovery and Control). + + + +
+ + + CEC All Device Types Flags + + &cs-def; + + + CEC_OP_ALL_DEVTYPE_TV + 0x80 + This supports the TV type. + + + CEC_OP_ALL_DEVTYPE_RECORD + 0x40 + This supports the Recording type. + + + CEC_OP_ALL_DEVTYPE_TUNER + 0x20 + This supports the Tuner type. + + + CEC_OP_ALL_DEVTYPE_PLAYBACK + 0x10 + This supports the Playback type. + + + CEC_OP_ALL_DEVTYPE_AUDIOSYSTEM + 0x08 + This supports the Audio System type. + + + CEC_OP_ALL_DEVTYPE_SWITCH + 0x04 + This supports the CEC Switch or Video Processing type. + + + +
+ + + + &return-value; + + diff --git a/Documentation/DocBook/media/v4l/cec-ioc-adap-g-phys-addr.xml b/Documentation/DocBook/media/v4l/cec-ioc-adap-g-phys-addr.xml new file mode 100644 index 000000000000..d95f1785080c --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-ioc-adap-g-phys-addr.xml @@ -0,0 +1,86 @@ + + + ioctl CEC_ADAP_G_PHYS_ADDR, CEC_ADAP_S_PHYS_ADDR + &manvol; + + + + CEC_ADAP_G_PHYS_ADDR + CEC_ADAP_S_PHYS_ADDR + Get or set the physical address + + + + + + int ioctl + int fd + int request + __u16 *argp + + + + + + Arguments + + + + fd + + File descriptor returned by + open(). + + + + request + + CEC_ADAP_G_PHYS_ADDR, CEC_ADAP_S_PHYS_ADDR + + + + argp + + + + + + + + + Description + + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + To query the current physical address applications call the +CEC_ADAP_G_PHYS_ADDR ioctl with a pointer to an __u16 +where the driver stores the physical address. + + To set a new physical address applications store the physical address in +an __u16 and call the CEC_ADAP_S_PHYS_ADDR ioctl with a +pointer to this integer. CEC_ADAP_S_PHYS_ADDR is only +available if CEC_CAP_PHYS_ADDR is set (&ENOTTY; will be returned +otherwise). CEC_ADAP_S_PHYS_ADDR +can only be called by a file handle in initiator mode (see &CEC-S-MODE;), if not +&EBUSY; will be returned. + + The physical address is a 16-bit number where each group of 4 bits +represent a digit of the physical address a.b.c.d where the most significant +4 bits represent 'a'. The CEC root device (usually the TV) has address 0.0.0.0. +Every device that is hooked up to an input of the TV has address a.0.0.0 (where +'a' is ≥ 1), devices hooked up to those in turn have addresses a.b.0.0, etc. +So a topology of up to 5 devices deep is supported. The physical address a +device shall use is stored in the EDID of the sink. + +For example, the EDID for each HDMI input of the TV will have a different +physical address of the form a.0.0.0 that the sources will read out and use as +their physical address. + + + + &return-value; + + diff --git a/Documentation/DocBook/media/v4l/cec-ioc-dqevent.xml b/Documentation/DocBook/media/v4l/cec-ioc-dqevent.xml new file mode 100644 index 000000000000..697dde575cd4 --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-ioc-dqevent.xml @@ -0,0 +1,202 @@ + + + ioctl CEC_DQEVENT + &manvol; + + + + CEC_DQEVENT + Dequeue a CEC event + + + + + + int ioctl + int fd + int request + struct cec_event *argp + + + + + + Arguments + + + + fd + + File descriptor returned by + open(). + + + + request + + CEC_DQEVENT + + + + argp + + + + + + + + + Description + + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + CEC devices can send asynchronous events. These can be retrieved by calling + the CEC_DQEVENT ioctl. If the file descriptor is in non-blocking + mode and no event is pending, then it will return -1 and set errno to the &EAGAIN;. + + The internal event queues are per-filehandle and per-event type. If there is + no more room in a queue then the last event is overwritten with the new one. This + means that intermediate results can be thrown away but that the latest event is always + available. This also means that is it possible to read two successive events that have + the same value (e.g. two CEC_EVENT_STATE_CHANGE events with the same state). In that + case the intermediate state changes were lost but it is guaranteed that the state + did change in between the two events. + + + struct <structname>cec_event_state_change</structname> + + &cs-str; + + + __u16 + phys_addr + The current physical address. + + + __u16 + log_addr_mask + The current set of claimed logical addresses. + + + +
+ + + struct <structname>cec_event_lost_msgs</structname> + + &cs-str; + + + __u32 + lost_msgs + Set to the number of lost messages since the filehandle + was opened or since the last time this event was dequeued for + this filehandle. The messages lost are the oldest messages. So + when a new message arrives and there is no more room, then the + oldest message is discarded to make room for the new one. The + internal size of the message queue guarantees that all messages + received in the last two seconds will be stored. Since messages + should be replied to within a second according to the CEC + specification, this is more than enough. + + + + +
+ + + struct <structname>cec_event</structname> + + &cs-str; + + + __u64 + ts + Timestamp of the event in ns. + + + + __u32 + event + The CEC event type, see . + + + + __u32 + flags + Event flags, see . + + + + union + (anonymous) + + + + + + struct cec_event_state_change + state_change + The new adapter state as sent by the CEC_EVENT_STATE_CHANGE + event. + + + + struct cec_event_lost_msgs + lost_msgs + The number of lost messages as sent by the CEC_EVENT_LOST_MSGS + event. + + + +
+ + + CEC Events Types + + &cs-def; + + + CEC_EVENT_STATE_CHANGE + 1 + Generated when the CEC Adapter's state changes. When open() is + called an initial event will be generated for that filehandle with the + CEC Adapter's state at that time. + + + + CEC_EVENT_LOST_MSGS + 2 + Generated if one or more CEC messages were lost because the + application didn't dequeue CEC messages fast enough. + + + +
+ + + CEC Event Flags + + &cs-def; + + + CEC_EVENT_FL_INITIAL_VALUE + 1 + Set for the initial events that are generated when the device is + opened. See the table above for which events do this. This allows + applications to learn the initial state of the CEC adapter at open() + time. + + + +
+ + + + &return-value; + + diff --git a/Documentation/DocBook/media/v4l/cec-ioc-g-mode.xml b/Documentation/DocBook/media/v4l/cec-ioc-g-mode.xml new file mode 100644 index 000000000000..26b4282ad134 --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-ioc-g-mode.xml @@ -0,0 +1,255 @@ + + + ioctl CEC_G_MODE, CEC_S_MODE + &manvol; + + + + CEC_G_MODE + CEC_S_MODE + Get or set exclusive use of the CEC adapter + + + + + + int ioctl + int fd + int request + __u32 *argp + + + + + + Arguments + + + + fd + + File descriptor returned by + open(). + + + + request + + CEC_G_MODE, CEC_S_MODE + + + + argp + + + + + + + + + Description + + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + By default any filehandle can use &CEC-TRANSMIT; and &CEC-RECEIVE;, but +in order to prevent applications from stepping on each others toes it must be possible +to obtain exclusive access to the CEC adapter. This ioctl sets the filehandle +to initiator and/or follower mode which can be exclusive depending on the chosen +mode. The initiator is the filehandle that is used +to initiate messages, i.e. it commands other CEC devices. The follower is the filehandle +that receives messages sent to the CEC adapter and processes them. The same filehandle +can be both initiator and follower, or this role can be taken by two different +filehandles. + + When a CEC message is received, then the CEC framework will decide how +it will be processed. If the message is a reply to an earlier transmitted message, +then the reply is sent back to the filehandle that is waiting for it. In addition +the CEC framework will process it. + + If the message is not a reply, then the CEC framework will process it +first. If there is no follower, then the message is just discarded and a feature +abort is sent back to the initiator if the framework couldn't process it. If there +is a follower, then the message is passed on to the follower who will use +&CEC-RECEIVE; to dequeue the new message. The framework expects the follower to +make the right decisions. + + The CEC framework will process core messages unless requested otherwise +by the follower. The follower can enable the passthrough mode. In that case, the +CEC framework will pass on most core messages without processing them and +the follower will have to implement those messages. There are some messages +that the core will always process, regardless of the passthrough mode. See + for details. + + If there is no initiator, then any CEC filehandle can use &CEC-TRANSMIT;. +If there is an exclusive initiator then only that initiator can call &CEC-TRANSMIT;. +The follower can of course always call &CEC-TRANSMIT;. + + Available initiator modes are: + + + Initiator Modes + + &cs-def; + + + CEC_MODE_NO_INITIATOR + 0x0 + This is not an initiator, i.e. it cannot transmit CEC messages + or make any other changes to the CEC adapter. + + + CEC_MODE_INITIATOR + 0x1 + This is an initiator (the default when the device is opened) and it + can transmit CEC messages and make changes to the CEC adapter, unless there + is an exclusive initiator. + + + CEC_MODE_EXCL_INITIATOR + 0x2 + This is an exclusive initiator and this file descriptor is the only one + that can transmit CEC messages and make changes to the CEC adapter. If someone + else is already the exclusive initiator then an attempt to become one will return + the &EBUSY; error. + + + +
+ + Available follower modes are: + + + Follower Modes + + &cs-def; + + + CEC_MODE_NO_FOLLOWER + 0x00 + This is not a follower (the default when the device is opened). + + + CEC_MODE_FOLLOWER + 0x10 + This is a follower and it will receive CEC messages unless there is + an exclusive follower. You cannot become a follower if CEC_CAP_TRANSMIT + is not set or if CEC_MODE_NO_INITIATOR was specified, + &EINVAL; is returned in that case. + + + CEC_MODE_EXCL_FOLLOWER + 0x20 + This is an exclusive follower and only this file descriptor will receive + CEC messages for processing. If someone else is already the exclusive follower + then an attempt to become one will return the &EBUSY; error. You cannot become + a follower if CEC_CAP_TRANSMIT is not set or if + CEC_MODE_NO_INITIATOR was specified, &EINVAL; is returned + in that case. + + + CEC_MODE_EXCL_FOLLOWER_PASSTHRU + 0x30 + This is an exclusive follower and only this file descriptor will receive + CEC messages for processing. In addition it will put the CEC device into + passthrough mode, allowing the exclusive follower to handle most core messages + instead of relying on the CEC framework for that. If someone else is already the + exclusive follower then an attempt to become one will return the &EBUSY; error. + You cannot become a follower if CEC_CAP_TRANSMIT + is not set or if CEC_MODE_NO_INITIATOR was specified, + &EINVAL; is returned in that case. + + + CEC_MODE_MONITOR + 0xe0 + Put the file descriptor into monitor mode. Can only be used in combination + with CEC_MODE_NO_INITIATOR, otherwise &EINVAL; will be + returned. In monitor mode all messages this CEC device transmits and all messages + it receives (both broadcast messages and directed messages for one its logical + addresses) will be reported. This is very useful for debugging. This is only + allowed if the process has the CAP_NET_ADMIN + capability. If that is not set, then &EPERM; is returned. + + + CEC_MODE_MONITOR_ALL + 0xf0 + Put the file descriptor into 'monitor all' mode. Can only be used in combination + with CEC_MODE_NO_INITIATOR, otherwise &EINVAL; will be + returned. In 'monitor all' mode all messages this CEC device transmits and all messages + it receives, including directed messages for other CEC devices will be reported. This + is very useful for debugging, but not all devices support this. This mode requires that + the CEC_CAP_MONITOR_ALL capability is set, otherwise &EINVAL; is + returned. This is only allowed if the process has the CAP_NET_ADMIN + capability. If that is not set, then &EPERM; is returned. + + + +
+ + Core message processing details: + + + Core Message Processing + + &cs-def; + + + CEC_MSG_GET_CEC_VERSION + When in passthrough mode this message has to be handled by userspace, + otherwise the core will return the CEC version that was set with &CEC-ADAP-S-LOG-ADDRS;. + + + CEC_MSG_GIVE_DEVICE_VENDOR_ID + When in passthrough mode this message has to be handled by userspace, + otherwise the core will return the vendor ID that was set with &CEC-ADAP-S-LOG-ADDRS;. + + + CEC_MSG_ABORT + When in passthrough mode this message has to be handled by userspace, + otherwise the core will return a feature refused message as per the specification. + + + CEC_MSG_GIVE_PHYSICAL_ADDR + When in passthrough mode this message has to be handled by userspace, + otherwise the core will report the current physical address. + + + CEC_MSG_GIVE_OSD_NAME + When in passthrough mode this message has to be handled by userspace, + otherwise the core will report the current OSD name as was set with + &CEC-ADAP-S-LOG-ADDRS;. + + + CEC_MSG_GIVE_FEATURES + When in passthrough mode this message has to be handled by userspace, + otherwise the core will report the current features as was set with + &CEC-ADAP-S-LOG-ADDRS; or the message is ignore if the CEC version was + older than 2.0. + + + CEC_MSG_USER_CONTROL_PRESSED + If CEC_CAP_RC is set, then generate a remote control + key press. This message is always passed on to userspace. + + + CEC_MSG_USER_CONTROL_RELEASED + If CEC_CAP_RC is set, then generate a remote control + key release. This message is always passed on to userspace. + + + CEC_MSG_REPORT_PHYSICAL_ADDR + The CEC framework will make note of the reported physical address + and then just pass the message on to userspace. + + + +
+ + + + &return-value; + + diff --git a/Documentation/DocBook/media/v4l/cec-ioc-receive.xml b/Documentation/DocBook/media/v4l/cec-ioc-receive.xml new file mode 100644 index 000000000000..fde9f8678e67 --- /dev/null +++ b/Documentation/DocBook/media/v4l/cec-ioc-receive.xml @@ -0,0 +1,274 @@ + + + ioctl CEC_RECEIVE, CEC_TRANSMIT + &manvol; + + + + CEC_RECEIVE + CEC_TRANSMIT + Receive or transmit a CEC message + + + + + + int ioctl + int fd + int request + struct cec_msg *argp + + + + + + Arguments + + + + fd + + File descriptor returned by + open(). + + + + request + + CEC_RECEIVE, CEC_TRANSMIT + + + + argp + + + + + + + + + Description + + + Note: this documents the proposed CEC API. This API is not yet finalized and + is currently only available as a staging kernel module. + + + To receive a CEC message the application has to fill in the + cec_msg structure and pass it to the + CEC_RECEIVE ioctl. CEC_RECEIVE is + only available if CEC_CAP_RECEIVE is set. If the + file descriptor is in non-blocking mode and there are no received + messages pending, then it will return -1 and set errno to the &EAGAIN;. + If the file descriptor is in blocking mode and timeout + is non-zero and no message arrived within timeout + milliseconds, then it will return -1 and set errno to the &ETIMEDOUT;. + + To send a CEC message the application has to fill in the + cec_msg structure and pass it to the + CEC_TRANSMIT ioctl. CEC_TRANSMIT is + only available if CEC_CAP_TRANSMIT is set. + If there is no more room in the transmit queue, then it will return + -1 and set errno to the &EBUSY;. + + + struct <structname>cec_msg</structname> + + &cs-str; + + + __u64 + ts + Timestamp of when the message was transmitted in ns in the case + of CEC_TRANSMIT with reply + set to 0, or the timestamp of the received message in all other cases. + + + __u32 + len + The length of the message. For CEC_TRANSMIT this + is filled in by the application. The driver will fill this in for + CEC_RECEIVE and for CEC_TRANSMIT + it will be filled in with the length of the reply message if + reply was set. + + + __u32 + timeout + The timeout in milliseconds. This is the time the device will wait for a message to + be received before timing out. If it is set to 0, then it will wait indefinitely when it + is called by CEC_RECEIVE. If it is 0 and it is called by + CEC_TRANSMIT, then it will be replaced by 1000 if the + reply is non-zero or ignored if reply + is 0. + + + __u32 + sequence + The sequence number is automatically assigned by the CEC + framework for all transmitted messages. It can be later used by the + framework to generate an event if a reply for a message was + requested and the message was transmitted in a non-blocking mode. + + + + __u32 + flags + Flags. No flags are defined yet, so set this to 0. + + + __u8 + rx_status + The status bits of the received message. See + for the possible status values. It is 0 if this message was transmitted, not + received, unless this is the reply to a transmitted message. In that case both + rx_status and tx_status + are set. + + + __u8 + tx_status + The status bits of the transmitted message. See + for the possible status values. It is 0 if this messages was received, not + transmitted. + + + __u8 + msg[16] + The message payload. For CEC_TRANSMIT this + is filled in by the application. The driver will fill this in for + CEC_RECEIVE and for CEC_TRANSMIT + it will be filled in with the payload of the reply message if + reply was set. + + + __u8 + reply + Wait until this message is replied. If reply + is 0 and the timeout is 0, then don't wait for a reply but + return after transmitting the message. If there was an error as indicated by a non-zero + tx_status field, then reply and + timeout are both set to 0 by the driver. Ignored by + CEC_RECEIVE. The case where reply is 0 + (this is the opcode for the Feature Abort message) and timeout + is non-zero is specifically allowed to send a message and wait up to timeout + milliseconds for a Feature Abort reply. In this case rx_status + will either be set to CEC_RX_STATUS_TIMEOUT or + CEC_RX_STATUS_FEATURE_ABORT. + + + __u8 + tx_arb_lost_cnt + A counter of the number of transmit attempts that resulted in the + Arbitration Lost error. This is only set if the hardware supports this, otherwise + it is always 0. This counter is only valid if the CEC_TX_STATUS_ARB_LOST + status bit is set. + + + __u8 + tx_nack_cnt + A counter of the number of transmit attempts that resulted in the + Not Acknowledged error. This is only set if the hardware supports this, otherwise + it is always 0. This counter is only valid if the CEC_TX_STATUS_NACK + status bit is set. + + + __u8 + tx_low_drive_cnt + A counter of the number of transmit attempts that resulted in the + Arbitration Lost error. This is only set if the hardware supports this, otherwise + it is always 0. This counter is only valid if the CEC_TX_STATUS_LOW_DRIVE + status bit is set. + + + __u8 + tx_error_cnt + A counter of the number of transmit errors other than Arbitration Lost + or Not Acknowledged. This is only set if the hardware supports this, otherwise + it is always 0. This counter is only valid if the CEC_TX_STATUS_ERROR + status bit is set. + + + +
+ + + CEC Transmit Status + + &cs-def; + + + CEC_TX_STATUS_OK + 0x01 + The message was transmitted successfully. This is mutually exclusive with + CEC_TX_STATUS_MAX_RETRIES. Other bits can still be set if + earlier attempts met with failure before the transmit was eventually successful. + + + CEC_TX_STATUS_ARB_LOST + 0x02 + CEC line arbitration was lost. + + + CEC_TX_STATUS_NACK + 0x04 + Message was not acknowledged. + + + CEC_TX_STATUS_LOW_DRIVE + 0x08 + Low drive was detected on the CEC bus. This indicates that a follower + detected an error on the bus and requests a retransmission. + + + CEC_TX_STATUS_ERROR + 0x10 + Some error occurred. This is used for any errors that do not + fit the previous two, either because the hardware could not tell + which error occurred, or because the hardware tested for other conditions + besides those two. + + + CEC_TX_STATUS_MAX_RETRIES + 0x20 + The transmit failed after one or more retries. This status bit is mutually + exclusive with CEC_TX_STATUS_OK. Other bits can still be set + to explain which failures were seen. + + + +
+ + + CEC Receive Status + + &cs-def; + + + CEC_RX_STATUS_OK + 0x01 + The message was received successfully. + + + CEC_RX_STATUS_TIMEOUT + 0x02 + The reply to an earlier transmitted message timed out. + + + CEC_RX_STATUS_FEATURE_ABORT + 0x04 + The message was received successfully but the reply was + CEC_MSG_FEATURE_ABORT. This status is only + set if this message was the reply to an earlier transmitted + message. + + + +
+ + + + &return-value; + + diff --git a/Documentation/DocBook/media/v4l/io.xml b/Documentation/DocBook/media/v4l/io.xml index e09025db92bd..21a3dde8f95d 100644 --- a/Documentation/DocBook/media/v4l/io.xml +++ b/Documentation/DocBook/media/v4l/io.xml @@ -88,7 +88,7 @@ function. capabilities field of &v4l2-capability; returned by the &VIDIOC-QUERYCAP; ioctl is set. There are two streaming methods, to determine if the memory mapping flavor is -supported applications must call the &VIDIOC-REQBUFS; ioctl. +supported applications must call the &VIDIOC-REQBUFS; ioctl with the memory type set to V4L2_MEMORY_MMAP. Streaming is an I/O method where only pointers to buffers are exchanged between application and driver, the data itself is not @@ -369,7 +369,7 @@ rest should be evident. capabilities field of &v4l2-capability; returned by the &VIDIOC-QUERYCAP; ioctl is set. If the particular user pointer method (not only memory mapping) is supported must be -determined by calling the &VIDIOC-REQBUFS; ioctl. +determined by calling the &VIDIOC-REQBUFS; ioctl with the memory type set to V4L2_MEMORY_USERPTR. This I/O method combines advantages of the read/write and memory mapping methods. Buffers (planes) are allocated by the application diff --git a/Documentation/DocBook/media/v4l/lirc_device_interface.xml b/Documentation/DocBook/media/v4l/lirc_device_interface.xml index 34cada2ca710..71f9dbb81ec7 100644 --- a/Documentation/DocBook/media/v4l/lirc_device_interface.xml +++ b/Documentation/DocBook/media/v4l/lirc_device_interface.xml @@ -157,7 +157,7 @@ on working with the default settings initially. LIRC_SET_{SEND,REC}_CARRIER - Set send/receive carrier (in Hz). + Set send/receive carrier (in Hz). Return 0 on success. diff --git a/Documentation/DocBook/media/v4l/media-types.xml b/Documentation/DocBook/media/v4l/media-types.xml index 5e3f20fdcf17..95aa1f9c836a 100644 --- a/Documentation/DocBook/media/v4l/media-types.xml +++ b/Documentation/DocBook/media/v4l/media-types.xml @@ -121,6 +121,70 @@ MEDIA_ENT_F_AUDIO_MIXER Audio Mixer Function Entity. + + MEDIA_ENT_F_PROC_VIDEO_COMPOSER + Video composer (blender). An entity capable of video + composing must have at least two sink pads and one source + pad, and composes input video frames onto output video + frames. Composition can be performed using alpha blending, + color keying, raster operations (ROP), stitching or any other + means. + + + + MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER + Video pixel formatter. An entity capable of pixel formatting + must have at least one sink pad and one source pad. Read + pixel formatters read pixels from memory and perform a subset + of unpacking, cropping, color keying, alpha multiplication + and pixel encoding conversion. Write pixel formatters perform + a subset of dithering, pixel encoding conversion and packing + and write pixels to memory. + + + + MEDIA_ENT_F_PROC_VIDEO_PIXEL_ENC_CONV + Video pixel encoding converter. An entity capable of pixel + enconding conversion must have at least one sink pad and one + source pad, and convert the encoding of pixels received on + its sink pad(s) to a different encoding output on its source + pad(s). Pixel encoding conversion includes but isn't limited + to RGB to/from HSV, RGB to/from YUV and CFA (Bayer) to RGB + conversions. + + + + MEDIA_ENT_F_PROC_VIDEO_LUT + Video look-up table. An entity capable of video lookup table + processing must have one sink pad and one source pad. It uses + the values of the pixels received on its sink pad to look up + entries in internal tables and output them on its source pad. + The lookup processing can be performed on all components + separately or combine them for multi-dimensional table + lookups. + + + + MEDIA_ENT_F_PROC_VIDEO_SCALER + Video scaler. An entity capable of video scaling must have + at least one sink pad and one source pad, and scale the + video frame(s) received on its sink pad(s) to a different + resolution output on its source pad(s). The range of + supported scaling ratios is entity-specific and can differ + between the horizontal and vertical directions (in particular + scaling can be supported in one direction only). Binning and + skipping are considered as scaling. + + + + MEDIA_ENT_F_PROC_VIDEO_STATISTICS + Video statistics computation (histogram, 3A, ...). An entity + capable of statistics computation must have one sink pad and + one source pad. It computes statistics over the frames + received on its sink pad and outputs the statistics data on + its source pad. + + diff --git a/Documentation/DocBook/media/v4l/pixfmt-z16.xml b/Documentation/DocBook/media/v4l/pixfmt-z16.xml index 3d87e4bf87b8..1d9cb1684bd3 100644 --- a/Documentation/DocBook/media/v4l/pixfmt-z16.xml +++ b/Documentation/DocBook/media/v4l/pixfmt-z16.xml @@ -5,7 +5,7 @@ V4L2_PIX_FMT_Z16 - Interleaved grey-scale image, e.g. from a stereo-pair + 16-bit depth data with distance values at each pixel Description diff --git a/Documentation/DocBook/media/v4l/vidioc-reqbufs.xml b/Documentation/DocBook/media/v4l/vidioc-reqbufs.xml index 0f193fda0470..6f529e100ea4 100644 --- a/Documentation/DocBook/media/v4l/vidioc-reqbufs.xml +++ b/Documentation/DocBook/media/v4l/vidioc-reqbufs.xml @@ -6,7 +6,7 @@ VIDIOC_REQBUFS - Initiate Memory Mapping or User Pointer I/O + Initiate Memory Mapping, User Pointer or DMA Buffer I/O diff --git a/Documentation/DocBook/media_api.tmpl b/Documentation/DocBook/media_api.tmpl index 7b77e0f7b87d..a2765d8ad05c 100644 --- a/Documentation/DocBook/media_api.tmpl +++ b/Documentation/DocBook/media_api.tmpl @@ -75,7 +75,7 @@ The media infrastructure API was designed to control such - devices. It is divided into four parts. + devices. It is divided into five parts. The first part covers radio, video capture and output, cameras, analog TV devices and codecs. The second part covers the @@ -87,6 +87,7 @@ . The third part covers the Remote Controller API. The fourth part covers the Media Controller API. + The fifth part covers the CEC (Consumer Electronics Control) API. It should also be noted that a media device may also have audio components, like mixers, PCM capture, PCM playback, etc, which are controlled via ALSA API. @@ -107,6 +108,9 @@ &sub-media-controller; + +&sub-cec-api; + &sub-gen-errors; diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx new file mode 100644 index 000000000000..d8d13c92a178 --- /dev/null +++ b/Documentation/Makefile.sphinx @@ -0,0 +1,78 @@ +# -*- makefile -*- +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXBUILD = sphinx-build +SPHINXOPTS = +PAPER = +BUILDDIR = $(obj)/output + +# User-friendly check for sphinx-build +HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi) + +ifeq ($(HAVE_SPHINX),0) + +.DEFAULT: + $(warning The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed and in PATH, or set the SPHINXBUILD make variable to point to the full path of the '$(SPHINXBUILD)' executable.) + @echo " SKIP Sphinx $@ target." + +else ifneq ($(DOCBOOKS),) + +# Skip Sphinx build if the user explicitly requested DOCBOOKS. +.DEFAULT: + @echo " SKIP Sphinx $@ target (DOCBOOKS specified)." + +else # HAVE_SPHINX + +# User-friendly check for rst2pdf +HAVE_RST2PDF := $(shell if python -c "import rst2pdf" >/dev/null 2>&1; then echo 1; else echo 0; fi) + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +KERNELDOC = $(srctree)/scripts/kernel-doc +KERNELDOC_CONF = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC) +ALLSPHINXOPTS = -D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) -d $(BUILDDIR)/.doctrees $(KERNELDOC_CONF) $(PAPEROPT_$(PAPER)) -c $(srctree)/$(src) $(SPHINXOPTS) $(srctree)/$(src) +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +quiet_cmd_sphinx = SPHINX $@ + cmd_sphinx = $(SPHINXBUILD) -b $2 $(ALLSPHINXOPTS) $(BUILDDIR)/$2 + +htmldocs: + $(call cmd,sphinx,html) + +pdfdocs: +ifeq ($(HAVE_RST2PDF),0) + $(warning The Python 'rst2pdf' module was not found. Make sure you have the module installed to produce PDF output.) + @echo " SKIP Sphinx $@ target." +else # HAVE_RST2PDF + $(call cmd,sphinx,pdf) +endif # HAVE_RST2PDF + +epubdocs: + $(call cmd,sphinx,epub) + +xmldocs: + $(call cmd,sphinx,xml) + +# no-ops for the Sphinx toolchain +sgmldocs: +psdocs: +mandocs: +installmandocs: +cleanmediadocs: + +cleandocs: + $(Q)rm -rf $(BUILDDIR) + +dochelp: + @echo ' Linux kernel internal documentation in different formats (Sphinx):' + @echo ' htmldocs - HTML' + @echo ' pdfdocs - PDF' + @echo ' epubdocs - EPUB' + @echo ' xmldocs - XML' + @echo ' cleandocs - clean all generated files' + +endif # HAVE_SPHINX diff --git a/Documentation/RCU/Design/Requirements/Requirements.html b/Documentation/RCU/Design/Requirements/Requirements.html index e7e24b3e86e2..ece410f40436 100644 --- a/Documentation/RCU/Design/Requirements/Requirements.html +++ b/Documentation/RCU/Design/Requirements/Requirements.html @@ -2391,6 +2391,41 @@ and RCU_NONIDLE() on the other while inspecting idle-loop code. Steven Rostedt supplied _rcuidle event tracing, which is used quite heavily in the idle loop. +However, there are some restrictions on the code placed within +RCU_NONIDLE(): + +
    +
  1. Blocking is prohibited. + In practice, this is not a serious restriction given that idle + tasks are prohibited from blocking to begin with. +
  2. Although nesting RCU_NONIDLE() is permited, they cannot + nest indefinitely deeply. + However, given that they can be nested on the order of a million + deep, even on 32-bit systems, this should not be a serious + restriction. + This nesting limit would probably be reached long after the + compiler OOMed or the stack overflowed. +
  3. Any code path that enters RCU_NONIDLE() must sequence + out of that same RCU_NONIDLE(). + For example, the following is grossly illegal: + +
    + 
    + 1     RCU_NONIDLE({
+ 2       do_something();
+ 3       goto bad_idea;  /* BUG!!! */
+ 4       do_something_else();});
+ 5   bad_idea:
+
    +
    + +

    + It is just as illegal to transfer control into the middle of + RCU_NONIDLE()'s argument. + Yes, in theory, you could transfer in as long as you also + transferred out, but in practice you could also expect to get sharply + worded review comments. +

It is similarly socially unacceptable to interrupt an diff --git a/Documentation/RCU/stallwarn.txt b/Documentation/RCU/stallwarn.txt index 0f7fb4298e7e..e93d04133fe7 100644 --- a/Documentation/RCU/stallwarn.txt +++ b/Documentation/RCU/stallwarn.txt @@ -49,7 +49,7 @@ rcupdate.rcu_task_stall_timeout This boot/sysfs parameter controls the RCU-tasks stall warning interval. A value of zero or less suppresses RCU-tasks stall warnings. A positive value sets the stall-warning interval - in jiffies. An RCU-tasks stall warning starts wtih the line: + in jiffies. An RCU-tasks stall warning starts with the line: INFO: rcu_tasks detected stalls on tasks: diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt index 111770ffa10e..204422719197 100644 --- a/Documentation/RCU/whatisRCU.txt +++ b/Documentation/RCU/whatisRCU.txt @@ -5,6 +5,9 @@ to start learning about RCU: 2. What is RCU? Part 2: Usage http://lwn.net/Articles/263130/ 3. RCU part 3: the RCU API http://lwn.net/Articles/264090/ 4. The RCU API, 2010 Edition http://lwn.net/Articles/418853/ + 2010 Big API Table http://lwn.net/Articles/419086/ +5. The RCU API, 2014 Edition http://lwn.net/Articles/609904/ + 2014 Big API Table http://lwn.net/Articles/609973/ What is RCU? diff --git a/Documentation/acpi/aml-debugger.txt b/Documentation/acpi/aml-debugger.txt new file mode 100644 index 000000000000..5f62aa4a493b --- /dev/null +++ b/Documentation/acpi/aml-debugger.txt @@ -0,0 +1,66 @@ +The AML Debugger + +Copyright (C) 2016, Intel Corporation +Author: Lv Zheng + + +This document describes the usage of the AML debugger embedded in the Linux +kernel. + +1. Build the debugger + + The following kernel configuration items are required to enable the AML + debugger interface from the Linux kernel: + + CONFIG_ACPI_DEBUGGER=y + CONFIG_ACPI_DEBUGGER_USER=m + + The userspace utlities can be built from the kernel source tree using + the following commands: + + $ cd tools + $ make acpi + + The resultant userspace tool binary is then located at: + + tools/acpi/power/acpi/acpidbg/acpidbg + + It can be installed to system directories by running "make install" (as a + sufficiently privileged user). + +2. Start the userspace debugger interface + + After booting the kernel with the debugger built-in, the debugger can be + started by using the following commands: + + # mount -t debugfs none /sys/kernel/debug + # modprobe acpi_dbg + # tools/acpi/power/acpi/acpidbg/acpidbg + + That spawns the interactive AML debugger environment where you can execute + debugger commands. + + The commands are documented in the "ACPICA Overview and Programmer Reference" + that can be downloaded from + + https://acpica.org/documentation + + The detailed debugger commands reference is located in Chapter 12 "ACPICA + Debugger Reference". The "help" command can be used for a quick reference. + +3. Stop the userspace debugger interface + + The interactive debugger interface can be closed by pressing Ctrl+C or using + the "quit" or "exit" commands. When finished, unload the module with: + + # rmmod acpi_dbg + + The module unloading may fail if there is an acpidbg instance running. + +4. Run the debugger in a script + + It may be useful to run the AML debugger in a test script. "acpidbg" supports + this in a special "batch" mode. For example, the following command outputs + the entire ACPI namespace: + + # acpidbg -b "namespace" diff --git a/Documentation/acpi/linuxized-acpica.txt b/Documentation/acpi/linuxized-acpica.txt new file mode 100644 index 000000000000..defe2eec5331 --- /dev/null +++ b/Documentation/acpi/linuxized-acpica.txt @@ -0,0 +1,262 @@ +Linuxized ACPICA - Introduction to ACPICA Release Automation + +Copyright (C) 2013-2016, Intel Corporation +Author: Lv Zheng + + +Abstract: + +This document describes the ACPICA project and the relationship between +ACPICA and Linux. It also describes how ACPICA code in drivers/acpi/acpica, +include/acpi and tools/power/acpi is automatically updated to follow the +upstream. + + +1. ACPICA Project + + The ACPI Component Architecture (ACPICA) project provides an operating + system (OS)-independent reference implementation of the Advanced + Configuration and Power Interface Specification (ACPI). It has been + adapted by various host OSes. By directly integrating ACPICA, Linux can + also benefit from the application experiences of ACPICA from other host + OSes. + + The homepage of ACPICA project is: www.acpica.org, it is maintained and + supported by Intel Corporation. + + The following figure depicts the Linux ACPI subystem where the ACPICA + adaptation is included: + + +---------------------------------------------------------+ + | | + | +---------------------------------------------------+ | + | | +------------------+ | | + | | | Table Management | | | + | | +------------------+ | | + | | +----------------------+ | | + | | | Namespace Management | | | + | | +----------------------+ | | + | | +------------------+ ACPICA Components | | + | | | Event Management | | | + | | +------------------+ | | + | | +---------------------+ | | + | | | Resource Management | | | + | | +---------------------+ | | + | | +---------------------+ | | + | | | Hardware Management | | | + | | +---------------------+ | | + | +---------------------------------------------------+ | | + | | | +------------------+ | | | + | | | | OS Service Layer | | | | + | | | +------------------+ | | | + | | +-------------------------------------------------|-+ | + | | +--------------------+ | | + | | | Device Enumeration | | | + | | +--------------------+ | | + | | +------------------+ | | + | | | Power Management | | | + | | +------------------+ Linux/ACPI Components | | + | | +--------------------+ | | + | | | Thermal Management | | | + | | +--------------------+ | | + | | +--------------------------+ | | + | | | Drivers for ACPI Devices | | | + | | +--------------------------+ | | + | | +--------+ | | + | | | ...... | | | + | | +--------+ | | + | +---------------------------------------------------+ | + | | + +---------------------------------------------------------+ + + Figure 1. Linux ACPI Software Components + + NOTE: + A. OS Service Layer - Provided by Linux to offer OS dependent + implementation of the predefined ACPICA interfaces (acpi_os_*). + include/acpi/acpiosxf.h + drivers/acpi/osl.c + include/acpi/platform + include/asm/acenv.h + B. ACPICA Functionality - Released from ACPICA code base to offer + OS independent implementation of the ACPICA interfaces (acpi_*). + drivers/acpi/acpica + include/acpi/ac*.h + tools/power/acpi + C. Linux/ACPI Functionality - Providing Linux specific ACPI + functionality to the other Linux kernel subsystems and user space + programs. + drivers/acpi + include/linux/acpi.h + include/linux/acpi*.h + include/acpi + tools/power/acpi + D. Architecture Specific ACPICA/ACPI Functionalities - Provided by the + ACPI subsystem to offer architecture specific implementation of the + ACPI interfaces. They are Linux specific components and are out of + the scope of this document. + include/asm/acpi.h + include/asm/acpi*.h + arch/*/acpi + +2. ACPICA Release + + The ACPICA project maintains its code base at the following repository URL: + https://github.com/acpica/acpica.git. As a rule, a release is made every + month. + + As the coding style adopted by the ACPICA project is not acceptable by + Linux, there is a release process to convert the ACPICA git commits into + Linux patches. The patches generated by this process are referred to as + "linuxized ACPICA patches". The release process is carried out on a local + copy the ACPICA git repository. Each commit in the monthly release is + converted into a linuxized ACPICA patch. Together, they form the montly + ACPICA release patchset for the Linux ACPI community. This process is + illustrated in the following figure: + + +-----------------------------+ + | acpica / master (-) commits | + +-----------------------------+ + /|\ | + | \|/ + | /---------------------\ +----------------------+ + | < Linuxize repo Utility >-->| old linuxized acpica |--+ + | \---------------------/ +----------------------+ | + | | + /---------\ | + < git reset > \ + \---------/ \ + /|\ /+-+ + | / | + +-----------------------------+ | | + | acpica / master (+) commits | | | + +-----------------------------+ | | + | | | + \|/ | | + /-----------------------\ +----------------------+ | | + < Linuxize repo Utilities >-->| new linuxized acpica |--+ | + \-----------------------/ +----------------------+ | + \|/ + +--------------------------+ /----------------------\ + | Linuxized ACPICA Patches |<----------------< Linuxize patch Utility > + +--------------------------+ \----------------------/ + | + \|/ + /---------------------------\ + < Linux ACPI Community Review > + \---------------------------/ + | + \|/ + +-----------------------+ /------------------\ +----------------+ + | linux-pm / linux-next |-->< Linux Merge Window >-->| linux / master | + +-----------------------+ \------------------/ +----------------+ + + Figure 2. ACPICA -> Linux Upstream Process + + NOTE: + A. Linuxize Utilities - Provided by the ACPICA repository, including a + utility located in source/tools/acpisrc folder and a number of + scripts located in generate/linux folder. + B. acpica / master - "master" branch of the git repository at + . + C. linux-pm / linux-next - "linux-next" branch of the git repository at + . + D. linux / master - "master" branch of the git repository at + . + + Before the linuxized ACPICA patches are sent to the Linux ACPI community + for review, there is a quality ensurance build test process to reduce + porting issues. Currently this build process only takes care of the + following kernel configuration options: + CONFIG_ACPI/CONFIG_ACPI_DEBUG/CONFIG_ACPI_DEBUGGER + +3. ACPICA Divergences + + Ideally, all of the ACPICA commits should be converted into Linux patches + automatically without manual modifications, the "linux / master" tree should + contain the ACPICA code that exactly corresponds to the ACPICA code + contained in "new linuxized acpica" tree and it should be possible to run + the release process fully automatically. + + As a matter of fact, however, there are source code differences between + the ACPICA code in Linux and the upstream ACPICA code, referred to as + "ACPICA Divergences". + + The various sources of ACPICA divergences include: + 1. Legacy divergences - Before the current ACPICA release process was + established, there already had been divergences between Linux and + ACPICA. Over the past several years those divergences have been greatly + reduced, but there still are several ones and it takes time to figure + out the underlying reasons for their existence. + 2. Manual modifications - Any manual modification (eg. coding style fixes) + made directly in the Linux sources obviously hurts the ACPICA release + automation. Thus it is recommended to fix such issues in the ACPICA + upstream source code and generate the linuxized fix using the ACPICA + release utilities (please refer to Section 4 below for the details). + 3. Linux specific features - Sometimes it's impossible to use the + current ACPICA APIs to implement features required by the Linux kernel, + so Linux developers occasionaly have to change ACPICA code directly. + Those changes may not be acceptable by ACPICA upstream and in such cases + they are left as committed ACPICA divergences unless the ACPICA side can + implement new mechanisms as replacements for them. + 4. ACPICA release fixups - ACPICA only tests commits using a set of the + user space simulation utilies, thus the linuxized ACPICA patches may + break the Linux kernel, leaving us build/boot failures. In order to + avoid breaking Linux bisection, fixes are applied directly to the + linuxized ACPICA patches during the release process. When the release + fixups are backported to the upstream ACPICA sources, they must follow + the upstream ACPICA rules and so further modifications may appear. + That may result in the appearance of new divergences. + 5. Fast tracking of ACPICA commits - Some ACPICA commits are regression + fixes or stable-candidate material, so they are applied in advance with + respect to the ACPICA release process. If such commits are reverted or + rebased on the ACPICA side in order to offer better solutions, new ACPICA + divergences are generated. + +4. ACPICA Development + + This paragraph guides Linux developers to use the ACPICA upstream release + utilities to obtain Linux patches corresponding to upstream ACPICA commits + before they become available from the ACPICA release process. + + 1. Cherry-pick an ACPICA commit + + First you need to git clone the ACPICA repository and the ACPICA change + you want to cherry pick must be committed into the local repository. + + Then the gen-patch.sh command can help to cherry-pick an ACPICA commit + from the ACPICA local repository: + + $ git clone https://github.com/acpica/acpica + $ cd acpica + $ generate/linux/gen-patch.sh -u [commit ID] + + Here the commit ID is the ACPICA local repository commit ID you want to + cherry pick. It can be omitted if the commit is "HEAD". + + 2. Cherry-pick recent ACPICA commits + + Sometimes you need to rebase your code on top of the most recent ACPICA + changes that haven't been applied to Linux yet. + + You can generate the ACPICA release series yourself and rebase your code on + top of the generated ACPICA release patches: + + $ git clone https://github.com/acpica/acpica + $ cd acpica + $ generate/linux/make-patches.sh -u [commit ID] + + The commit ID should be the last ACPICA commit accepted by Linux. Usually, + it is the commit modifying ACPI_CA_VERSION. It can be found by executing + "git blame source/include/acpixf.h" and referencing the line that contains + "ACPI_CA_VERSION". + + 3. Inspect the current divergences + + If you have local copies of both Linux and upstream ACPICA, you can generate + a diff file indicating the state of the current divergences: + + # git clone https://github.com/acpica/acpica + # git clone http://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git + # cd acpica + # generate/linux/divergences.sh -s ../linux diff --git a/Documentation/acpi/ssdt-overlays.txt b/Documentation/acpi/ssdt-overlays.txt new file mode 100644 index 000000000000..5ae13f161ea2 --- /dev/null +++ b/Documentation/acpi/ssdt-overlays.txt @@ -0,0 +1,172 @@ + +In order to support ACPI open-ended hardware configurations (e.g. development +boards) we need a way to augment the ACPI configuration provided by the firmware +image. A common example is connecting sensors on I2C / SPI buses on development +boards. + +Although this can be accomplished by creating a kernel platform driver or +recompiling the firmware image with updated ACPI tables, neither is practical: +the former proliferates board specific kernel code while the latter requires +access to firmware tools which are often not publicly available. + +Because ACPI supports external references in AML code a more practical +way to augment firmware ACPI configuration is by dynamically loading +user defined SSDT tables that contain the board specific information. + +For example, to enumerate a Bosch BMA222E accelerometer on the I2C bus of the +Minnowboard MAX development board exposed via the LSE connector [1], the +following ASL code can be used: + +DefinitionBlock ("minnowmax.aml", "SSDT", 1, "Vendor", "Accel", 0x00000003) +{ + External (\_SB.I2C6, DeviceObj) + + Scope (\_SB.I2C6) + { + Device (STAC) + { + Name (_ADR, Zero) + Name (_HID, "BMA222E") + + Method (_CRS, 0, Serialized) + { + Name (RBUF, ResourceTemplate () + { + I2cSerialBus (0x0018, ControllerInitiated, 0x00061A80, + AddressingMode7Bit, "\\_SB.I2C6", 0x00, + ResourceConsumer, ,) + GpioInt (Edge, ActiveHigh, Exclusive, PullDown, 0x0000, + "\\_SB.GPO2", 0x00, ResourceConsumer, , ) + { // Pin list + 0 + } + }) + Return (RBUF) + } + } + } +} + +which can then be compiled to AML binary format: + +$ iasl minnowmax.asl + +Intel ACPI Component Architecture +ASL Optimizing Compiler version 20140214-64 [Mar 29 2014] +Copyright (c) 2000 - 2014 Intel Corporation + +ASL Input: minnomax.asl - 30 lines, 614 bytes, 7 keywords +AML Output: minnowmax.aml - 165 bytes, 6 named objects, 1 executable opcodes + +[1] http://wiki.minnowboard.org/MinnowBoard_MAX#Low_Speed_Expansion_Connector_.28Top.29 + +The resulting AML code can then be loaded by the kernel using one of the methods +below. + +== Loading ACPI SSDTs from initrd == + +This option allows loading of user defined SSDTs from initrd and it is useful +when the system does not support EFI or when there is not enough EFI storage. + +It works in a similar way with initrd based ACPI tables override/upgrade: SSDT +aml code must be placed in the first, uncompressed, initrd under the +"kernel/firmware/acpi" path. Multiple files can be used and this will translate +in loading multiple tables. Only SSDT and OEM tables are allowed. See +initrd_table_override.txt for more details. + +Here is an example: + +# Add the raw ACPI tables to an uncompressed cpio archive. +# They must be put into a /kernel/firmware/acpi directory inside the +# cpio archive. +# The uncompressed cpio archive must be the first. +# Other, typically compressed cpio archives, must be +# concatenated on top of the uncompressed one. +mkdir -p kernel/firmware/acpi +cp ssdt.aml kernel/firmware/acpi + +# Create the uncompressed cpio archive and concatenate the original initrd +# on top: +find kernel | cpio -H newc --create > /boot/instrumented_initrd +cat /boot/initrd >>/boot/instrumented_initrd + +== Loading ACPI SSDTs from EFI variables == + +This is the preferred method, when EFI is supported on the platform, because it +allows a persistent, OS independent way of storing the user defined SSDTs. There +is also work underway to implement EFI support for loading user defined SSDTs +and using this method will make it easier to convert to the EFI loading +mechanism when that will arrive. + +In order to load SSDTs from an EFI variable the efivar_ssdt kernel command line +parameter can be used. The argument for the option is the variable name to +use. If there are multiple variables with the same name but with different +vendor GUIDs, all of them will be loaded. + +In order to store the AML code in an EFI variable the efivarfs filesystem can be +used. It is enabled and mounted by default in /sys/firmware/efi/efivars in all +recent distribution. + +Creating a new file in /sys/firmware/efi/efivars will automatically create a new +EFI variable. Updating a file in /sys/firmware/efi/efivars will update the EFI +variable. Please note that the file name needs to be specially formatted as +"Name-GUID" and that the first 4 bytes in the file (little-endian format) +represent the attributes of the EFI variable (see EFI_VARIABLE_MASK in +include/linux/efi.h). Writing to the file must also be done with one write +operation. + +For example, you can use the following bash script to create/update an EFI +variable with the content from a given file: + +#!/bin/sh -e + +while ! [ -z "$1" ]; do + case "$1" in + "-f") filename="$2"; shift;; + "-g") guid="$2"; shift;; + *) name="$1";; + esac + shift +done + +usage() +{ + echo "Syntax: ${0##*/} -f filename [ -g guid ] name" + exit 1 +} + +[ -n "$name" -a -f "$filename" ] || usage + +EFIVARFS="/sys/firmware/efi/efivars" + +[ -d "$EFIVARFS" ] || exit 2 + +if stat -tf $EFIVARFS | grep -q -v de5e81e4; then + mount -t efivarfs none $EFIVARFS +fi + +# try to pick up an existing GUID +[ -n "$guid" ] || guid=$(find "$EFIVARFS" -name "$name-*" | head -n1 | cut -f2- -d-) + +# use a randomly generated GUID +[ -n "$guid" ] || guid="$(cat /proc/sys/kernel/random/uuid)" + +# efivarfs expects all of the data in one write +tmp=$(mktemp) +/bin/echo -ne "\007\000\000\000" | cat - $filename > $tmp +dd if=$tmp of="$EFIVARFS/$name-$guid" bs=$(stat -c %s $tmp) +rm $tmp + +== Loading ACPI SSDTs from configfs == + +This option allows loading of user defined SSDTs from userspace via the configfs +interface. The CONFIG_ACPI_CONFIGFS option must be select and configfs must be +mounted. In the following examples, we assume that configfs has been mounted in +/config. + +New tables can be loading by creating new directories in /config/acpi/table/ and +writing the SSDT aml code in the aml attribute: + +cd /config/acpi/table +mkdir my_ssdt +cat ~/ssdt.aml > my_ssdt/aml diff --git a/Documentation/arm64/acpi_object_usage.txt b/Documentation/arm64/acpi_object_usage.txt index a6e1a1805e51..c77010c5c1f0 100644 --- a/Documentation/arm64/acpi_object_usage.txt +++ b/Documentation/arm64/acpi_object_usage.txt @@ -13,14 +13,14 @@ For ACPI on arm64, tables also fall into the following categories: -- Required: DSDT, FADT, GTDT, MADT, MCFG, RSDP, SPCR, XSDT - -- Recommended: BERT, EINJ, ERST, HEST, SSDT + -- Recommended: BERT, EINJ, ERST, HEST, PCCT, SSDT - -- Optional: BGRT, CPEP, CSRT, DRTM, ECDT, FACS, FPDT, MCHI, MPST, - MSCT, RASF, SBST, SLIT, SPMI, SRAT, TCPA, TPM2, UEFI - - -- Not supported: BOOT, DBG2, DBGP, DMAR, ETDT, HPET, IBFT, IVRS, - LPIT, MSDM, RSDT, SLIC, WAET, WDAT, WDRT, WPBT + -- Optional: BGRT, CPEP, CSRT, DBG2, DRTM, ECDT, FACS, FPDT, IORT, + MCHI, MPST, MSCT, NFIT, PMTT, RASF, SBST, SLIT, SPMI, SRAT, STAO, + TCPA, TPM2, UEFI, XENV + -- Not supported: BOOT, DBGP, DMAR, ETDT, HPET, IBFT, IVRS, LPIT, + MSDM, OEMx, PSDT, RSDT, SLIC, WAET, WDAT, WDRT, WPBT Table Usage for ARMv8 Linux ----- ---------------------------------------------------------------- @@ -50,7 +50,8 @@ CSRT Signature Reserved (signature == "CSRT") DBG2 Signature Reserved (signature == "DBG2") == DeBuG port table 2 == - Microsoft only table, will not be supported. + License has changed and should be usable. Optional if used instead + of earlycon= on the command line. DBGP Signature Reserved (signature == "DBGP") == DeBuG Port table == @@ -133,10 +134,11 @@ GTDT Section 5.2.24 (signature == "GTDT") HEST Section 18.3.2 (signature == "HEST") == Hardware Error Source Table == - Until further error source types are defined, use only types 6 (AER - Root Port), 7 (AER Endpoint), 8 (AER Bridge), or 9 (Generic Hardware - Error Source). Firmware first error handling is possible if and only - if Trusted Firmware is being used on arm64. + ARM-specific error sources have been defined; please use those or the + PCI types such as type 6 (AER Root Port), 7 (AER Endpoint), or 8 (AER + Bridge), or use type 9 (Generic Hardware Error Source). Firmware first + error handling is possible if and only if Trusted Firmware is being + used on arm64. Must be supplied if RAS support is provided by the platform. It is recommended this table be supplied. @@ -149,20 +151,30 @@ IBFT Signature Reserved (signature == "IBFT") == iSCSI Boot Firmware Table == Microsoft defined table, support TBD. +IORT Signature Reserved (signature == "IORT") + == Input Output Remapping Table == + arm64 only table, required in order to describe IO topology, SMMUs, + and GIC ITSs, and how those various components are connected together, + such as identifying which components are behind which SMMUs/ITSs. + This table will only be required on certain SBSA platforms (e.g., + when using GICv3-ITS and an SMMU); on SBSA Level 0 platforms, it + remains optional. + IVRS Signature Reserved (signature == "IVRS") == I/O Virtualization Reporting Structure == x86_64 (AMD) only table, will not be supported. LPIT Signature Reserved (signature == "LPIT") == Low Power Idle Table == - x86 only table as of ACPI 5.1; future versions have been adapted for - use with ARM and will be recommended in order to support ACPI power - management. + x86 only table as of ACPI 5.1; starting with ACPI 6.0, processor + descriptions and power states on ARM platforms should use the DSDT + and define processor container devices (_HID ACPI0010, Section 8.4, + and more specifically 8.4.3 and and 8.4.4). MADT Section 5.2.12 (signature == "APIC") == Multiple APIC Description Table == Required for arm64. Only the GIC interrupt controller structures - should be used (types 0xA - 0xE). + should be used (types 0xA - 0xF). MCFG Signature Reserved (signature == "MCFG") == Memory-mapped ConFiGuration space == @@ -176,14 +188,38 @@ MPST Section 5.2.21 (signature == "MPST") == Memory Power State Table == Optional, not currently supported. +MSCT Section 5.2.19 (signature == "MSCT") + == Maximum System Characteristic Table == + Optional, not currently supported. + MSDM Signature Reserved (signature == "MSDM") == Microsoft Data Management table == Microsoft only table, will not be supported. -MSCT Section 5.2.19 (signature == "MSCT") - == Maximum System Characteristic Table == +NFIT Section 5.2.25 (signature == "NFIT") + == NVDIMM Firmware Interface Table == + Optional, not currently supported. + +OEMx Signature of "OEMx" only + == OEM Specific Tables == + All tables starting with a signature of "OEM" are reserved for OEM + use. Since these are not meant to be of general use but are limited + to very specific end users, they are not recommended for use and are + not supported by the kernel for arm64. + +PCCT Section 14.1 (signature == "PCCT) + == Platform Communications Channel Table == + Recommend for use on arm64; use of PCC is recommended when using CPPC + to control performance and power for platform processors. + +PMTT Section 5.2.21.12 (signature == "PMTT") + == Platform Memory Topology Table == Optional, not currently supported. +PSDT Section 5.2.11.3 (signature == "PSDT") + == Persistent System Description Table == + Obsolete table, will not be supported. + RASF Section 5.2.20 (signature == "RASF") == RAS Feature table == Optional, not currently supported. @@ -195,7 +231,7 @@ RSDP Section 5.2.5 (signature == "RSD PTR") RSDT Section 5.2.7 (signature == "RSDT") == Root System Description Table == Since this table can only provide 32-bit addresses, it is deprecated - on arm64, and will not be used. + on arm64, and will not be used. If provided, it will be ignored. SBST Section 5.2.14 (signature == "SBST") == Smart Battery Subsystem Table == @@ -220,7 +256,7 @@ SPMI Signature Reserved (signature == "SPMI") SRAT Section 5.2.16 (signature == "SRAT") == System Resource Affinity Table == Optional, but if used, only the GICC Affinity structures are read. - To support NUMA, this table is required. + To support arm64 NUMA, this table is required. SSDT Section 5.2.11.2 (signature == "SSDT") == Secondary System Description Table == @@ -235,6 +271,11 @@ SSDT Section 5.2.11.2 (signature == "SSDT") These tables are optional, however. ACPI tables should contain only one DSDT but can contain many SSDTs. +STAO Signature Reserved (signature == "STAO") + == _STA Override table == + Optional, but only necessary in virtualized environments in order to + hide devices from guest OSs. + TCPA Signature Reserved (signature == "TCPA") == Trusted Computing Platform Alliance table == Optional, not currently supported, and may need changes to fully @@ -266,6 +307,10 @@ WPBT Signature Reserved (signature == "WPBT") == Windows Platform Binary Table == Microsoft only table, will not be supported. +XENV Signature Reserved (signature == "XENV") + == Xen project table == + Optional, used only by Xen at present. + XSDT Section 5.2.8 (signature == "XSDT") == eXtended System Description Table == Required for arm64. @@ -273,44 +318,46 @@ XSDT Section 5.2.8 (signature == "XSDT") ACPI Objects ------------ -The expectations on individual ACPI objects are discussed in the list that -follows: +The expectations on individual ACPI objects that are likely to be used are +shown in the list that follows; any object not explicitly mentioned below +should be used as needed for a particular platform or particular subsystem, +such as power management or PCI. Name Section Usage for ARMv8 Linux ---- ------------ ------------------------------------------------- -_ADR 6.1.1 Use as needed. - -_BBN 6.5.5 Use as needed; PCI-specific. +_CCA 6.2.17 This method must be defined for all bus masters + on arm64 -- there are no assumptions made about + whether such devices are cache coherent or not. + The _CCA value is inherited by all descendants of + these devices so it does not need to be repeated. + Without _CCA on arm64, the kernel does not know what + to do about setting up DMA for the device. -_BDN 6.5.3 Optional; not likely to be used on arm64. + NB: this method provides default cache coherency + attributes; the presence of an SMMU can be used to + modify that, however. For example, a master could + default to non-coherent, but be made coherent with + the appropriate SMMU configuration (see Table 17 of + the IORT specification, ARM Document DEN 0049B). -_CCA 6.2.17 This method should be defined for all bus masters - on arm64. While cache coherency is assumed, making - it explicit ensures the kernel will set up DMA as - it should. +_CID 6.1.2 Use as needed, see also _HID. -_CDM 6.2.1 Optional, to be used only for processor devices. +_CLS 6.1.3 Use as needed, see also _HID. -_CID 6.1.2 Use as needed. - -_CLS 6.1.3 Use as needed. +_CPC 8.4.7.1 Use as needed, power management specific. CPPC is + recommended on arm64. _CRS 6.2.2 Required on arm64. -_DCK 6.5.2 Optional; not likely to be used on arm64. +_CSD 8.4.2.2 Use as needed, used only in conjunction with _CST. + +_CST 8.4.2.1 Low power idle states (8.4.4) are recommended instead + of C-states. _DDN 6.1.4 This field can be used for a device name. However, it is meant for DOS device names (e.g., COM1), so be careful of its use across OSes. -_DEP 6.5.8 Use as needed. - -_DIS 6.2.3 Optional, for power management use. - -_DLM 5.7.5 Optional. - -_DMA 6.2.4 Optional. - _DSD 6.2.5 To be used with caution. If this object is used, try to use it within the constraints already defined by the Device Properties UUID. Only in rare circumstances @@ -325,20 +372,10 @@ _DSD 6.2.5 To be used with caution. If this object is used, try with the UEFI Forum; this may cause some iteration as more than one OS will be registering entries. -_DSM Do not use this method. It is not standardized, the +_DSM 9.1.1 Do not use this method. It is not standardized, the return values are not well documented, and it is currently a frequent source of error. -_DSW 7.2.1 Use as needed; power management specific. - -_EDL 6.3.1 Optional. - -_EJD 6.3.2 Optional. - -_EJx 6.3.3 Optional. - -_FIX 6.2.7 x86 specific, not used on arm64. - \_GL 5.7.1 This object is not to be used in hardware reduced mode, and therefore should not be used on arm64. @@ -349,35 +386,22 @@ _GLK 6.5.7 This object requires a global lock be defined; there \_GPE 5.3.1 This namespace is for x86 use only. Do not use it on arm64. -_GSB 6.2.7 Optional. - -_HID 6.1.5 Use as needed. This is the primary object to use in - device probing, though _CID and _CLS may also be used. - -_HPP 6.2.8 Optional, PCI specific. - -_HPX 6.2.9 Optional, PCI specific. - -_HRV 6.1.6 Optional, use as needed to clarify device behavior; in - some cases, this may be easier to use than _DSD. +_HID 6.1.5 This is the primary object to use in device probing, + though _CID and _CLS may also be used. _INI 6.5.1 Not required, but can be useful in setting up devices when UEFI leaves them in a state that may not be what the driver expects before it starts probing. -_IRC 7.2.15 Use as needed; power management specific. - -_LCK 6.3.4 Optional. - -_MAT 6.2.10 Optional; see also the MADT. +_LPI 8.4.4.3 Recommended for use with processor definitions (_HID + ACPI0010) on arm64. See also _RDI. -_MLS 6.1.7 Optional, but highly recommended for use in - internationalization. +_MLS 6.1.7 Highly recommended for use in internationalization. -_OFF 7.1.2 It is recommended to define this method for any device +_OFF 7.2.2 It is recommended to define this method for any device that can be turned on or off. -_ON 7.1.3 It is recommended to define this method for any device +_ON 7.2.3 It is recommended to define this method for any device that can be turned on or off. \_OS 5.7.3 This method will return "Linux" by default (this is @@ -398,122 +422,107 @@ _OSC 6.2.11 This method can be a global method in ACPI (i.e., by the kernel community, then register it with the UEFI Forum. -\_OSI 5.7.2 Deprecated on ARM64. Any invocation of this method - will print a warning on the console and return false. - That is, as far as ACPI firmware is concerned, _OSI - cannot be used to determine what sort of system is - being used or what functionality is provided. The - _OSC method is to be used instead. - -_OST 6.3.5 Optional. +\_OSI 5.7.2 Deprecated on ARM64. As far as ACPI firmware is + concerned, _OSI is not to be used to determine what + sort of system is being used or what functionality + is provided. The _OSC method is to be used instead. _PDC 8.4.1 Deprecated, do not use on arm64. \_PIC 5.8.1 The method should not be used. On arm64, the only interrupt model available is GIC. -_PLD 6.1.8 Optional. - \_PR 5.3.1 This namespace is for x86 use only on legacy systems. Do not use it on arm64. -_PRS 6.2.12 Optional. - _PRT 6.2.13 Required as part of the definition of all PCI root devices. -_PRW 7.2.13 Use as needed; power management specific. - -_PRx 7.2.8-11 Use as needed; power management specific. If _PR0 is +_PRx 7.3.8-11 Use as needed; power management specific. If _PR0 is defined, _PR3 must also be defined. -_PSC 7.2.6 Use as needed; power management specific. - -_PSE 7.2.7 Use as needed; power management specific. - -_PSW 7.2.14 Use as needed; power management specific. - -_PSx 7.2.2-5 Use as needed; power management specific. If _PS0 is +_PSx 7.3.2-5 Use as needed; power management specific. If _PS0 is defined, _PS3 must also be defined. If clocks or regulators need adjusting to be consistent with power usage, change them in these methods. -\_PTS 7.3.1 Use as needed; power management specific. - -_PXM 6.2.14 Optional. - -_REG 6.5.4 Use as needed. +_RDI 8.4.4.4 Recommended for use with processor definitions (_HID + ACPI0010) on arm64. This should only be used in + conjunction with _LPI. \_REV 5.7.4 Always returns the latest version of ACPI supported. -_RMV 6.3.6 Optional. - \_SB 5.3.1 Required on arm64; all devices must be defined in this namespace. -_SEG 6.5.6 Use as needed; PCI-specific. - -\_SI 5.3.1, Optional. - 9.1 - -_SLI 6.2.15 Optional; recommended when SLIT table is in use. +_SLI 6.2.15 Use is recommended when SLIT table is in use. _STA 6.3.7, It is recommended to define this method for any device - 7.1.4 that can be turned on or off. + 7.2.4 that can be turned on or off. See also the STAO table + that provides overrides to hide devices in virtualized + environments. -_SRS 6.2.16 Optional; see also _PRS. +_SRS 6.2.16 Use as needed; see also _PRS. _STR 6.1.10 Recommended for conveying device names to end users; this is preferred over using _DDN. _SUB 6.1.9 Use as needed; _HID or _CID are preferred. -_SUN 6.1.11 Optional. - -\_Sx 7.3.2 Use as needed; power management specific. - -_SxD 7.2.16-19 Use as needed; power management specific. - -_SxW 7.2.20-24 Use as needed; power management specific. +_SUN 6.1.11 Use as needed, but recommended. -_SWS 7.3.3 Use as needed; power management specific; this may +_SWS 7.4.3 Use as needed; power management specific; this may require specification changes for use on arm64. -\_TTS 7.3.4 Use as needed; power management specific. - -\_TZ 5.3.1 Optional. - _UID 6.1.12 Recommended for distinguishing devices of the same class; define it if at all possible. -\_WAK 7.3.5 Use as needed; power management specific. + ACPI Event Model ---------------- Do not use GPE block devices; these are not supported in the hardware reduced profile used by arm64. Since there are no GPE blocks defined for use on ARM -platforms, GPIO-signaled interrupts should be used for creating system events. +platforms, ACPI events must be signaled differently. + +There are two options: GPIO-signaled interrupts (Section 5.6.5), and +interrupt-signaled events (Section 5.6.9). Interrupt-signaled events are a +new feature in the ACPI 6.1 specification. Either -- or both -- can be used +on a given platform, and which to use may be dependent of limitations in any +given SoC. If possible, interrupt-signaled events are recommended. ACPI Processor Control ---------------------- -Section 8 of the ACPI specification is currently undergoing change that -should be completed in the 6.0 version of the specification. Processor -performance control will be handled differently for arm64 at that point -in time. Processor aggregator devices (section 8.5) will not be used, -for example, but another similar mechanism instead. - -While UEFI constrains what we can say until the release of 6.0, it is -recommended that CPPC (8.4.5) be used as the primary model. This will -still be useful into the future. C-states and P-states will still be -provided, but most of the current design work appears to favor CPPC. +Section 8 of the ACPI specification changed significantly in version 6.0. +Processors should now be defined as Device objects with _HID ACPI0007; do +not use the deprecated Processor statement in ASL. All multiprocessor systems +should also define a hierarchy of processors, done with Processor Container +Devices (see Section 8.4.3.1, _HID ACPI0010); do not use processor aggregator +devices (Section 8.5) to describe processor topology. Section 8.4 of the +specification describes the semantics of these object definitions and how +they interrelate. + +Most importantly, the processor hierarchy defined also defines the low power +idle states that are available to the platform, along with the rules for +determining which processors can be turned on or off and the circumstances +that control that. Without this information, the processors will run in +whatever power state they were left in by UEFI. + +Note too, that the processor Device objects defined and the entries in the +MADT for GICs are expected to be in synchronization. The _UID of the Device +object must correspond to processor IDs used in the MADT. + +It is recommended that CPPC (8.4.5) be used as the primary model for processor +performance control on arm64. C-states and P-states may become available at +some point in the future, but most current design work appears to favor CPPC. Further, it is essential that the ARMv8 SoC provide a fully functional implementation of PSCI; this will be the only mechanism supported by ACPI -to control CPU power state (including secondary CPU booting). - -More details will be provided on the release of the ACPI 6.0 specification. +to control CPU power state. Booting of secondary CPUs using the ACPI +parking protocol is possible, but discouraged, since only PSCI is supported +for ARM servers. ACPI System Address Map Interfaces @@ -535,21 +544,25 @@ used to indicate fatal errors that cannot be corrected, and require immediate attention. Since there is no direct equivalent of the x86 SCI or NMI, arm64 handles -these slightly differently. The SCI is handled as a normal GPIO-signaled -interrupt; given that these are corrected (or correctable) errors being -reported, this is sufficient. The NMI is emulated as the highest priority -GPIO-signaled interrupt possible. This implies some caution must be used -since there could be interrupts at higher privilege levels or even interrupts -at the same priority as the emulated NMI. In Linux, this should not be the -case but one should be aware it could happen. +these slightly differently. The SCI is handled as a high priority interrupt; +given that these are corrected (or correctable) errors being reported, this +is sufficient. The NMI is emulated as the highest priority interrupt +possible. This implies some caution must be used since there could be +interrupts at higher privilege levels or even interrupts at the same priority +as the emulated NMI. In Linux, this should not be the case but one should +be aware it could happen. ACPI Objects Not Supported on ARM64 ----------------------------------- While this may change in the future, there are several classes of objects that can be defined, but are not currently of general interest to ARM servers. +Some of these objects have x86 equivalents, and may actually make sense in ARM +servers. However, there is either no hardware available at present, or there +may not even be a non-ARM implementation yet. Hence, they are not currently +supported. -These are not supported: +The following classes of objects are not supported: -- Section 9.2: ambient light sensor devices @@ -571,16 +584,6 @@ These are not supported: -- Section 9.18: time and alarm devices (see 9.15) - -ACPI Objects Not Yet Implemented --------------------------------- -While these objects have x86 equivalents, and they do make some sense in ARM -servers, there is either no hardware available at present, or in some cases -there may not yet be a non-ARM implementation. Hence, they are currently not -implemented though that may change in the future. - -Not yet implemented are: - -- Section 10: power source and power meter devices -- Section 11: thermal management @@ -589,5 +592,31 @@ Not yet implemented are: -- Section 13: SMBus interfaces - -- Section 17: NUMA support (prototypes have been submitted for - review) + +This also means that there is no support for the following objects: + +Name Section Name Section +---- ------------ ---- ------------ +_ALC 9.3.4 _FDM 9.10.3 +_ALI 9.3.2 _FIX 6.2.7 +_ALP 9.3.6 _GAI 10.4.5 +_ALR 9.3.5 _GHL 10.4.7 +_ALT 9.3.3 _GTM 9.9.2.1.1 +_BCT 10.2.2.10 _LID 9.5.1 +_BDN 6.5.3 _PAI 10.4.4 +_BIF 10.2.2.1 _PCL 10.3.2 +_BIX 10.2.2.1 _PIF 10.3.3 +_BLT 9.2.3 _PMC 10.4.1 +_BMA 10.2.2.4 _PMD 10.4.8 +_BMC 10.2.2.12 _PMM 10.4.3 +_BMD 10.2.2.11 _PRL 10.3.4 +_BMS 10.2.2.5 _PSR 10.3.1 +_BST 10.2.2.6 _PTP 10.4.2 +_BTH 10.2.2.7 _SBS 10.1.3 +_BTM 10.2.2.9 _SHL 10.4.6 +_BTP 10.2.2.8 _STM 9.9.2.1.1 +_DCK 6.5.2 _UPD 9.16.1 +_EC 12.12 _UPP 9.16.2 +_FDE 9.10.1 _WPC 10.5.2 +_FDI 9.10.2 _WPP 10.5.3 + diff --git a/Documentation/arm64/arm-acpi.txt b/Documentation/arm64/arm-acpi.txt index 570a4f8e1a01..1a74a041a443 100644 --- a/Documentation/arm64/arm-acpi.txt +++ b/Documentation/arm64/arm-acpi.txt @@ -34,7 +34,7 @@ of the summary text almost directly, to be honest. The short form of the rationale for ACPI on ARM is: --- ACPI’s bytecode (AML) allows the platform to encode hardware behavior, +-- ACPI’s byte code (AML) allows the platform to encode hardware behavior, while DT explicitly does not support this. For hardware vendors, being able to encode behavior is a key tool used in supporting operating system releases on new hardware. @@ -57,11 +57,11 @@ The short form of the rationale for ACPI on ARM is: -- The new ACPI governance process works well and Linux is now at the same table as hardware vendors and other OS vendors. In fact, there is no - longer any reason to feel that ACPI is only belongs to Windows or that + longer any reason to feel that ACPI only belongs to Windows or that Linux is in any way secondary to Microsoft in this arena. The move of ACPI governance into the UEFI forum has significantly opened up the specification development process, and currently, a large portion of the - changes being made to ACPI is being driven by Linux. + changes being made to ACPI are being driven by Linux. Key to the use of ACPI is the support model. For servers in general, the responsibility for hardware behaviour cannot solely be the domain of the @@ -110,7 +110,7 @@ ACPI support in drivers and subsystems for ARMv8 should never be mutually exclusive with DT support at compile time. At boot time the kernel will only use one description method depending on -parameters passed from the bootloader (including kernel bootargs). +parameters passed from the boot loader (including kernel bootargs). Regardless of whether DT or ACPI is used, the kernel must always be capable of booting with either scheme (in kernels with both schemes enabled at compile @@ -159,7 +159,7 @@ Further, the ACPI core will only use the 64-bit address fields in the FADT (Fixed ACPI Description Table). Any 32-bit address fields in the FADT will be ignored on arm64. -Hardware reduced mode (see Section 4.1 of the ACPI 5.1 specification) will +Hardware reduced mode (see Section 4.1 of the ACPI 6.1 specification) will be enforced by the ACPI core on arm64. Doing so allows the ACPI core to run less complex code since it no longer has to provide support for legacy hardware from other architectures. Any fields that are not to be used for @@ -167,7 +167,7 @@ hardware reduced mode must be set to zero. For the ACPI core to operate properly, and in turn provide the information the kernel needs to configure devices, it expects to find the following -tables (all section numbers refer to the ACPI 5.1 specfication): +tables (all section numbers refer to the ACPI 6.1 specification): -- RSDP (Root System Description Pointer), section 5.2.5 @@ -185,9 +185,23 @@ tables (all section numbers refer to the ACPI 5.1 specfication): -- If PCI is supported, the MCFG (Memory mapped ConFiGuration Table), section 5.2.6, specifically Table 5-31. + -- If booting without a console= kernel parameter is + supported, the SPCR (Serial Port Console Redirection table), + section 5.2.6, specifically Table 5-31. + + -- If necessary to describe the I/O topology, SMMUs and GIC ITSs, + the IORT (Input Output Remapping Table, section 5.2.6, specifically + Table 5-31). + + -- If NUMA is supported, the SRAT (System Resource Affinity Table) + and SLIT (System Locality distance Information Table), sections + 5.2.16 and 5.2.17, respectively. + If the above tables are not all present, the kernel may or may not be able to boot properly since it may not be able to configure all of the -devices available. +devices available. This list of tables is not meant to be all inclusive; +in some environments other tables may be needed (e.g., any of the APEI +tables from section 18) to support specific functionality. ACPI Detection @@ -198,7 +212,7 @@ the device structure. This is detailed further in the "Driver Recommendations" section. In non-driver code, if the presence of ACPI needs to be detected at -runtime, then check the value of acpi_disabled. If CONFIG_ACPI is not +run time, then check the value of acpi_disabled. If CONFIG_ACPI is not set, acpi_disabled will always be 1. @@ -233,7 +247,7 @@ that looks like this: Name(KEY0, "value0"). An ACPI device driver would then retrieve the value of the property by evaluating the KEY0 object. However, using Name() this way has multiple problems: (1) ACPI limits names ("KEY0") to four characters unlike DT; (2) there is no industry -wide registry that maintains a list of names, minimzing re-use; (3) +wide registry that maintains a list of names, minimizing re-use; (3) there is also no registry for the definition of property values ("value0"), again making re-use difficult; and (4) how does one maintain backward compatibility as new hardware comes out? The _DSD method was created @@ -434,7 +448,8 @@ The ACPI specification changes regularly. During the year 2014, for instance, version 5.1 was released and version 6.0 substantially completed, with most of the changes being driven by ARM-specific requirements. Proposed changes are presented and discussed in the ASWG (ACPI Specification Working Group) which -is a part of the UEFI Forum. +is a part of the UEFI Forum. The current version of the ACPI specification +is 6.1 release in January 2016. Participation in this group is open to all UEFI members. Please see http://www.uefi.org/workinggroup for details on group membership. @@ -443,7 +458,7 @@ It is the intent of the ARMv8 ACPI kernel code to follow the ACPI specification as closely as possible, and to only implement functionality that complies with the released standards from UEFI ASWG. As a practical matter, there will be vendors that provide bad ACPI tables or violate the standards in some way. -If this is because of errors, quirks and fixups may be necessary, but will +If this is because of errors, quirks and fix-ups may be necessary, but will be avoided if possible. If there are features missing from ACPI that preclude it from being used on a platform, ECRs (Engineering Change Requests) should be submitted to ASWG and go through the normal approval process; for those that @@ -480,8 +495,7 @@ References Software on ARM Platforms", dated 16 Aug 2014 [2] http://www.secretlab.ca/archives/151, 10 Jan 2015, Copyright (c) 2015, - Linaro Ltd., written by Grant Likely. A copy of the verbatim text (apart - from formatting) is also in Documentation/arm64/why_use_acpi.txt. + Linaro Ltd., written by Grant Likely. [3] AMD ACPI for Seattle platform documentation: http://amd-dev.wpengine.netdna-cdn.com/wordpress/media/2012/10/Seattle_ACPI_Guide.pdf diff --git a/Documentation/bcache.txt b/Documentation/bcache.txt index 32b6c3189d98..a9259b562d5c 100644 --- a/Documentation/bcache.txt +++ b/Documentation/bcache.txt @@ -1,4 +1,4 @@ -Say you've got a big slow raid 6, and an X-25E or three. Wouldn't it be +Say you've got a big slow raid 6, and an ssd or three. Wouldn't it be nice if you could use them as cache... Hence bcache. Wiki and git repositories are at: @@ -8,7 +8,7 @@ Wiki and git repositories are at: It's designed around the performance characteristics of SSDs - it only allocates in erase block sized buckets, and it uses a hybrid btree/log to track cached -extants (which can be anywhere from a single sector to the bucket size). It's +extents (which can be anywhere from a single sector to the bucket size). It's designed to avoid random writes at all costs; it fills up an erase block sequentially, then issues a discard before reusing it. @@ -55,7 +55,10 @@ immediately. Without udev, you can manually register devices like this: Registering the backing device makes the bcache device show up in /dev; you can now format it and use it as normal. But the first time using a new bcache device, it'll be running in passthrough mode until you attach it to a cache. -See the section on attaching. +If you are thinking about using bcache later, it is recommended to setup all your +slow devices as bcache backing devices without a cache, and you can choose to add +a caching device later. +See 'ATTACHING' section below. The devices show up as: @@ -72,12 +75,14 @@ To get started: mount /dev/bcache0 /mnt You can control bcache devices through sysfs at /sys/block/bcache/bcache . +You can also control them through /sys/fs//bcache// . Cache devices are managed as sets; multiple caches per set isn't supported yet but will allow for mirroring of metadata and dirty data in the future. Your new cache set shows up as /sys/fs/bcache/ -ATTACHING: +ATTACHING +--------- After your cache device and backing device are registered, the backing device must be attached to your cache set to enable caching. Attaching a backing @@ -105,7 +110,8 @@ but all the cached data will be invalidated. If there was dirty data in the cache, don't expect the filesystem to be recoverable - you will have massive filesystem corruption, though ext4's fsck does work miracles. -ERROR HANDLING: +ERROR HANDLING +-------------- Bcache tries to transparently handle IO errors to/from the cache device without affecting normal operation; if it sees too many errors (the threshold is @@ -127,12 +133,181 @@ the backing devices to passthrough mode. writeback mode). It currently doesn't do anything intelligent if it fails to read some of the dirty data, though. -TROUBLESHOOTING PERFORMANCE: + +HOWTO/COOKBOOK +-------------- + +A) Starting a bcache with a missing caching device + +If registering the backing device doesn't help, it's already there, you just need +to force it to run without the cache: + host:~# echo /dev/sdb1 > /sys/fs/bcache/register + [ 119.844831] bcache: register_bcache() error opening /dev/sdb1: device already registered + +Next, you try to register your caching device if it's present. However +if it's absent, or registration fails for some reason, you can still +start your bcache without its cache, like so: + host:/sys/block/sdb/sdb1/bcache# echo 1 > running + +Note that this may cause data loss if you were running in writeback mode. + + +B) Bcache does not find its cache + + host:/sys/block/md5/bcache# echo 0226553a-37cf-41d5-b3ce-8b1e944543a8 > attach + [ 1933.455082] bcache: bch_cached_dev_attach() Couldn't find uuid for md5 in set + [ 1933.478179] bcache: __cached_dev_store() Can't attach 0226553a-37cf-41d5-b3ce-8b1e944543a8 + [ 1933.478179] : cache set not found + +In this case, the caching device was simply not registered at boot +or disappeared and came back, and needs to be (re-)registered: + host:/sys/block/md5/bcache# echo /dev/sdh2 > /sys/fs/bcache/register + + +C) Corrupt bcache crashes the kernel at device registration time: + +This should never happen. If it does happen, then you have found a bug! +Please report it to the bcache development list: linux-bcache@vger.kernel.org + +Be sure to provide as much information that you can including kernel dmesg +output if available so that we may assist. + + +D) Recovering data without bcache: + +If bcache is not available in the kernel, a filesystem on the backing +device is still available at an 8KiB offset. So either via a loopdev +of the backing device created with --offset 8K, or any value defined by +--data-offset when you originally formatted bcache with `make-bcache`. + +For example: + losetup -o 8192 /dev/loop0 /dev/your_bcache_backing_dev + +This should present your unmodified backing device data in /dev/loop0 + +If your cache is in writethrough mode, then you can safely discard the +cache device without loosing data. + + +E) Wiping a cache device + +host:~# wipefs -a /dev/sdh2 +16 bytes were erased at offset 0x1018 (bcache) +they were: c6 85 73 f6 4e 1a 45 ca 82 65 f5 7f 48 ba 6d 81 + +After you boot back with bcache enabled, you recreate the cache and attach it: +host:~# make-bcache -C /dev/sdh2 +UUID: 7be7e175-8f4c-4f99-94b2-9c904d227045 +Set UUID: 5bc072a8-ab17-446d-9744-e247949913c1 +version: 0 +nbuckets: 106874 +block_size: 1 +bucket_size: 1024 +nr_in_set: 1 +nr_this_dev: 0 +first_bucket: 1 +[ 650.511912] bcache: run_cache_set() invalidating existing data +[ 650.549228] bcache: register_cache() registered cache device sdh2 + +start backing device with missing cache: +host:/sys/block/md5/bcache# echo 1 > running + +attach new cache: +host:/sys/block/md5/bcache# echo 5bc072a8-ab17-446d-9744-e247949913c1 > attach +[ 865.276616] bcache: bch_cached_dev_attach() Caching md5 as bcache0 on set 5bc072a8-ab17-446d-9744-e247949913c1 + + +F) Remove or replace a caching device + + host:/sys/block/sda/sda7/bcache# echo 1 > detach + [ 695.872542] bcache: cached_dev_detach_finish() Caching disabled for sda7 + + host:~# wipefs -a /dev/nvme0n1p4 + wipefs: error: /dev/nvme0n1p4: probing initialization failed: Device or resource busy + Ooops, it's disabled, but not unregistered, so it's still protected + +We need to go and unregister it: + host:/sys/fs/bcache/b7ba27a1-2398-4649-8ae3-0959f57ba128# ls -l cache0 + lrwxrwxrwx 1 root root 0 Feb 25 18:33 cache0 -> ../../../devices/pci0000:00/0000:00:1d.0/0000:70:00.0/nvme/nvme0/nvme0n1/nvme0n1p4/bcache/ + host:/sys/fs/bcache/b7ba27a1-2398-4649-8ae3-0959f57ba128# echo 1 > stop + kernel: [ 917.041908] bcache: cache_set_free() Cache set b7ba27a1-2398-4649-8ae3-0959f57ba128 unregistered + +Now we can wipe it: + host:~# wipefs -a /dev/nvme0n1p4 + /dev/nvme0n1p4: 16 bytes were erased at offset 0x00001018 (bcache): c6 85 73 f6 4e 1a 45 ca 82 65 f5 7f 48 ba 6d 81 + + +G) dm-crypt and bcache + +First setup bcache unencrypted and then install dmcrypt on top of +/dev/bcache This will work faster than if you dmcrypt both the backing +and caching devices and then install bcache on top. [benchmarks?] + + +H) Stop/free a registered bcache to wipe and/or recreate it + +Suppose that you need to free up all bcache references so that you can +fdisk run and re-register a changed partition table, which won't work +if there are any active backing or caching devices left on it: + +1) Is it present in /dev/bcache* ? (there are times where it won't be) + +If so, it's easy: + host:/sys/block/bcache0/bcache# echo 1 > stop + +2) But if your backing device is gone, this won't work: + host:/sys/block/bcache0# cd bcache + bash: cd: bcache: No such file or directory + +In this case, you may have to unregister the dmcrypt block device that +references this bcache to free it up: + host:~# dmsetup remove oldds1 + bcache: bcache_device_free() bcache0 stopped + bcache: cache_set_free() Cache set 5bc072a8-ab17-446d-9744-e247949913c1 unregistered + +This causes the backing bcache to be removed from /sys/fs/bcache and +then it can be reused. This would be true of any block device stacking +where bcache is a lower device. + +3) In other cases, you can also look in /sys/fs/bcache/: + +host:/sys/fs/bcache# ls -l */{cache?,bdev?} +lrwxrwxrwx 1 root root 0 Mar 5 09:39 0226553a-37cf-41d5-b3ce-8b1e944543a8/bdev1 -> ../../../devices/virtual/block/dm-1/bcache/ +lrwxrwxrwx 1 root root 0 Mar 5 09:39 0226553a-37cf-41d5-b3ce-8b1e944543a8/cache0 -> ../../../devices/virtual/block/dm-4/bcache/ +lrwxrwxrwx 1 root root 0 Mar 5 09:39 5bc072a8-ab17-446d-9744-e247949913c1/cache0 -> ../../../devices/pci0000:00/0000:00:01.0/0000:01:00.0/ata10/host9/target9:0:0/9:0:0:0/block/sdl/sdl2/bcache/ + +The device names will show which UUID is relevant, cd in that directory +and stop the cache: + host:/sys/fs/bcache/5bc072a8-ab17-446d-9744-e247949913c1# echo 1 > stop + +This will free up bcache references and let you reuse the partition for +other purposes. + + + +TROUBLESHOOTING PERFORMANCE +--------------------------- Bcache has a bunch of config options and tunables. The defaults are intended to be reasonable for typical desktop and server workloads, but they're not what you want for getting the best possible numbers when benchmarking. + - Backing device alignment + + The default metadata size in bcache is 8k. If your backing device is + RAID based, then be sure to align this by a multiple of your stride + width using `make-bcache --data-offset`. If you intend to expand your + disk array in the future, then multiply a series of primes by your + raid stripe size to get the disk multiples that you would like. + + For example: If you have a 64k stripe size, then the following offset + would provide alignment for many common RAID5 data spindle counts: + 64k * 2*2*2*3*3*5*7 bytes = 161280k + + That space is wasted, but for only 157.5MB you can grow your RAID 5 + volume to the following data-spindle counts without re-aligning: + 3,4,5,6,7,8,9,10,12,14,15,18,20,21 ... + - Bad write performance If write performance is not what you expected, you probably wanted to be @@ -140,7 +315,7 @@ want for getting the best possible numbers when benchmarking. maturity, but simply because in writeback mode you'll lose data if something happens to your SSD) - # echo writeback > /sys/block/bcache0/cache_mode + # echo writeback > /sys/block/bcache0/bcache/cache_mode - Bad performance, or traffic not going to the SSD that you'd expect @@ -193,7 +368,9 @@ want for getting the best possible numbers when benchmarking. Solution: warm the cache by doing writes, or use the testing branch (there's a fix for the issue there). -SYSFS - BACKING DEVICE: + +SYSFS - BACKING DEVICE +---------------------- Available at /sys/block//bcache, /sys/block/bcache*/bcache and (if attached) /sys/fs/bcache//bdev* @@ -238,7 +415,7 @@ sequential_merge against all new requests to determine which new requests are sequential continuations of previous requests for the purpose of determining sequential cutoff. This is necessary if the sequential cutoff value is greater than the - maximum acceptable sequential size for any single request. + maximum acceptable sequential size for any single request. state The backing device can be in one of four different states: @@ -325,7 +502,7 @@ bucket_size Size of buckets cache<0..n> - Symlink to each of the cache devices comprising this cache set. + Symlink to each of the cache devices comprising this cache set. cache_available_percent Percentage of cache device which doesn't contain dirty data, and could diff --git a/Documentation/block/queue-sysfs.txt b/Documentation/block/queue-sysfs.txt index dce25d848d92..d515d58962b9 100644 --- a/Documentation/block/queue-sysfs.txt +++ b/Documentation/block/queue-sysfs.txt @@ -53,7 +53,7 @@ disk. logical_block_size (RO) ----------------------- -This is the logcal block size of the device, in bytes. +This is the logical block size of the device, in bytes. max_hw_sectors_kb (RO) ---------------------- diff --git a/Documentation/block/writeback_cache_control.txt b/Documentation/block/writeback_cache_control.txt index 59e0516cbf6b..8a6bdada5f6b 100644 --- a/Documentation/block/writeback_cache_control.txt +++ b/Documentation/block/writeback_cache_control.txt @@ -20,11 +20,11 @@ a forced cache flush, and the Force Unit Access (FUA) flag for requests. Explicit cache flushes ---------------------- -The REQ_FLUSH flag can be OR ed into the r/w flags of a bio submitted from +The REQ_PREFLUSH flag can be OR ed into the r/w flags of a bio submitted from the filesystem and will make sure the volatile cache of the storage device has been flushed before the actual I/O operation is started. This explicitly guarantees that previously completed write requests are on non-volatile -storage before the flagged bio starts. In addition the REQ_FLUSH flag can be +storage before the flagged bio starts. In addition the REQ_PREFLUSH flag can be set on an otherwise empty bio structure, which causes only an explicit cache flush without any dependent I/O. It is recommend to use the blkdev_issue_flush() helper for a pure cache flush. @@ -41,21 +41,21 @@ signaled after the data has been committed to non-volatile storage. Implementation details for filesystems -------------------------------------- -Filesystems can simply set the REQ_FLUSH and REQ_FUA bits and do not have to +Filesystems can simply set the REQ_PREFLUSH and REQ_FUA bits and do not have to worry if the underlying devices need any explicit cache flushing and how -the Forced Unit Access is implemented. The REQ_FLUSH and REQ_FUA flags +the Forced Unit Access is implemented. The REQ_PREFLUSH and REQ_FUA flags may both be set on a single bio. Implementation details for make_request_fn based block drivers -------------------------------------------------------------- -These drivers will always see the REQ_FLUSH and REQ_FUA bits as they sit +These drivers will always see the REQ_PREFLUSH and REQ_FUA bits as they sit directly below the submit_bio interface. For remapping drivers the REQ_FUA bits need to be propagated to underlying devices, and a global flush needs -to be implemented for bios with the REQ_FLUSH bit set. For real device -drivers that do not have a volatile cache the REQ_FLUSH and REQ_FUA bits -on non-empty bios can simply be ignored, and REQ_FLUSH requests without +to be implemented for bios with the REQ_PREFLUSH bit set. For real device +drivers that do not have a volatile cache the REQ_PREFLUSH and REQ_FUA bits +on non-empty bios can simply be ignored, and REQ_PREFLUSH requests without data can be completed successfully without doing any work. Drivers for devices with volatile caches need to implement the support for these flags themselves without any help from the block layer. @@ -65,17 +65,17 @@ Implementation details for request_fn based block drivers -------------------------------------------------------------- For devices that do not support volatile write caches there is no driver -support required, the block layer completes empty REQ_FLUSH requests before -entering the driver and strips off the REQ_FLUSH and REQ_FUA bits from +support required, the block layer completes empty REQ_PREFLUSH requests before +entering the driver and strips off the REQ_PREFLUSH and REQ_FUA bits from requests that have a payload. For devices with volatile write caches the driver needs to tell the block layer that it supports flushing caches by doing: blk_queue_write_cache(sdkp->disk->queue, true, false); -and handle empty REQ_FLUSH requests in its prep_fn/request_fn. Note that -REQ_FLUSH requests with a payload are automatically turned into a sequence -of an empty REQ_FLUSH request followed by the actual write by the block +and handle empty REQ_OP_FLUSH requests in its prep_fn/request_fn. Note that +REQ_PREFLUSH requests with a payload are automatically turned into a sequence +of an empty REQ_OP_FLUSH request followed by the actual write by the block layer. For devices that also support the FUA bit the block layer needs to be told to pass through the REQ_FUA bit using: @@ -83,4 +83,4 @@ to be told to pass through the REQ_FUA bit using: and the driver must handle write requests that have the REQ_FUA bit set in prep_fn/request_fn. If the FUA bit is not natively supported the block -layer turns it into an empty REQ_FLUSH request after the actual write. +layer turns it into an empty REQ_OP_FLUSH request after the actual write. diff --git a/Documentation/blockdev/zram.txt b/Documentation/blockdev/zram.txt index 13100fb3c26d..0535ae1f73e5 100644 --- a/Documentation/blockdev/zram.txt +++ b/Documentation/blockdev/zram.txt @@ -59,23 +59,23 @@ num_devices parameter is optional and tells zram how many devices should be pre-created. Default: 1. 2) Set max number of compression streams - Regardless the value passed to this attribute, ZRAM will always - allocate multiple compression streams - one per online CPUs - thus - allowing several concurrent compression operations. The number of - allocated compression streams goes down when some of the CPUs - become offline. There is no single-compression-stream mode anymore, - unless you are running a UP system or has only 1 CPU online. - - To find out how many streams are currently available: +Regardless the value passed to this attribute, ZRAM will always +allocate multiple compression streams - one per online CPUs - thus +allowing several concurrent compression operations. The number of +allocated compression streams goes down when some of the CPUs +become offline. There is no single-compression-stream mode anymore, +unless you are running a UP system or has only 1 CPU online. + +To find out how many streams are currently available: cat /sys/block/zram0/max_comp_streams 3) Select compression algorithm - Using comp_algorithm device attribute one can see available and - currently selected (shown in square brackets) compression algorithms, - change selected compression algorithm (once the device is initialised - there is no way to change compression algorithm). +Using comp_algorithm device attribute one can see available and +currently selected (shown in square brackets) compression algorithms, +change selected compression algorithm (once the device is initialised +there is no way to change compression algorithm). - Examples: +Examples: #show supported compression algorithms cat /sys/block/zram0/comp_algorithm lzo [lz4] @@ -83,17 +83,27 @@ pre-created. Default: 1. #select lzo compression algorithm echo lzo > /sys/block/zram0/comp_algorithm +For the time being, the `comp_algorithm' content does not necessarily +show every compression algorithm supported by the kernel. We keep this +list primarily to simplify device configuration and one can configure +a new device with a compression algorithm that is not listed in +`comp_algorithm'. The thing is that, internally, ZRAM uses Crypto API +and, if some of the algorithms were built as modules, it's impossible +to list all of them using, for instance, /proc/crypto or any other +method. This, however, has an advantage of permitting the usage of +custom crypto compression modules (implementing S/W or H/W compression). + 4) Set Disksize - Set disk size by writing the value to sysfs node 'disksize'. - The value can be either in bytes or you can use mem suffixes. - Examples: - # Initialize /dev/zram0 with 50MB disksize - echo $((50*1024*1024)) > /sys/block/zram0/disksize +Set disk size by writing the value to sysfs node 'disksize'. +The value can be either in bytes or you can use mem suffixes. +Examples: + # Initialize /dev/zram0 with 50MB disksize + echo $((50*1024*1024)) > /sys/block/zram0/disksize - # Using mem suffixes - echo 256K > /sys/block/zram0/disksize - echo 512M > /sys/block/zram0/disksize - echo 1G > /sys/block/zram0/disksize + # Using mem suffixes + echo 256K > /sys/block/zram0/disksize + echo 512M > /sys/block/zram0/disksize + echo 1G > /sys/block/zram0/disksize Note: There is little point creating a zram of greater than twice the size of memory @@ -101,20 +111,20 @@ since we expect a 2:1 compression ratio. Note that zram uses about 0.1% of the size of the disk when not in use so a huge zram is wasteful. 5) Set memory limit: Optional - Set memory limit by writing the value to sysfs node 'mem_limit'. - The value can be either in bytes or you can use mem suffixes. - In addition, you could change the value in runtime. - Examples: - # limit /dev/zram0 with 50MB memory - echo $((50*1024*1024)) > /sys/block/zram0/mem_limit - - # Using mem suffixes - echo 256K > /sys/block/zram0/mem_limit - echo 512M > /sys/block/zram0/mem_limit - echo 1G > /sys/block/zram0/mem_limit - - # To disable memory limit - echo 0 > /sys/block/zram0/mem_limit +Set memory limit by writing the value to sysfs node 'mem_limit'. +The value can be either in bytes or you can use mem suffixes. +In addition, you could change the value in runtime. +Examples: + # limit /dev/zram0 with 50MB memory + echo $((50*1024*1024)) > /sys/block/zram0/mem_limit + + # Using mem suffixes + echo 256K > /sys/block/zram0/mem_limit + echo 512M > /sys/block/zram0/mem_limit + echo 1G > /sys/block/zram0/mem_limit + + # To disable memory limit + echo 0 > /sys/block/zram0/mem_limit 6) Activate: mkswap /dev/zram0 diff --git a/Documentation/cec.txt b/Documentation/cec.txt new file mode 100644 index 000000000000..75155fe37153 --- /dev/null +++ b/Documentation/cec.txt @@ -0,0 +1,267 @@ +CEC Kernel Support +================== + +The CEC framework provides a unified kernel interface for use with HDMI CEC +hardware. It is designed to handle a multiple types of hardware (receivers, +transmitters, USB dongles). The framework also gives the option to decide +what to do in the kernel driver and what should be handled by userspace +applications. In addition it integrates the remote control passthrough +feature into the kernel's remote control framework. + + +The CEC Protocol +---------------- + +The CEC protocol enables consumer electronic devices to communicate with each +other through the HDMI connection. The protocol uses logical addresses in the +communication. The logical address is strictly connected with the functionality +provided by the device. The TV acting as the communication hub is always +assigned address 0. The physical address is determined by the physical +connection between devices. + +The CEC framework described here is up to date with the CEC 2.0 specification. +It is documented in the HDMI 1.4 specification with the new 2.0 bits documented +in the HDMI 2.0 specification. But for most of the features the freely available +HDMI 1.3a specification is sufficient: + +http://www.microprocessor.org/HDMISpecification13a.pdf + + +The Kernel Interface +==================== + +CEC Adapter +----------- + +The struct cec_adapter represents the CEC adapter hardware. It is created by +calling cec_allocate_adapter() and deleted by calling cec_delete_adapter(): + +struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops, + void *priv, const char *name, u32 caps, u8 available_las, + struct device *parent); +void cec_delete_adapter(struct cec_adapter *adap); + +To create an adapter you need to pass the following information: + +ops: adapter operations which are called by the CEC framework and that you +have to implement. + +priv: will be stored in adap->priv and can be used by the adapter ops. + +name: the name of the CEC adapter. Note: this name will be copied. + +caps: capabilities of the CEC adapter. These capabilities determine the + capabilities of the hardware and which parts are to be handled + by userspace and which parts are handled by kernelspace. The + capabilities are returned by CEC_ADAP_G_CAPS. + +available_las: the number of simultaneous logical addresses that this + adapter can handle. Must be 1 <= available_las <= CEC_MAX_LOG_ADDRS. + +parent: the parent device. + + +To register the /dev/cecX device node and the remote control device (if +CEC_CAP_RC is set) you call: + +int cec_register_adapter(struct cec_adapter *adap); + +To unregister the devices call: + +void cec_unregister_adapter(struct cec_adapter *adap); + +Note: if cec_register_adapter() fails, then call cec_delete_adapter() to +clean up. But if cec_register_adapter() succeeded, then only call +cec_unregister_adapter() to clean up, never cec_delete_adapter(). The +unregister function will delete the adapter automatically once the last user +of that /dev/cecX device has closed its file handle. + + +Implementing the Low-Level CEC Adapter +-------------------------------------- + +The following low-level adapter operations have to be implemented in +your driver: + +struct cec_adap_ops { + /* Low-level callbacks */ + int (*adap_enable)(struct cec_adapter *adap, bool enable); + int (*adap_monitor_all_enable)(struct cec_adapter *adap, bool enable); + int (*adap_log_addr)(struct cec_adapter *adap, u8 logical_addr); + int (*adap_transmit)(struct cec_adapter *adap, u8 attempts, + u32 signal_free_time, struct cec_msg *msg); + void (*adap_log_status)(struct cec_adapter *adap); + + /* High-level callbacks */ + ... +}; + +The three low-level ops deal with various aspects of controlling the CEC adapter +hardware: + + +To enable/disable the hardware: + + int (*adap_enable)(struct cec_adapter *adap, bool enable); + +This callback enables or disables the CEC hardware. Enabling the CEC hardware +means powering it up in a state where no logical addresses are claimed. This +op assumes that the physical address (adap->phys_addr) is valid when enable is +true and will not change while the CEC adapter remains enabled. The initial +state of the CEC adapter after calling cec_allocate_adapter() is disabled. + +Note that adap_enable must return 0 if enable is false. + + +To enable/disable the 'monitor all' mode: + + int (*adap_monitor_all_enable)(struct cec_adapter *adap, bool enable); + +If enabled, then the adapter should be put in a mode to also monitor messages +that not for us. Not all hardware supports this and this function is only +called if the CEC_CAP_MONITOR_ALL capability is set. This callback is optional +(some hardware may always be in 'monitor all' mode). + +Note that adap_monitor_all_enable must return 0 if enable is false. + + +To program a new logical address: + + int (*adap_log_addr)(struct cec_adapter *adap, u8 logical_addr); + +If logical_addr == CEC_LOG_ADDR_INVALID then all programmed logical addresses +are to be erased. Otherwise the given logical address should be programmed. +If the maximum number of available logical addresses is exceeded, then it +should return -ENXIO. Once a logical address is programmed the CEC hardware +can receive directed messages to that address. + +Note that adap_log_addr must return 0 if logical_addr is CEC_LOG_ADDR_INVALID. + + +To transmit a new message: + + int (*adap_transmit)(struct cec_adapter *adap, u8 attempts, + u32 signal_free_time, struct cec_msg *msg); + +This transmits a new message. The attempts argument is the suggested number of +attempts for the transmit. + +The signal_free_time is the number of data bit periods that the adapter should +wait when the line is free before attempting to send a message. This value +depends on whether this transmit is a retry, a message from a new initiator or +a new message for the same initiator. Most hardware will handle this +automatically, but in some cases this information is needed. + +The CEC_FREE_TIME_TO_USEC macro can be used to convert signal_free_time to +microseconds (one data bit period is 2.4 ms). + + +To log the current CEC hardware status: + + void (*adap_status)(struct cec_adapter *adap, struct seq_file *file); + +This optional callback can be used to show the status of the CEC hardware. +The status is available through debugfs: cat /sys/kernel/debug/cec/cecX/status + + +Your adapter driver will also have to react to events (typically interrupt +driven) by calling into the framework in the following situations: + +When a transmit finished (successfully or otherwise): + +void cec_transmit_done(struct cec_adapter *adap, u8 status, u8 arb_lost_cnt, + u8 nack_cnt, u8 low_drive_cnt, u8 error_cnt); + +The status can be one of: + +CEC_TX_STATUS_OK: the transmit was successful. +CEC_TX_STATUS_ARB_LOST: arbitration was lost: another CEC initiator +took control of the CEC line and you lost the arbitration. +CEC_TX_STATUS_NACK: the message was nacked (for a directed message) or +acked (for a broadcast message). A retransmission is needed. +CEC_TX_STATUS_LOW_DRIVE: low drive was detected on the CEC bus. This +indicates that a follower detected an error on the bus and requested a +retransmission. +CEC_TX_STATUS_ERROR: some unspecified error occurred: this can be one of +the previous two if the hardware cannot differentiate or something else +entirely. +CEC_TX_STATUS_MAX_RETRIES: could not transmit the message after +trying multiple times. Should only be set by the driver if it has hardware +support for retrying messages. If set, then the framework assumes that it +doesn't have to make another attempt to transmit the message since the +hardware did that already. + +The *_cnt arguments are the number of error conditions that were seen. +This may be 0 if no information is available. Drivers that do not support +hardware retry can just set the counter corresponding to the transmit error +to 1, if the hardware does support retry then either set these counters to +0 if the hardware provides no feedback of which errors occurred and how many +times, or fill in the correct values as reported by the hardware. + +When a CEC message was received: + +void cec_received_msg(struct cec_adapter *adap, struct cec_msg *msg); + +Speaks for itself. + +Implementing the High-Level CEC Adapter +--------------------------------------- + +The low-level operations drive the hardware, the high-level operations are +CEC protocol driven. The following high-level callbacks are available: + +struct cec_adap_ops { + /* Low-level callbacks */ + ... + + /* High-level CEC message callback */ + int (*received)(struct cec_adapter *adap, struct cec_msg *msg); +}; + +The received() callback allows the driver to optionally handle a newly +received CEC message + + int (*received)(struct cec_adapter *adap, struct cec_msg *msg); + +If the driver wants to process a CEC message, then it can implement this +callback. If it doesn't want to handle this message, then it should return +-ENOMSG, otherwise the CEC framework assumes it processed this message and +it will not no anything with it. + + +CEC framework functions +----------------------- + +CEC Adapter drivers can call the following CEC framework functions: + +int cec_transmit_msg(struct cec_adapter *adap, struct cec_msg *msg, + bool block); + +Transmit a CEC message. If block is true, then wait until the message has been +transmitted, otherwise just queue it and return. + +void cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr, bool block); + +Change the physical address. This function will set adap->phys_addr and +send an event if it has changed. If cec_s_log_addrs() has been called and +the physical address has become valid, then the CEC framework will start +claiming the logical addresses. If block is true, then this function won't +return until this process has finished. + +When the physical address is set to a valid value the CEC adapter will +be enabled (see the adap_enable op). When it is set to CEC_PHYS_ADDR_INVALID, +then the CEC adapter will be disabled. If you change a valid physical address +to another valid physical address, then this function will first set the +address to CEC_PHYS_ADDR_INVALID before enabling the new physical address. + +int cec_s_log_addrs(struct cec_adapter *adap, + struct cec_log_addrs *log_addrs, bool block); + +Claim the CEC logical addresses. Should never be called if CEC_CAP_LOG_ADDRS +is set. If block is true, then wait until the logical addresses have been +claimed, otherwise just queue it and return. To unconfigure all logical +addresses call this function with log_addrs set to NULL or with +log_addrs->num_log_addrs set to 0. The block argument is ignored when +unconfiguring. This function will just return if the physical address is +invalid. Once the physical address becomes valid, then the framework will +attempt to claim these logical addresses. diff --git a/Documentation/conf.py b/Documentation/conf.py new file mode 100644 index 000000000000..792b6338ef19 --- /dev/null +++ b/Documentation/conf.py @@ -0,0 +1,414 @@ +# -*- coding: utf-8 -*- +# +# The Linux Kernel documentation build configuration file, created by +# sphinx-quickstart on Fri Feb 12 13:51:46 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('sphinx')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['kernel-doc', 'rstFlatTable'] + +# Gracefully handle missing rst2pdf. +try: + import rst2pdf + extensions += ['rst2pdf.pdfbuilder'] +except ImportError: + pass + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'The Linux Kernel' +copyright = '2016, The kernel development community' +author = 'The kernel development community' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# In a normal build, version and release are are set to KERNELVERSION and +# KERNELRELEASE, respectively, from the Makefile via Sphinx command line +# arguments. +# +# The following code tries to extract the information by reading the Makefile, +# when Sphinx is run directly (e.g. by Read the Docs). +try: + makefile_version = None + makefile_patchlevel = None + for line in open('../Makefile'): + key, val = [x.strip() for x in line.split('=', 2)] + if key == 'VERSION': + makefile_version = val + elif key == 'PATCHLEVEL': + makefile_patchlevel = val + if makefile_version and makefile_patchlevel: + break +except: + pass +finally: + if makefile_version and makefile_patchlevel: + version = release = makefile_version + '.' + makefile_patchlevel + else: + sys.stderr.write('Warning: Could not extract kernel version\n') + version = release = "unknown version" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['output'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +primary_domain = 'C' +highlight_language = 'C' + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. + +# The Read the Docs theme is available from +# - https://github.com/snide/sphinx_rtd_theme +# - https://pypi.python.org/pypi/sphinx_rtd_theme +# - python-sphinx-rtd-theme package (on Debian) +try: + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +except ImportError: + sys.stderr.write('Warning: The Sphinx \'sphinx_rtd_theme\' HTML theme was not found. Make sure you have the theme installed to produce pretty HTML output. Falling back to the default theme.\n') + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'TheLinuxKerneldoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'TheLinuxKernel.tex', 'The Linux Kernel Documentation', + 'The kernel development community', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'thelinuxkernel', 'The Linux Kernel Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'TheLinuxKernel', 'The Linux Kernel Documentation', + author, 'TheLinuxKernel', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The basename for the epub file. It defaults to the project name. +#epub_basename = project + +# The HTML theme for the epub output. Since the default themes are not +# optimized for small screen space, using the same theme for HTML and epub +# output is usually not wise. This defaults to 'epub', a theme designed to save +# visual space. +#epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or 'en' if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +#epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files that should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +#epub_tocscope = 'default' + +# Fix unsupported image types using the Pillow. +#epub_fix_images = False + +# Scale large images. +#epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#epub_show_urls = 'inline' + +# If false, no index is generated. +#epub_use_index = True + +#======= +# rst2pdf +# +# Grouping the document tree into PDF files. List of tuples +# (source start file, target name, title, author, options). +# +# See the Sphinx chapter of http://ralsina.me/static/manual.pdf +# +# FIXME: Do not add the index file here; the result will be too big. Adding +# multiple PDF files here actually tries to get the cross-referencing right +# *between* PDF files. +pdf_documents = [ + ('index', u'Kernel', u'Kernel', u'J. Random Bozo'), +] + +# kernel-doc extension configuration for running Sphinx directly (e.g. by Read +# the Docs). In a normal build, these are supplied from the Makefile via command +# line arguments. +kerneldoc_bin = '../scripts/kernel-doc' +kerneldoc_srctree = '..' diff --git a/Documentation/cpu-freq/core.txt b/Documentation/cpu-freq/core.txt index ba78e7c2a069..4bc7287806de 100644 --- a/Documentation/cpu-freq/core.txt +++ b/Documentation/cpu-freq/core.txt @@ -96,7 +96,7 @@ new - new frequency For details about OPP, see Documentation/power/opp.txt dev_pm_opp_init_cpufreq_table - cpufreq framework typically is initialized with - cpufreq_frequency_table_cpuinfo which is provided with the list of + cpufreq_table_validate_and_show() which is provided with the list of frequencies that are available for operation. This function provides a ready to use conversion routine to translate the OPP layer's internal information about the available frequencies into a format readily @@ -110,7 +110,7 @@ dev_pm_opp_init_cpufreq_table - cpufreq framework typically is initialized with /* Do things */ r = dev_pm_opp_init_cpufreq_table(dev, &freq_table); if (!r) - cpufreq_frequency_table_cpuinfo(policy, freq_table); + cpufreq_table_validate_and_show(policy, freq_table); /* Do other things */ } diff --git a/Documentation/cpu-freq/cpu-drivers.txt b/Documentation/cpu-freq/cpu-drivers.txt index 14f4e6336d88..772b94fde264 100644 --- a/Documentation/cpu-freq/cpu-drivers.txt +++ b/Documentation/cpu-freq/cpu-drivers.txt @@ -231,7 +231,7 @@ if you want to skip one entry in the table, set the frequency to CPUFREQ_ENTRY_INVALID. The entries don't need to be in ascending order. -By calling cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, +By calling cpufreq_table_validate_and_show(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table); the cpuinfo.min_freq and cpuinfo.max_freq values are detected, and policy->min and policy->max are set to the same values. This is @@ -244,14 +244,12 @@ policy->max, and all other criteria are met. This is helpful for the ->verify call. int cpufreq_frequency_table_target(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table, unsigned int target_freq, - unsigned int relation, - unsigned int *index); + unsigned int relation); is the corresponding frequency table helper for the ->target -stage. Just pass the values to this function, and the unsigned int -index returns the number of the frequency table entry which contains +stage. Just pass the values to this function, and this function +returns the number of the frequency table entry which contains the frequency the CPU shall be set to. The following macros can be used as iterators over cpufreq_frequency_table: diff --git a/Documentation/cpu-freq/pcc-cpufreq.txt b/Documentation/cpu-freq/pcc-cpufreq.txt index 0a94224ad296..9e3c3b33514c 100644 --- a/Documentation/cpu-freq/pcc-cpufreq.txt +++ b/Documentation/cpu-freq/pcc-cpufreq.txt @@ -159,8 +159,8 @@ to be strictly associated with a P-state. 2.2 cpuinfo_transition_latency: ------------------------------- -The cpuinfo_transition_latency field is CPUFREQ_ETERNAL. The PCC specification -does not include a field to expose this value currently. +The cpuinfo_transition_latency field is 0. The PCC specification does +not include a field to expose this value currently. 2.3 cpuinfo_cur_freq: --------------------- diff --git a/Documentation/cputopology.txt b/Documentation/cputopology.txt index 12b1b25b4da9..f722f227a73b 100644 --- a/Documentation/cputopology.txt +++ b/Documentation/cputopology.txt @@ -20,48 +20,70 @@ to /proc/cpuinfo output of some architectures: identifier (rather than the kernel's). The actual value is architecture and platform dependent. -4) /sys/devices/system/cpu/cpuX/topology/thread_siblings: +4) /sys/devices/system/cpu/cpuX/topology/drawer_id: + + the drawer ID of cpuX. Typically it is the hardware platform's + identifier (rather than the kernel's). The actual value is + architecture and platform dependent. + +5) /sys/devices/system/cpu/cpuX/topology/thread_siblings: internal kernel map of cpuX's hardware threads within the same core as cpuX. -5) /sys/devices/system/cpu/cpuX/topology/thread_siblings_list: +6) /sys/devices/system/cpu/cpuX/topology/thread_siblings_list: human-readable list of cpuX's hardware threads within the same core as cpuX. -6) /sys/devices/system/cpu/cpuX/topology/core_siblings: +7) /sys/devices/system/cpu/cpuX/topology/core_siblings: internal kernel map of cpuX's hardware threads within the same physical_package_id. -7) /sys/devices/system/cpu/cpuX/topology/core_siblings_list: +8) /sys/devices/system/cpu/cpuX/topology/core_siblings_list: human-readable list of cpuX's hardware threads within the same physical_package_id. -8) /sys/devices/system/cpu/cpuX/topology/book_siblings: +9) /sys/devices/system/cpu/cpuX/topology/book_siblings: internal kernel map of cpuX's hardware threads within the same book_id. -9) /sys/devices/system/cpu/cpuX/topology/book_siblings_list: +10) /sys/devices/system/cpu/cpuX/topology/book_siblings_list: human-readable list of cpuX's hardware threads within the same book_id. +11) /sys/devices/system/cpu/cpuX/topology/drawer_siblings: + + internal kernel map of cpuX's hardware threads within the same + drawer_id. + +12) /sys/devices/system/cpu/cpuX/topology/drawer_siblings_list: + + human-readable list of cpuX's hardware threads within the same + drawer_id. + To implement it in an architecture-neutral way, a new source file, -drivers/base/topology.c, is to export the 6 or 9 attributes. The three book -related sysfs files will only be created if CONFIG_SCHED_BOOK is selected. +drivers/base/topology.c, is to export the 6 to 12 attributes. The book +and drawer related sysfs files will only be created if CONFIG_SCHED_BOOK +and CONFIG_SCHED_DRAWER are selected. + +CONFIG_SCHED_BOOK and CONFIG_DRAWER are currently only used on s390, where +they reflect the cpu and cache hierarchy. For an architecture to support this feature, it must define some of these macros in include/asm-XXX/topology.h: #define topology_physical_package_id(cpu) #define topology_core_id(cpu) #define topology_book_id(cpu) +#define topology_drawer_id(cpu) #define topology_sibling_cpumask(cpu) #define topology_core_cpumask(cpu) #define topology_book_cpumask(cpu) +#define topology_drawer_cpumask(cpu) The type of **_id macros is int. The type of **_cpumask macros is (const) struct cpumask *. The latter @@ -78,6 +100,8 @@ not defined by include/asm-XXX/topology.h: For architectures that don't support books (CONFIG_SCHED_BOOK) there are no default definitions for topology_book_id() and topology_book_cpumask(). +For architectures that don't support drawes (CONFIG_SCHED_DRAWER) there are +no default definitions for topology_drawer_id() and topology_drawer_cpumask(). Additionally, CPU topology information is provided under /sys/devices/system/cpu and includes these files. The internal diff --git a/Documentation/crypto/asymmetric-keys.txt b/Documentation/crypto/asymmetric-keys.txt index 8c07e0ea6bc0..2b7816dea370 100644 --- a/Documentation/crypto/asymmetric-keys.txt +++ b/Documentation/crypto/asymmetric-keys.txt @@ -76,7 +76,7 @@ the criterion string: Looking in /proc/keys, the last 8 hex digits of the key fingerprint are displayed, along with the subtype: - 1a39e171 I----- 1 perm 3f010000 0 0 asymmetri modsign.0: DSA 5acc2142 [] + 1a39e171 I----- 1 perm 3f010000 0 0 asymmetric modsign.0: DSA 5acc2142 [] ========================= diff --git a/Documentation/development-process/4.Coding b/Documentation/development-process/4.Coding index e3cb6a56653a..9a3ee77cefb1 100644 --- a/Documentation/development-process/4.Coding +++ b/Documentation/development-process/4.Coding @@ -346,7 +346,7 @@ which have not been so documented, there is no harm in adding kerneldoc comments for the future; indeed, this can be a useful activity for beginning kernel developers. The format of these comments, along with some information on how to create kerneldoc templates can be found in the file -Documentation/kernel-doc-nano-HOWTO.txt. +Documentation/kernel-documentation.rst. Anybody who reads through a significant amount of existing kernel code will note that, often, comments are most notable by their absence. Once again, diff --git a/Documentation/device-mapper/dm-raid.txt b/Documentation/device-mapper/dm-raid.txt index df2d636b6088..e5b6497116f4 100644 --- a/Documentation/device-mapper/dm-raid.txt +++ b/Documentation/device-mapper/dm-raid.txt @@ -14,8 +14,12 @@ The target is named "raid" and it accepts the following parameters: <#raid_devs> [.. ] : + raid0 RAID0 striping (no resilience) raid1 RAID1 mirroring - raid4 RAID4 dedicated parity disk + raid4 RAID4 with dedicated last parity disk + raid5_n RAID5 with dedicated last parity disk suporting takeover + Same as raid4 + -Transitory layout raid5_la RAID5 left asymmetric - rotating parity 0 with data continuation raid5_ra RAID5 right asymmetric @@ -30,7 +34,19 @@ The target is named "raid" and it accepts the following parameters: - rotating parity N (right-to-left) with data restart raid6_nc RAID6 N continue - rotating parity N (right-to-left) with data continuation + raid6_n_6 RAID6 with dedicate parity disks + - parity and Q-syndrome on the last 2 disks; + laylout for takeover from/to raid4/raid5_n + raid6_la_6 Same as "raid_la" plus dedicated last Q-syndrome disk + - layout for takeover from raid5_la from/to raid6 + raid6_ra_6 Same as "raid5_ra" dedicated last Q-syndrome disk + - layout for takeover from raid5_ra from/to raid6 + raid6_ls_6 Same as "raid5_ls" dedicated last Q-syndrome disk + - layout for takeover from raid5_ls from/to raid6 + raid6_rs_6 Same as "raid5_rs" dedicated last Q-syndrome disk + - layout for takeover from raid5_rs from/to raid6 raid10 Various RAID10 inspired algorithms chosen by additional params + (see raid10_format and raid10_copies below) - RAID10: Striped Mirrors (aka 'Striping on top of mirrors') - RAID1E: Integrated Adjacent Stripe Mirroring - RAID1E: Integrated Offset Stripe Mirroring @@ -116,10 +132,41 @@ The target is named "raid" and it accepts the following parameters: Here we see layouts closely akin to 'RAID1E - Integrated Offset Stripe Mirroring'. + [delta_disks ] + The delta_disks option value (-251 < N < +251) triggers + device removal (negative value) or device addition (positive + value) to any reshape supporting raid levels 4/5/6 and 10. + RAID levels 4/5/6 allow for addition of devices (metadata + and data device tupel), raid10_near and raid10_offset only + allow for device addtion. raid10_far does not support any + reshaping at all. + A minimum of devices have to be kept to enforce resilience, + which is 3 devices for raid4/5 and 4 devices for raid6. + + [data_offset ] + This option value defines the offset into each data device + where the data starts. This is used to provide out-of-place + reshaping space to avoid writing over data whilst + changing the layout of stripes, hence an interruption/crash + may happen at any time without the risk of losing data. + E.g. when adding devices to an existing raid set during + forward reshaping, the out-of-place space will be allocated + at the beginning of each raid device. The kernel raid4/5/6/10 + MD personalities supporting such device addition will read the data from + the existing first stripes (those with smaller number of stripes) + starting at data_offset to fill up a new stripe with the larger + number of stripes, calculate the redundancy blocks (CRC/Q-syndrome) + and write that new stripe to offset 0. Same will be applied to all + N-1 other new stripes. This out-of-place scheme is used to change + the RAID type (i.e. the allocation algorithm) as well, e.g. + changing from raid5_ls to raid5_n. + <#raid_devs>: The number of devices composing the array. Each device consists of two entries. The first is the device containing the metadata (if any); the second is the one containing the - data. + data. A Maximum of 64 metadata/data device entries are supported + up to target version 1.8.0. + 1.9.0 supports up to 253 which is enforced by the used MD kernel runtime. If a drive has failed or is missing at creation time, a '-' can be given for both the metadata and data drives for a given position. @@ -207,7 +254,6 @@ include: "recover"- Initiate/continue a recover process. "check" - Initiate a check (i.e. a "scrub") of the array. "repair" - Initiate a repair of the array. - "reshape"- Currently unsupported (-EINVAL). Discard Support @@ -257,3 +303,9 @@ Version History 1.5.2 'mismatch_cnt' is zero unless [last_]sync_action is "check". 1.6.0 Add discard support (and devices_handle_discard_safely module param). 1.7.0 Add support for MD RAID0 mappings. +1.8.0 Explictely check for compatible flags in the superblock metadata + and reject to start the raid set if any are set by a newer + target version, thus avoiding data corruption on a raid set + with a reshape in progress. +1.9.0 Add support for RAID level takeover/reshape/region size + and set size reduction. diff --git a/Documentation/device-mapper/log-writes.txt b/Documentation/device-mapper/log-writes.txt index c10f30c9b534..f4ebcbaf50f3 100644 --- a/Documentation/device-mapper/log-writes.txt +++ b/Documentation/device-mapper/log-writes.txt @@ -14,14 +14,14 @@ Log Ordering We log things in order of completion once we are sure the write is no longer in cache. This means that normal WRITE requests are not actually logged until the -next REQ_FLUSH request. This is to make it easier for userspace to replay the -log in a way that correlates to what is on disk and not what is in cache, to -make it easier to detect improper waiting/flushing. +next REQ_PREFLUSH request. This is to make it easier for userspace to replay +the log in a way that correlates to what is on disk and not what is in cache, +to make it easier to detect improper waiting/flushing. This works by attaching all WRITE requests to a list once the write completes. -Once we see a REQ_FLUSH request we splice this list onto the request and once +Once we see a REQ_PREFLUSH request we splice this list onto the request and once the FLUSH request completes we log all of the WRITEs and then the FLUSH. Only -completed WRITEs, at the time the REQ_FLUSH is issued, are added in order to +completed WRITEs, at the time the REQ_PREFLUSH is issued, are added in order to simulate the worst case scenario with regard to power failures. Consider the following example (W means write, C means complete): diff --git a/Documentation/devicetree/bindings/arm/altera/socfpga-eccmgr.txt b/Documentation/devicetree/bindings/arm/altera/socfpga-eccmgr.txt index 5a6b16070a33..b545856a444f 100644 --- a/Documentation/devicetree/bindings/arm/altera/socfpga-eccmgr.txt +++ b/Documentation/devicetree/bindings/arm/altera/socfpga-eccmgr.txt @@ -61,7 +61,9 @@ Required Properties: - #address-cells: must be 1 - #size-cells: must be 1 - interrupts : Should be single bit error interrupt, then double bit error - interrupt. Note the rising edge type. + interrupt. +- interrupt-controller : boolean indicator that ECC Manager is an interrupt controller +- #interrupt-cells : must be set to 2. - ranges : standard definition, should translate from local addresses Subcomponents: @@ -70,11 +72,23 @@ L2 Cache ECC Required Properties: - compatible : Should be "altr,socfpga-a10-l2-ecc" - reg : Address and size for ECC error interrupt clear registers. +- interrupts : Should be single bit error interrupt, then double bit error + interrupt, in this order. On-Chip RAM ECC Required Properties: - compatible : Should be "altr,socfpga-a10-ocram-ecc" - reg : Address and size for ECC block registers. +- interrupts : Should be single bit error interrupt, then double bit error + interrupt, in this order. + +Ethernet FIFO ECC +Required Properties: +- compatible : Should be "altr,socfpga-eth-mac-ecc" +- reg : Address and size for ECC block registers. +- altr,ecc-parent : phandle to parent Ethernet node. +- interrupts : Should be single bit error interrupt, then double bit error + interrupt, in this order. Example: @@ -85,15 +99,37 @@ Example: #size-cells = <1>; interrupts = <0 2 IRQ_TYPE_LEVEL_HIGH>, <0 0 IRQ_TYPE_LEVEL_HIGH>; + interrupt-controller; + #interrupt-cells = <2>; ranges; l2-ecc@ffd06010 { compatible = "altr,socfpga-a10-l2-ecc"; reg = <0xffd06010 0x4>; + interrupts = <0 IRQ_TYPE_LEVEL_HIGH>, + <32 IRQ_TYPE_LEVEL_HIGH>; }; ocram-ecc@ff8c3000 { compatible = "altr,socfpga-a10-ocram-ecc"; reg = <0xff8c3000 0x90>; + interrupts = <1 IRQ_TYPE_LEVEL_HIGH>, + <33 IRQ_TYPE_LEVEL_HIGH> ; + }; + + emac0-rx-ecc@ff8c0800 { + compatible = "altr,socfpga-eth-mac-ecc"; + reg = <0xff8c0800 0x400>; + altr,ecc-parent = <&gmac0>; + interrupts = <4 IRQ_TYPE_LEVEL_HIGH>, + <36 IRQ_TYPE_LEVEL_HIGH>; + }; + + emac0-tx-ecc@ff8c0c00 { + compatible = "altr,socfpga-eth-mac-ecc"; + reg = <0xff8c0c00 0x400>; + altr,ecc-parent = <&gmac0>; + interrupts = <5 IRQ_TYPE_LEVEL_HIGH>, + <37 IRQ_TYPE_LEVEL_HIGH>; }; }; diff --git a/Documentation/devicetree/bindings/arm/pmu.txt b/Documentation/devicetree/bindings/arm/pmu.txt index 74d5417d0410..61c8b4620415 100644 --- a/Documentation/devicetree/bindings/arm/pmu.txt +++ b/Documentation/devicetree/bindings/arm/pmu.txt @@ -39,7 +39,9 @@ Optional properties: When using a PPI, specifies a list of phandles to CPU nodes corresponding to the set of CPUs which have a PMU of this type signalling the PPI listed in the - interrupts property. + interrupts property, unless this is already specified + by the PPI interrupt specifier itself (in which case + the interrupt-affinity property shouldn't be present). This property should be present when there is more than a single SPI. diff --git a/Documentation/devicetree/bindings/arm/xen.txt b/Documentation/devicetree/bindings/arm/xen.txt index 0f7b9c2109f8..c9b9321434ea 100644 --- a/Documentation/devicetree/bindings/arm/xen.txt +++ b/Documentation/devicetree/bindings/arm/xen.txt @@ -11,10 +11,32 @@ the following properties: memory where the grant table should be mapped to, using an HYPERVISOR_memory_op hypercall. The memory region is large enough to map the whole grant table (it is larger or equal to gnttab_max_grant_frames()). + This property is unnecessary when booting Dom0 using ACPI. - interrupts: the interrupt used by Xen to inject event notifications. A GIC node is also required. + This property is unnecessary when booting Dom0 using ACPI. +To support UEFI on Xen ARM virtual platforms, Xen populates the FDT "uefi" node +under /hypervisor with following parameters: + +________________________________________________________________________________ +Name | Size | Description +================================================================================ +xen,uefi-system-table | 64-bit | Guest physical address of the UEFI System + | | Table. +-------------------------------------------------------------------------------- +xen,uefi-mmap-start | 64-bit | Guest physical address of the UEFI memory + | | map. +-------------------------------------------------------------------------------- +xen,uefi-mmap-size | 32-bit | Size in bytes of the UEFI memory map + | | pointed to in previous entry. +-------------------------------------------------------------------------------- +xen,uefi-mmap-desc-size | 32-bit | Size in bytes of each entry in the UEFI + | | memory map. +-------------------------------------------------------------------------------- +xen,uefi-mmap-desc-ver | 32-bit | Version of the mmap descriptor format. +-------------------------------------------------------------------------------- Example (assuming #address-cells = <2> and #size-cells = <2>): @@ -22,4 +44,17 @@ hypervisor { compatible = "xen,xen-4.3", "xen,xen"; reg = <0 0xb0000000 0 0x20000>; interrupts = <1 15 0xf08>; + uefi { + xen,uefi-system-table = <0xXXXXXXXX>; + xen,uefi-mmap-start = <0xXXXXXXXX>; + xen,uefi-mmap-size = <0xXXXXXXXX>; + xen,uefi-mmap-desc-size = <0xXXXXXXXX>; + xen,uefi-mmap-desc-ver = <0xXXXXXXXX>; + }; }; + +The format and meaning of the "xen,uefi-*" parameters are similar to those in +Documentation/arm/uefi.txt, which are provided by the regular UEFI stub. However +they differ because they are provided by the Xen hypervisor, together with a set +of UEFI runtime services implemented via hypercalls, see +http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,platform.h.html. diff --git a/Documentation/devicetree/bindings/ata/brcm,sata-brcmstb.txt b/Documentation/devicetree/bindings/ata/brcm,sata-brcm.txt similarity index 81% rename from Documentation/devicetree/bindings/ata/brcm,sata-brcmstb.txt rename to Documentation/devicetree/bindings/ata/brcm,sata-brcm.txt index 60872838f1ad..0a5b3b47f217 100644 --- a/Documentation/devicetree/bindings/ata/brcm,sata-brcmstb.txt +++ b/Documentation/devicetree/bindings/ata/brcm,sata-brcm.txt @@ -1,13 +1,14 @@ -* Broadcom SATA3 AHCI Controller for STB +* Broadcom SATA3 AHCI Controller SATA nodes are defined to describe on-chip Serial ATA controllers. Each SATA controller should have its own node. Required properties: - compatible : should be one or more of - "brcm,bcm7425-ahci" - "brcm,bcm7445-ahci" - "brcm,sata3-ahci" + "brcm,bcm7425-ahci" + "brcm,bcm7445-ahci" + "brcm,bcm-nsp-ahci" + "brcm,sata3-ahci" - reg : register mappings for AHCI and SATA_TOP_CTRL - reg-names : "ahci" and "top-ctrl" - interrupts : interrupt mapping for SATA IRQ diff --git a/Documentation/devicetree/bindings/extcon/extcon-arizona.txt b/Documentation/devicetree/bindings/extcon/extcon-arizona.txt index e27341f8a4c7..7f3d94ae81ff 100644 --- a/Documentation/devicetree/bindings/extcon/extcon-arizona.txt +++ b/Documentation/devicetree/bindings/extcon/extcon-arizona.txt @@ -46,7 +46,8 @@ Optional properties: The second cell represents the MICBIAS to be used. The third cell represents the value of the micd-pol-gpio pin. - - wlf,gpsw : Settings for the general purpose switch + - wlf,gpsw : Settings for the general purpose switch, set as one of the + ARIZONA_GPSW_XXX defines. Example: diff --git a/Documentation/devicetree/bindings/gpio/cirrus,clps711x-mctrl-gpio.txt b/Documentation/devicetree/bindings/gpio/cirrus,clps711x-mctrl-gpio.txt index 94ae9f82dcf8..fd42e7280f72 100644 --- a/Documentation/devicetree/bindings/gpio/cirrus,clps711x-mctrl-gpio.txt +++ b/Documentation/devicetree/bindings/gpio/cirrus,clps711x-mctrl-gpio.txt @@ -1,7 +1,7 @@ * ARM Cirrus Logic CLPS711X SYSFLG1 MCTRL GPIOs Required properties: -- compatible: Should contain "cirrus,clps711x-mctrl-gpio". +- compatible: Should contain "cirrus,ep7209-mctrl-gpio". - gpio-controller: Marks the device node as a gpio controller. - #gpio-cells: Should be two. The first cell is the pin number and the second cell is used to specify the gpio polarity: @@ -11,7 +11,7 @@ Required properties: Example: sysgpio: sysgpio { compatible = "cirrus,ep7312-mctrl-gpio", - "cirrus,clps711x-mctrl-gpio"; + "cirrus,ep7209-mctrl-gpio"; gpio-controller; #gpio-cells = <2>; }; diff --git a/Documentation/devicetree/bindings/gpio/gpio-clps711x.txt b/Documentation/devicetree/bindings/gpio/gpio-clps711x.txt index e0d0446a6b78..0a304ad29d81 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-clps711x.txt +++ b/Documentation/devicetree/bindings/gpio/gpio-clps711x.txt @@ -1,7 +1,7 @@ Cirrus Logic CLPS711X GPIO controller Required properties: -- compatible: Should be "cirrus,clps711x-gpio" +- compatible: Should be "cirrus,ep7209-gpio" - reg: Physical base GPIO controller registers location and length. There should be two registers, first is DATA register, the second is DIRECTION. @@ -21,7 +21,7 @@ aliases { }; porta: gpio@80000000 { - compatible = "cirrus,clps711x-gpio"; + compatible = "cirrus,ep7312-gpio","cirrus,ep7209-gpio"; reg = <0x80000000 0x1>, <0x80000040 0x1>; gpio-controller; #gpio-cells = <2>; diff --git a/Documentation/devicetree/bindings/gpio/gpio-max77620.txt b/Documentation/devicetree/bindings/gpio/gpio-max77620.txt new file mode 100644 index 000000000000..410e716fd3d2 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/gpio-max77620.txt @@ -0,0 +1,25 @@ +GPIO driver for MAX77620 Power management IC from Maxim Semiconductor. + +Device has 8 GPIO pins which can be configured as GPIO as well as the +special IO functions. + +Required properties: +------------------- +- gpio-controller : Marks the device node as a gpio controller. +- #gpio-cells : Should be two. The first cell is the pin number and + the second cell is used to specify the gpio polarity: + 0 = active high + 1 = active low +For more details, please refer generic GPIO DT binding document +. + +Example: +-------- +#include +... +max77620@3c { + compatible = "maxim,max77620"; + + gpio-controller; + #gpio-cells = <2>; +}; diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca953x.txt b/Documentation/devicetree/bindings/gpio/gpio-pca953x.txt index 6b4a98f74be3..08dd15f89ba9 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-pca953x.txt +++ b/Documentation/devicetree/bindings/gpio/gpio-pca953x.txt @@ -21,6 +21,7 @@ Required properties: maxim,max7313 maxim,max7315 ti,pca6107 + ti,pca9536 ti,tca6408 ti,tca6416 ti,tca6424 diff --git a/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt b/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt index f60e2f477e93..8da26b35b5c3 100644 --- a/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt +++ b/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt @@ -7,6 +7,7 @@ Required Properties: - "renesas,gpio-r8a7779": for R8A7779 (R-Car H1) compatible GPIO controller. - "renesas,gpio-r8a7790": for R8A7790 (R-Car H2) compatible GPIO controller. - "renesas,gpio-r8a7791": for R8A7791 (R-Car M2-W) compatible GPIO controller. + - "renesas,gpio-r8a7792": for R8A7792 (R-Car V2H) compatible GPIO controller. - "renesas,gpio-r8a7793": for R8A7793 (R-Car M2-N) compatible GPIO controller. - "renesas,gpio-r8a7794": for R8A7794 (R-Car E2) compatible GPIO controller. - "renesas,gpio-r8a7795": for R8A7795 (R-Car H3) compatible GPIO controller. diff --git a/Documentation/devicetree/bindings/hwmon/apm-xgene-hwmon.txt b/Documentation/devicetree/bindings/hwmon/apm-xgene-hwmon.txt new file mode 100644 index 000000000000..59b38557f1bb --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/apm-xgene-hwmon.txt @@ -0,0 +1,14 @@ +APM X-Gene hwmon driver + +APM X-Gene SOC sensors are accessed over the "SLIMpro" mailbox. + +Required properties : + - compatible : should be "apm,xgene-slimpro-hwmon" + - mboxes : use the label reference for the mailbox as the first parameter. + The second parameter is the channel number. + +Example : + hwmonslimpro { + compatible = "apm,xgene-slimpro-hwmon"; + mboxes = <&mailbox 7>; + }; diff --git a/Documentation/devicetree/bindings/hwmon/jc42.txt b/Documentation/devicetree/bindings/hwmon/jc42.txt new file mode 100644 index 000000000000..07a250498fbb --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/jc42.txt @@ -0,0 +1,42 @@ +Properties for Jedec JC-42.4 compatible temperature sensors + +Required properties: +- compatible: May include a device-specific string consisting of the + manufacturer and the name of the chip. A list of supported + chip names follows. + Must include "jedec,jc-42.4-temp" for any Jedec JC-42.4 + compatible temperature sensor. + + Supported chip names: + adi,adt7408 + atmel,at30ts00 + atmel,at30tse004 + onnn,cat6095 + onnn,cat34ts02 + maxim,max6604 + microchip,mcp9804 + microchip,mcp9805 + microchip,mcp9808 + microchip,mcp98243 + microchip,mcp98244 + microchip,mcp9843 + nxp,se97 + nxp,se98 + st,stts2002 + st,stts2004 + st,stts3000 + st,stts424 + st,stts424e + idt,tse2002 + idt,tse2004 + idt,ts3000 + idt,ts3001 + +- reg: I2C address + +Example: + +temp-sensor@1a { + compatible = "jedec,jc-42.4-temp"; + reg = <0x1a>; +}; diff --git a/Documentation/devicetree/bindings/i2c/trivial-devices.txt b/Documentation/devicetree/bindings/i2c/trivial-devices.txt index 539874490492..acc5cd64711c 100644 --- a/Documentation/devicetree/bindings/i2c/trivial-devices.txt +++ b/Documentation/devicetree/bindings/i2c/trivial-devices.txt @@ -56,6 +56,70 @@ maxim,ds1050 5 Bit Programmable, Pulse-Width Modulator maxim,max1237 Low-Power, 4-/12-Channel, 2-Wire Serial, 12-Bit ADCs maxim,max6625 9-Bit/12-Bit Temperature Sensors with I²C-Compatible Serial Interface mc,rv3029c2 Real Time Clock Module with I2C-Bus +microchip,mcp4531-502 Microchip 7-bit Single I2C Digital Potentiometer (5k) +microchip,mcp4531-103 Microchip 7-bit Single I2C Digital Potentiometer (10k) +microchip,mcp4531-503 Microchip 7-bit Single I2C Digital Potentiometer (50k) +microchip,mcp4531-104 Microchip 7-bit Single I2C Digital Potentiometer (100k) +microchip,mcp4532-502 Microchip 7-bit Single I2C Digital Potentiometer (5k) +microchip,mcp4532-103 Microchip 7-bit Single I2C Digital Potentiometer (10k) +microchip,mcp4532-503 Microchip 7-bit Single I2C Digital Potentiometer (50k) +microchip,mcp4532-104 Microchip 7-bit Single I2C Digital Potentiometer (100k) +microchip,mcp4541-502 Microchip 7-bit Single I2C Digital Potentiometer with NV Memory (5k) +microchip,mcp4541-103 Microchip 7-bit Single I2C Digital Potentiometer with NV Memory (10k) +microchip,mcp4541-503 Microchip 7-bit Single I2C Digital Potentiometer with NV Memory (50k) +microchip,mcp4541-104 Microchip 7-bit Single I2C Digital Potentiometer with NV Memory (100k) +microchip,mcp4542-502 Microchip 7-bit Single I2C Digital Potentiometer with NV Memory (5k) +microchip,mcp4542-103 Microchip 7-bit Single I2C Digital Potentiometer with NV Memory (10k) +microchip,mcp4542-503 Microchip 7-bit Single I2C Digital Potentiometer with NV Memory (50k) +microchip,mcp4542-104 Microchip 7-bit Single I2C Digital Potentiometer with NV Memory (100k) +microchip,mcp4551-502 Microchip 8-bit Single I2C Digital Potentiometer (5k) +microchip,mcp4551-103 Microchip 8-bit Single I2C Digital Potentiometer (10k) +microchip,mcp4551-503 Microchip 8-bit Single I2C Digital Potentiometer (50k) +microchip,mcp4551-104 Microchip 8-bit Single I2C Digital Potentiometer (100k) +microchip,mcp4552-502 Microchip 8-bit Single I2C Digital Potentiometer (5k) +microchip,mcp4552-103 Microchip 8-bit Single I2C Digital Potentiometer (10k) +microchip,mcp4552-503 Microchip 8-bit Single I2C Digital Potentiometer (50k) +microchip,mcp4552-104 Microchip 8-bit Single I2C Digital Potentiometer (100k) +microchip,mcp4561-502 Microchip 8-bit Single I2C Digital Potentiometer with NV Memory (5k) +microchip,mcp4561-103 Microchip 8-bit Single I2C Digital Potentiometer with NV Memory (10k) +microchip,mcp4561-503 Microchip 8-bit Single I2C Digital Potentiometer with NV Memory (50k) +microchip,mcp4561-104 Microchip 8-bit Single I2C Digital Potentiometer with NV Memory (100k) +microchip,mcp4562-502 Microchip 8-bit Single I2C Digital Potentiometer with NV Memory (5k) +microchip,mcp4562-103 Microchip 8-bit Single I2C Digital Potentiometer with NV Memory (10k) +microchip,mcp4562-503 Microchip 8-bit Single I2C Digital Potentiometer with NV Memory (50k) +microchip,mcp4562-104 Microchip 8-bit Single I2C Digital Potentiometer with NV Memory (100k) +microchip,mcp4631-502 Microchip 7-bit Dual I2C Digital Potentiometer (5k) +microchip,mcp4631-103 Microchip 7-bit Dual I2C Digital Potentiometer (10k) +microchip,mcp4631-503 Microchip 7-bit Dual I2C Digital Potentiometer (50k) +microchip,mcp4631-104 Microchip 7-bit Dual I2C Digital Potentiometer (100k) +microchip,mcp4632-502 Microchip 7-bit Dual I2C Digital Potentiometer (5k) +microchip,mcp4632-103 Microchip 7-bit Dual I2C Digital Potentiometer (10k) +microchip,mcp4632-503 Microchip 7-bit Dual I2C Digital Potentiometer (50k) +microchip,mcp4632-104 Microchip 7-bit Dual I2C Digital Potentiometer (100k) +microchip,mcp4641-502 Microchip 7-bit Dual I2C Digital Potentiometer with NV Memory (5k) +microchip,mcp4641-103 Microchip 7-bit Dual I2C Digital Potentiometer with NV Memory (10k) +microchip,mcp4641-503 Microchip 7-bit Dual I2C Digital Potentiometer with NV Memory (50k) +microchip,mcp4641-104 Microchip 7-bit Dual I2C Digital Potentiometer with NV Memory (100k) +microchip,mcp4642-502 Microchip 7-bit Dual I2C Digital Potentiometer with NV Memory (5k) +microchip,mcp4642-103 Microchip 7-bit Dual I2C Digital Potentiometer with NV Memory (10k) +microchip,mcp4642-503 Microchip 7-bit Dual I2C Digital Potentiometer with NV Memory (50k) +microchip,mcp4642-104 Microchip 7-bit Dual I2C Digital Potentiometer with NV Memory (100k) +microchip,mcp4651-502 Microchip 8-bit Dual I2C Digital Potentiometer (5k) +microchip,mcp4651-103 Microchip 8-bit Dual I2C Digital Potentiometer (10k) +microchip,mcp4651-503 Microchip 8-bit Dual I2C Digital Potentiometer (50k) +microchip,mcp4651-104 Microchip 8-bit Dual I2C Digital Potentiometer (100k) +microchip,mcp4652-502 Microchip 8-bit Dual I2C Digital Potentiometer (5k) +microchip,mcp4652-103 Microchip 8-bit Dual I2C Digital Potentiometer (10k) +microchip,mcp4652-503 Microchip 8-bit Dual I2C Digital Potentiometer (50k) +microchip,mcp4652-104 Microchip 8-bit Dual I2C Digital Potentiometer (100k) +microchip,mcp4661-502 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (5k) +microchip,mcp4661-103 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (10k) +microchip,mcp4661-503 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (50k) +microchip,mcp4661-104 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (100k) +microchip,mcp4662-502 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (5k) +microchip,mcp4662-103 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (10k) +microchip,mcp4662-503 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (50k) +microchip,mcp4662-104 Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (100k) national,lm63 Temperature sensor with integrated fan control national,lm75 I2C TEMP SENSOR national,lm80 Serial Interface ACPI-Compatible Microprocessor System Hardware Monitor diff --git a/Documentation/devicetree/bindings/iio/adc/brcm,iproc-static-adc.txt b/Documentation/devicetree/bindings/iio/adc/brcm,iproc-static-adc.txt new file mode 100644 index 000000000000..caaaed765ce4 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/adc/brcm,iproc-static-adc.txt @@ -0,0 +1,41 @@ +* Broadcom's IPROC Static ADC controller + +Broadcom iProc ADC controller has 8 channels 10bit ADC. +Allows user to convert analog input voltage values to digital. + +Required properties: + +- compatible: Must be "brcm,iproc-static-adc" + +- adc-syscon: Handler of syscon node defining physical base address of the + controller and length of memory mapped region. + +- #io-channel-cells = <1>; As ADC has multiple outputs + refer to Documentation/devicetree/bindings/iio/iio-bindings.txt for details. + +- io-channel-ranges: + refer to Documentation/devicetree/bindings/iio/iio-bindings.txt for details. + +- clocks: Clock used for this block. + +- clock-names: Clock name should be given as tsc_clk. + +- interrupts: interrupt line number. + +For example: + + ts_adc_syscon: ts_adc_syscon@180a6000 { + compatible = "brcm,iproc-ts-adc-syscon","syscon"; + reg = <0x180a6000 0xc30>; + }; + + adc: adc@180a6000 { + compatible = "brcm,iproc-static-adc"; + adc-syscon = <&ts_adc_syscon>; + #io-channel-cells = <1>; + io-channel-ranges; + clocks = <&asiu_clks BCM_CYGNUS_ASIU_ADC_CLK>; + clock-names = "tsc_clk"; + interrupts = ; + status = "disabled"; + }; diff --git a/Documentation/devicetree/bindings/iio/adc/max1363.txt b/Documentation/devicetree/bindings/iio/adc/max1363.txt new file mode 100644 index 000000000000..94a9011dd860 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/adc/max1363.txt @@ -0,0 +1,63 @@ +* Maxim 1x3x/136x/116xx Analog to Digital Converter (ADC) + +The node for this driver must be a child node of a I2C controller, hence +all mandatory properties for your controller must be specified. See directory: + + Documentation/devicetree/bindings/i2c + +for more details. + +Required properties: + - compatible: Should be one of + "maxim,max1361" + "maxim,max1362" + "maxim,max1363" + "maxim,max1364" + "maxim,max1036" + "maxim,max1037" + "maxim,max1038" + "maxim,max1039" + "maxim,max1136" + "maxim,max1137" + "maxim,max1138" + "maxim,max1139" + "maxim,max1236" + "maxim,max1237" + "maxim,max1238" + "maxim,max1239" + "maxim,max11600" + "maxim,max11601" + "maxim,max11602" + "maxim,max11603" + "maxim,max11604" + "maxim,max11605" + "maxim,max11606" + "maxim,max11607" + "maxim,max11608" + "maxim,max11609" + "maxim,max11610" + "maxim,max11611" + "maxim,max11612" + "maxim,max11613" + "maxim,max11614" + "maxim,max11615" + "maxim,max11616" + "maxim,max11617" + "maxim,max11644" + "maxim,max11645" + "maxim,max11646" + "maxim,max11647" + - reg: Should contain the ADC I2C address + +Optional properties: + - vcc-supply: phandle to the regulator that provides power to the ADC. + - vref-supply: phandle to the regulator for ADC reference voltage. + - interrupts: IRQ line for the ADC. If not used the driver will use + polling. + +Example: +adc: max11644@36 { + compatible = "maxim,max11644"; + reg = <0x36>; + vref-supply = <&adc_vref>; +}; diff --git a/Documentation/devicetree/bindings/iio/chemical/atlas,ec-sm.txt b/Documentation/devicetree/bindings/iio/chemical/atlas,ec-sm.txt new file mode 100644 index 000000000000..2962bd9a2b3d --- /dev/null +++ b/Documentation/devicetree/bindings/iio/chemical/atlas,ec-sm.txt @@ -0,0 +1,22 @@ +* Atlas Scientific EC-SM OEM sensor + +http://www.atlas-scientific.com/_files/_datasheets/_oem/EC_oem_datasheet.pdf + +Required properties: + + - compatible: must be "atlas,ec-sm" + - reg: the I2C address of the sensor + - interrupt-parent: should be the phandle for the interrupt controller + - interrupts: the sole interrupt generated by the device + + Refer to interrupt-controller/interrupts.txt for generic interrupt client + node bindings. + +Example: + +atlas@64 { + compatible = "atlas,ec-sm"; + reg = <0x64>; + interrupt-parent = <&gpio1>; + interrupts = <16 2>; +}; diff --git a/Documentation/devicetree/bindings/iio/dac/ad5755.txt b/Documentation/devicetree/bindings/iio/dac/ad5755.txt new file mode 100644 index 000000000000..f0bbd7e1029b --- /dev/null +++ b/Documentation/devicetree/bindings/iio/dac/ad5755.txt @@ -0,0 +1,124 @@ +* Analog Device AD5755 IIO Multi-Channel DAC Linux Driver + +Required properties: + - compatible: Has to contain one of the following: + adi,ad5755 + adi,ad5755-1 + adi,ad5757 + adi,ad5735 + adi,ad5737 + + - reg: spi chip select number for the device + - spi-cpha or spi-cpol: is the only modes that is supported + +Recommended properties: + - spi-max-frequency: Definition as per + Documentation/devicetree/bindings/spi/spi-bus.txt + +Optional properties: +See include/dt-bindings/iio/ad5755.h + - adi,ext-dc-dc-compenstation-resistor: boolean set if the hardware have an + external resistor and thereby bypasses + the internal compensation resistor. + - adi,dc-dc-phase: + Valid values for DC DC Phase control is: + 0: All dc-to-dc converters clock on the same edge. + 1: Channel A and Channel B clock on the same edge, + Channel C and Channel D clock on opposite edges. + 2: Channel A and Channel C clock on the same edge, + Channel B and Channel D clock on opposite edges. + 3: Channel A, Channel B, Channel C, and Channel D + clock 90 degrees out of phase from each other. + - adi,dc-dc-freq-hz: + Valid values for DC DC frequency is [Hz]: + 250000 + 410000 + 650000 + - adi,dc-dc-max-microvolt: + Valid values for the maximum allowed Vboost voltage supplied by + the dc-to-dc converter is: + 23000000 + 24500000 + 27000000 + 29500000 + +Optional for every channel: + - adi,mode: + Valid values for DAC modes is: + 0: 0 V to 5 V voltage range. + 1: 0 V to 10 V voltage range. + 2: Plus minus 5 V voltage range. + 3: Plus minus 10 V voltage range. + 4: 4 mA to 20 mA current range. + 5: 0 mA to 20 mA current range. + 6: 0 mA to 24 mA current range. + - adi,ext-current-sense-resistor: boolean set if the hardware a external + current sense resistor. + - adi,enable-voltage-overrange: boolean enable voltage overrange + - adi,slew: Array of slewrate settings should contain 3 fields: + 1: Should be either 0 or 1 in order to enable or disable slewrate. + 2: Slew rate settings: + Valid values for the slew rate update frequency: + 64000 + 32000 + 16000 + 8000 + 4000 + 2000 + 1000 + 500 + 250 + 125 + 64 + 32 + 16 + 8 + 4 + 0 + 3: Slew step size: + Valid values for the step size LSBs: + 1 + 2 + 4 + 16 + 32 + 64 + 128 + 256 + +Example: +dac@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "adi,ad5755"; + reg = <0>; + spi-max-frequency = <1000000>; + spi-cpha; + adi,dc-dc-phase = <0>; + adi,dc-dc-freq-hz = <410000>; + adi,dc-dc-max-microvolt = <23000000>; + channel@0 { + reg = <0>; + adi,mode = <4>; + adi,ext-current-sense-resistor; + adi,slew = <0 64000 1>; + }; + channel@1 { + reg = <1>; + adi,mode = <4>; + adi,ext-current-sense-resistor; + adi,slew = <0 64000 1>; + }; + channel@2 { + reg = <2>; + adi,mode = <4>; + adi,ext-current-sense-resistor; + adi,slew = <0 64000 1>; + }; + channel@3 { + reg = <3>; + adi,mode = <4>; + adi,ext-current-sense-resistor; + adi,slew = <0 64000 1>; + }; +}; diff --git a/Documentation/devicetree/bindings/iio/pressure/bmp085.txt b/Documentation/devicetree/bindings/iio/pressure/bmp085.txt index d7a6deb6b21e..c7198a03c906 100644 --- a/Documentation/devicetree/bindings/iio/pressure/bmp085.txt +++ b/Documentation/devicetree/bindings/iio/pressure/bmp085.txt @@ -1,7 +1,11 @@ -BMP085/BMP18x digital pressure sensors +BMP085/BMP18x/BMP28x digital pressure sensors Required properties: -- compatible: bosch,bmp085 +- compatible: must be one of: + "bosch,bmp085" + "bosch,bmp180" + "bosch,bmp280" + "bosch,bme280" Optional properties: - chip-id: configurable chip id for non-default chip revisions @@ -10,6 +14,10 @@ Optional properties: value range is 0-3 with rising sensitivity. - interrupt-parent: should be the phandle for the interrupt controller - interrupts: interrupt mapping for IRQ +- reset-gpios: a GPIO line handling reset of the sensor: as the line is + active low, it should be marked GPIO_ACTIVE_LOW (see gpio/gpio.txt) +- vddd-supply: digital voltage regulator (see regulator/regulator.txt) +- vdda-supply: analog voltage regulator (see regulator/regulator.txt) Example: @@ -21,4 +29,7 @@ pressure@77 { default-oversampling = <2>; interrupt-parent = <&gpio0>; interrupts = <25 IRQ_TYPE_EDGE_RISING>; + reset-gpios = <&gpio0 26 GPIO_ACTIVE_LOW>; + vddd-supply = <&foo>; + vdda-supply = <&bar>; }; diff --git a/Documentation/devicetree/bindings/iio/st-sensors.txt b/Documentation/devicetree/bindings/iio/st-sensors.txt index 5844cf72862d..e41fe340162b 100644 --- a/Documentation/devicetree/bindings/iio/st-sensors.txt +++ b/Documentation/devicetree/bindings/iio/st-sensors.txt @@ -64,3 +64,4 @@ Pressure sensors: - st,lps001wp-press - st,lps25h-press - st,lps331ap-press +- st,lps22hb-press diff --git a/Documentation/devicetree/bindings/interrupt-controller/arm,gic.txt b/Documentation/devicetree/bindings/interrupt-controller/arm,gic.txt index 793c20ff8fcc..5393e2a45a42 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/arm,gic.txt +++ b/Documentation/devicetree/bindings/interrupt-controller/arm,gic.txt @@ -21,6 +21,7 @@ Main node required properties: "arm,pl390" "arm,tc11mp-gic" "brcm,brahma-b15-gic" + "nvidia,tegra210-agic" "qcom,msm-8660-qgic" "qcom,msm-qgic2" - interrupt-controller : Identifies the node as an interrupt controller @@ -68,7 +69,7 @@ Optional "ic_clk" (for "arm,arm11mp-gic") "PERIPHCLKEN" (for "arm,cortex-a15-gic") "PERIPHCLK", "PERIPHCLKEN" (for "arm,cortex-a9-gic") - "clk" (for "arm,gic-400") + "clk" (for "arm,gic-400" and "nvidia,tegra210") "gclk" (for "arm,pl390") - power-domains : A phandle and PM domain specifier as defined by bindings of diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-vic.txt b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-vic.txt new file mode 100644 index 000000000000..6c6e85324b9d --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-vic.txt @@ -0,0 +1,22 @@ +Aspeed Vectored Interrupt Controller + +These bindings are for the Aspeed AST2400 interrupt controller register layout. +The SoC has an legacy register layout, but this driver does not support that +mode of operation. + +Required properties: + +- compatible : should be "aspeed,ast2400-vic". + +- interrupt-controller : Identifies the node as an interrupt controller +- #interrupt-cells : Specifies the number of cells needed to encode an + interrupt source. The value shall be 1. + +Example: + + vic: interrupt-controller@1e6c0080 { + compatible = "aspeed,ast2400-vic"; + interrupt-controller; + #interrupt-cells = <1>; + reg = <0x1e6c0080 0x80>; + }; diff --git a/Documentation/devicetree/bindings/media/mediatek-vcodec.txt b/Documentation/devicetree/bindings/media/mediatek-vcodec.txt new file mode 100644 index 000000000000..59a47a5b924b --- /dev/null +++ b/Documentation/devicetree/bindings/media/mediatek-vcodec.txt @@ -0,0 +1,59 @@ +Mediatek Video Codec + +Mediatek Video Codec is the video codec hw present in Mediatek SoCs which +supports high resolution encoding functionalities. + +Required properties: +- compatible : "mediatek,mt8173-vcodec-enc" for encoder +- reg : Physical base address of the video codec registers and length of + memory mapped region. +- interrupts : interrupt number to the cpu. +- mediatek,larb : must contain the local arbiters in the current Socs. +- clocks : list of clock specifiers, corresponding to entries in + the clock-names property. +- clock-names: encoder must contain "venc_sel_src", "venc_sel", +- "venc_lt_sel_src", "venc_lt_sel". +- iommus : should point to the respective IOMMU block with master port as + argument, see Documentation/devicetree/bindings/iommu/mediatek,iommu.txt + for details. +- mediatek,vpu : the node of video processor unit + +Example: +vcodec_enc: vcodec@0x18002000 { + compatible = "mediatek,mt8173-vcodec-enc"; + reg = <0 0x18002000 0 0x1000>, /*VENC_SYS*/ + <0 0x19002000 0 0x1000>; /*VENC_LT_SYS*/ + interrupts = , + ; + mediatek,larb = <&larb3>, + <&larb5>; + iommus = <&iommu M4U_PORT_VENC_RCPU>, + <&iommu M4U_PORT_VENC_REC>, + <&iommu M4U_PORT_VENC_BSDMA>, + <&iommu M4U_PORT_VENC_SV_COMV>, + <&iommu M4U_PORT_VENC_RD_COMV>, + <&iommu M4U_PORT_VENC_CUR_LUMA>, + <&iommu M4U_PORT_VENC_CUR_CHROMA>, + <&iommu M4U_PORT_VENC_REF_LUMA>, + <&iommu M4U_PORT_VENC_REF_CHROMA>, + <&iommu M4U_PORT_VENC_NBM_RDMA>, + <&iommu M4U_PORT_VENC_NBM_WDMA>, + <&iommu M4U_PORT_VENC_RCPU_SET2>, + <&iommu M4U_PORT_VENC_REC_FRM_SET2>, + <&iommu M4U_PORT_VENC_BSDMA_SET2>, + <&iommu M4U_PORT_VENC_SV_COMA_SET2>, + <&iommu M4U_PORT_VENC_RD_COMA_SET2>, + <&iommu M4U_PORT_VENC_CUR_LUMA_SET2>, + <&iommu M4U_PORT_VENC_CUR_CHROMA_SET2>, + <&iommu M4U_PORT_VENC_REF_LUMA_SET2>, + <&iommu M4U_PORT_VENC_REC_CHROMA_SET2>; + mediatek,vpu = <&vpu>; + clocks = <&topckgen CLK_TOP_VENCPLL_D2>, + <&topckgen CLK_TOP_VENC_SEL>, + <&topckgen CLK_TOP_UNIVPLL1_D2>, + <&topckgen CLK_TOP_VENC_LT_SEL>; + clock-names = "venc_sel_src", + "venc_sel", + "venc_lt_sel_src", + "venc_lt_sel"; + }; diff --git a/Documentation/devicetree/bindings/media/mediatek-vpu.txt b/Documentation/devicetree/bindings/media/mediatek-vpu.txt new file mode 100644 index 000000000000..2a5bac37f9a2 --- /dev/null +++ b/Documentation/devicetree/bindings/media/mediatek-vpu.txt @@ -0,0 +1,31 @@ +* Mediatek Video Processor Unit + +Video Processor Unit is a HW video controller. It controls HW Codec including +H.264/VP8/VP9 Decode, H.264/VP8 Encode and Image Processor (scale/rotate/color convert). + +Required properties: + - compatible: "mediatek,mt8173-vpu" + - reg: Must contain an entry for each entry in reg-names. + - reg-names: Must include the following entries: + "tcm": tcm base + "cfg_reg": Main configuration registers base + - interrupts: interrupt number to the cpu. + - clocks : clock name from clock manager + - clock-names: must be main. It is the main clock of VPU + +Optional properties: + - memory-region: phandle to a node describing memory (see + Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt) + to be used for VPU extended memory; if not present, VPU may be located + anywhere in the memory + +Example: + vpu: vpu@10020000 { + compatible = "mediatek,mt8173-vpu"; + reg = <0 0x10020000 0 0x30000>, + <0 0x10050000 0 0x100>; + reg-names = "tcm", "cfg_reg"; + interrupts = ; + clocks = <&topckgen TOP_SCP_SEL>; + clock-names = "main"; + }; diff --git a/Documentation/devicetree/bindings/media/renesas,fcp.txt b/Documentation/devicetree/bindings/media/renesas,fcp.txt new file mode 100644 index 000000000000..6a12960609d8 --- /dev/null +++ b/Documentation/devicetree/bindings/media/renesas,fcp.txt @@ -0,0 +1,32 @@ +Renesas R-Car Frame Compression Processor (FCP) +----------------------------------------------- + +The FCP is a companion module of video processing modules in the Renesas R-Car +Gen3 SoCs. It provides data compression and decompression, data caching, and +conversion of AXI transactions in order to reduce the memory bandwidth. + +There are three types of FCP: FCP for Codec (FCPC), FCP for VSP (FCPV) and FCP +for FDP (FCPF). Their configuration and behaviour depend on the module they +are paired with. These DT bindings currently support the FCPV only. + + - compatible: Must be one or more of the following + + - "renesas,r8a7795-fcpv" for R8A7795 (R-Car H3) compatible 'FCP for VSP' + - "renesas,fcpv" for generic compatible 'FCP for VSP' + + When compatible with the generic version, nodes must list the + SoC-specific version corresponding to the platform first, followed by the + family-specific and/or generic versions. + + - reg: the register base and size for the device registers + - clocks: Reference to the functional clock + + +Device node example +------------------- + + fcpvd1: fcp@fea2f000 { + compatible = "renesas,r8a7795-fcpv", "renesas,fcpv"; + reg = <0 0xfea2f000 0 0x200>; + clocks = <&cpg CPG_MOD 602>; + }; diff --git a/Documentation/devicetree/bindings/media/renesas,vsp1.txt b/Documentation/devicetree/bindings/media/renesas,vsp1.txt index 627405abd144..9b695bcbf219 100644 --- a/Documentation/devicetree/bindings/media/renesas,vsp1.txt +++ b/Documentation/devicetree/bindings/media/renesas,vsp1.txt @@ -14,6 +14,11 @@ Required properties: - interrupts: VSP interrupt specifier. - clocks: A phandle + clock-specifier pair for the VSP functional clock. +Optional properties: + + - renesas,fcp: A phandle referencing the FCP that handles memory accesses + for the VSP. Not needed on Gen2, mandatory on Gen3. + Example: R8A7790 (R-Car H2) VSP1-S node diff --git a/Documentation/devicetree/bindings/media/s5p-cec.txt b/Documentation/devicetree/bindings/media/s5p-cec.txt new file mode 100644 index 000000000000..925ab4d72eaa --- /dev/null +++ b/Documentation/devicetree/bindings/media/s5p-cec.txt @@ -0,0 +1,31 @@ +* Samsung HDMI CEC driver + +The HDMI CEC module is present is Samsung SoCs and its purpose is to +handle communication between HDMI connected devices over the CEC bus. + +Required properties: + - compatible : value should be following + "samsung,s5p-cec" + + - reg : Physical base address of the IP registers and length of memory + mapped region. + + - interrupts : HDMI CEC interrupt number to the CPU. + - clocks : from common clock binding: handle to HDMI CEC clock. + - clock-names : from common clock binding: must contain "hdmicec", + corresponding to entry in the clocks property. + - samsung,syscon-phandle - phandle to the PMU system controller + +Example: + +hdmicec: cec@100B0000 { + compatible = "samsung,s5p-cec"; + reg = <0x100B0000 0x200>; + interrupts = <0 114 0>; + clocks = <&clock CLK_HDMI_CEC>; + clock-names = "hdmicec"; + samsung,syscon-phandle = <&pmu_system_controller>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_cec>; + status = "okay"; +}; diff --git a/Documentation/devicetree/bindings/media/s5p-mfc.txt b/Documentation/devicetree/bindings/media/s5p-mfc.txt index 2d5787eac91a..92c94f5ecbf1 100644 --- a/Documentation/devicetree/bindings/media/s5p-mfc.txt +++ b/Documentation/devicetree/bindings/media/s5p-mfc.txt @@ -21,15 +21,18 @@ Required properties: - clock-names : from common clock binding: must contain "mfc", corresponding to entry in the clocks property. - - samsung,mfc-r : Base address of the first memory bank used by MFC - for DMA contiguous memory allocation and its size. - - - samsung,mfc-l : Base address of the second memory bank used by MFC - for DMA contiguous memory allocation and its size. - Optional properties: - power-domains : power-domain property defined with a phandle to respective power domain. + - memory-region : from reserved memory binding: phandles to two reserved + memory regions, first is for "left" mfc memory bus interfaces, + second if for the "right" mfc memory bus, used when no SYSMMU + support is available + +Obsolete properties: + - samsung,mfc-r, samsung,mfc-l : support removed, please use memory-region + property instead + Example: SoC specific DT entry: @@ -43,9 +46,29 @@ mfc: codec@13400000 { clock-names = "mfc"; }; +Reserved memory specific DT entry for given board (see reserved memory binding +for more information): + +reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + mfc_left: region@51000000 { + compatible = "shared-dma-pool"; + no-map; + reg = <0x51000000 0x800000>; + }; + + mfc_right: region@43000000 { + compatible = "shared-dma-pool"; + no-map; + reg = <0x43000000 0x800000>; + }; +}; + Board specific DT entry: codec@13400000 { - samsung,mfc-r = <0x43000000 0x800000>; - samsung,mfc-l = <0x51000000 0x800000>; + memory-region = <&mfc_left>, <&mfc_right>; }; diff --git a/Documentation/devicetree/bindings/mfd/axp20x.txt b/Documentation/devicetree/bindings/mfd/axp20x.txt index d20b1034e967..585a95546288 100644 --- a/Documentation/devicetree/bindings/mfd/axp20x.txt +++ b/Documentation/devicetree/bindings/mfd/axp20x.txt @@ -22,6 +22,11 @@ Optional properties: AXP152/20X: range: 750-1875, Default: 1.5 MHz AXP22X/80X: range: 1800-4050, Default: 3 MHz +- x-powers,drive-vbus-en: axp221 / axp223 only boolean, set this when the + N_VBUSEN pin is used as an output pin to control an external + regulator to drive the OTG VBus, rather then as an input pin + which signals whether the board is driving OTG VBus or not. + - -supply: a phandle to the regulator supply node. May be omitted if inputs are unregulated, such as using the IPSOUT output from the PMIC. @@ -79,6 +84,7 @@ ELDO3 : LDO : eldoin-supply : shared supply LDO_IO0 : LDO : ips-supply : GPIO 0 LDO_IO1 : LDO : ips-supply : GPIO 1 RTC_LDO : LDO : ips-supply : always on +DRIVEVBUS : Enable output : drivevbus-supply : external regulator AXP809 regulators, type, and corresponding input supply names: diff --git a/Documentation/devicetree/bindings/mfd/rn5t618.txt b/Documentation/devicetree/bindings/mfd/rn5t618.txt index 937785a3eddc..9e6770b105c9 100644 --- a/Documentation/devicetree/bindings/mfd/rn5t618.txt +++ b/Documentation/devicetree/bindings/mfd/rn5t618.txt @@ -1,18 +1,21 @@ -* Ricoh RN5T618 PMIC +* Ricoh RN5T567/RN5T618 PMIC -Ricoh RN5T618 is a power management IC which integrates 3 step-down -DCDC converters, 7 low-dropout regulators, a Li-ion battery charger, -fuel gauge, ADC, GPIOs and a watchdog timer. It can be controlled -through a I2C interface. +Ricoh RN5T567/RN5T618 is a power management IC family which integrates +3 to 4 step-down DCDC converters, 7 low-dropout regulators, GPIOs and +a watchdog timer. The RN5T618 provides additionally a Li-ion battery +charger, fuel gauge and an ADC. It can be controlled through an I2C +interface. Required properties: - - compatible: should be "ricoh,rn5t618" + - compatible: must be one of + "ricoh,rn5t567" + "ricoh,rn5t618" - reg: the I2C slave address of the device Sub-nodes: - regulators: the node is required if the regulator functionality is - needed. The valid regulator names are: DCDC1, DCDC2, DCDC3, LDO1, - LDO2, LDO3, LDO4, LDO5, LDORTC1 and LDORTC2. + needed. The valid regulator names are: DCDC1, DCDC2, DCDC3, DCDC4 + (RN5T567), LDO1, LDO2, LDO3, LDO4, LDO5, LDORTC1 and LDORTC2. The common bindings for each individual regulator can be found in: Documentation/devicetree/bindings/regulator/regulator.txt diff --git a/Documentation/devicetree/bindings/misc/ramoops.txt b/Documentation/devicetree/bindings/misc/ramoops.txt new file mode 100644 index 000000000000..cd02cec67d38 --- /dev/null +++ b/Documentation/devicetree/bindings/misc/ramoops.txt @@ -0,0 +1,48 @@ +Ramoops oops/panic logger +========================= + +ramoops provides persistent RAM storage for oops and panics, so they can be +recovered after a reboot. It is a backend to pstore, so this node is named +"ramoops" after the backend, rather than "pstore" which is the subsystem. + +Parts of this storage may be set aside for other persistent log buffers, such +as kernel log messages, or for optional ECC error-correction data. The total +size of these optional buffers must fit in the reserved region. + +Any remaining space will be used for a circular buffer of oops and panic +records. These records have a configurable size, with a size of 0 indicating +that they should be disabled. + +At least one of "record-size", "console-size", "ftrace-size", or "pmsg-size" +must be set non-zero, but are otherwise optional as listed below. + + +Required properties: + +- compatible: must be "ramoops" + +- memory-region: phandle to a region of memory that is preserved between + reboots + + +Optional properties: + +- ecc-size: enables ECC support and specifies ECC buffer size in bytes + (defaults to 0: no ECC) + +- record-size: maximum size in bytes of each dump done on oops/panic + (defaults to 0: disabled) + +- console-size: size in bytes of log buffer reserved for kernel messages + (defaults to 0: disabled) + +- ftrace-size: size in bytes of log buffer reserved for function tracing and + profiling (defaults to 0: disabled) + +- pmsg-size: size in bytes of log buffer reserved for userspace messages + (defaults to 0: disabled) + +- unbuffered: if present, use unbuffered mappings to map the reserved region + (defaults to buffered mappings) + +- no-dump-oops: if present, only dump panics (defaults to panics and oops) diff --git a/Documentation/devicetree/bindings/net/apm-xgene-mdio.txt b/Documentation/devicetree/bindings/net/apm-xgene-mdio.txt new file mode 100644 index 000000000000..78722d74cea8 --- /dev/null +++ b/Documentation/devicetree/bindings/net/apm-xgene-mdio.txt @@ -0,0 +1,37 @@ +APM X-Gene SoC MDIO node + +MDIO node is defined to describe on-chip MDIO controller. + +Required properties: + - compatible: Must be "apm,xgene-mdio-rgmii" or "apm,xgene-mdio-xfi" + - #address-cells: Must be <1>. + - #size-cells: Must be <0>. + - reg: Address and length of the register set + - clocks: Reference to the clock entry + +For the phys on the mdio bus, there must be a node with the following fields: + - compatible: PHY identifier. Please refer ./phy.txt for the format. + - reg: The ID number for the phy. + +Example: + + mdio: mdio@17020000 { + compatible = "apm,xgene-mdio-rgmii"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x17020000 0x0 0xd100>; + clocks = <&menetclk 0>; + }; + + /* Board-specific peripheral configurations */ + &mdio { + menetphy: phy@3 { + reg = <0x3>; + }; + sgenet0phy: phy@4 { + reg = <0x4>; + }; + sgenet1phy: phy@5 { + reg = <0x5>; + }; + }; diff --git a/Documentation/devicetree/bindings/net/brcm,mdio-mux-iproc.txt b/Documentation/devicetree/bindings/net/brcm,mdio-mux-iproc.txt new file mode 100644 index 000000000000..dfe287a5d6f2 --- /dev/null +++ b/Documentation/devicetree/bindings/net/brcm,mdio-mux-iproc.txt @@ -0,0 +1,59 @@ +Properties for an MDIO bus multiplexer found in Broadcom iProc based SoCs. + +This MDIO bus multiplexer defines buses that could be internal as well as +external to SoCs and could accept MDIO transaction compatible to C-22 or +C-45 Clause. When child bus is selected, one needs to select these two +properties as well to generate desired MDIO transaction on appropriate bus. + +Required properties in addition to the generic multiplexer properties: + +MDIO multiplexer node: +- compatible: brcm,mdio-mux-iproc. + +Every non-ethernet PHY requires a compatible so that it could be probed based +on this compatible string. + +Additional information regarding generic multiplexer properties can be found +at- Documentation/devicetree/bindings/net/mdio-mux.txt + + +for example: + mdio_mux_iproc: mdio-mux@6602023c { + compatible = "brcm,mdio-mux-iproc"; + reg = <0x6602023c 0x14>; + #address-cells = <1>; + #size-cells = <0>; + + mdio@0 { + reg = <0x0>; + #address-cells = <1>; + #size-cells = <0>; + + pci_phy0: pci-phy@0 { + compatible = "brcm,ns2-pcie-phy"; + reg = <0x0>; + #phy-cells = <0>; + }; + }; + + mdio@7 { + reg = <0x7>; + #address-cells = <1>; + #size-cells = <0>; + + pci_phy1: pci-phy@0 { + compatible = "brcm,ns2-pcie-phy"; + reg = <0x0>; + #phy-cells = <0>; + }; + }; + mdio@10 { + reg = <0x10>; + #address-cells = <1>; + #size-cells = <0>; + + gphy0: eth-phy@10 { + reg = <0x10>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/net/can/rcar_canfd.txt b/Documentation/devicetree/bindings/net/can/rcar_canfd.txt new file mode 100644 index 000000000000..22a6f10bab05 --- /dev/null +++ b/Documentation/devicetree/bindings/net/can/rcar_canfd.txt @@ -0,0 +1,96 @@ +Renesas R-Car CAN FD controller Device Tree Bindings +---------------------------------------------------- + +Required properties: +- compatible: Must contain one or more of the following: + - "renesas,rcar-gen3-canfd" for R-Car Gen3 compatible controller. + - "renesas,r8a7795-canfd" for R8A7795 (R-Car H3) compatible controller. + + When compatible with the generic version, nodes must list the + SoC-specific version corresponding to the platform first, followed by the + family-specific and/or generic versions. + +- reg: physical base address and size of the R-Car CAN FD register map. +- interrupts: interrupt specifier for the Global & Channel interrupts +- clocks: phandles and clock specifiers for 3 clock inputs. +- clock-names: 3 clock input name strings: "fck", "canfd", "can_clk". +- pinctrl-0: pin control group to be used for this controller. +- pinctrl-names: must be "default". + +Required child nodes: +The controller supports two channels and each is represented as a child node. +The name of the child nodes are "channel0" and "channel1" respectively. Each +child node supports the "status" property only, which is used to +enable/disable the respective channel. + +Required properties for "renesas,r8a7795-canfd" compatible: +In R8A7795 SoC, canfd clock is a div6 clock and can be used by both CAN +and CAN FD controller at the same time. It needs to be scaled to maximum +frequency if any of these controllers use it. This is done using the +below properties. + +- assigned-clocks: phandle of canfd clock. +- assigned-clock-rates: maximum frequency of this clock. + +Optional property: +The controller can operate in either CAN FD only mode (default) or +Classical CAN only mode. The mode is global to both the channels. In order to +enable the later, define the following optional property. + - renesas,no-can-fd: puts the controller in Classical CAN only mode. + +Example +------- + +SoC common .dtsi file: + + canfd: can@e66c0000 { + compatible = "renesas,r8a7795-canfd", + "renesas,rcar-gen3-canfd"; + reg = <0 0xe66c0000 0 0x8000>; + interrupts = , + ; + clocks = <&cpg CPG_MOD 914>, + <&cpg CPG_CORE R8A7795_CLK_CANFD>, + <&can_clk>; + clock-names = "fck", "canfd", "can_clk"; + assigned-clocks = <&cpg CPG_CORE R8A7795_CLK_CANFD>; + assigned-clock-rates = <40000000>; + power-domains = <&cpg>; + status = "disabled"; + + channel0 { + status = "disabled"; + }; + + channel1 { + status = "disabled"; + }; + }; + +Board specific .dts file: + +E.g. below enables Channel 1 alone in the board in Classical CAN only mode. + +&canfd { + pinctrl-0 = <&canfd1_pins>; + pinctrl-names = "default"; + renesas,no-can-fd; + status = "okay"; + + channel1 { + status = "okay"; + }; +}; + +E.g. below enables Channel 0 alone in the board using External clock +as fCAN clock. + +&canfd { + pinctrl-0 = <&canfd0_pins &can_clk_pins>; + pinctrl-names = "default"; + status = "okay"; + + channel0 { + status = "okay"; + }; +}; diff --git a/Documentation/devicetree/bindings/net/cirrus,cs89x0.txt b/Documentation/devicetree/bindings/net/cirrus,cs89x0.txt new file mode 100644 index 000000000000..c070076bacb9 --- /dev/null +++ b/Documentation/devicetree/bindings/net/cirrus,cs89x0.txt @@ -0,0 +1,13 @@ +* Cirrus Logic CS8900/CS8920 Network Controller + +Required properties: +- compatible : Should be "cirrus,cs8900" or "cirrus,cs8920". +- reg : Address and length of the IO space. +- interrupts : Should contain the controller interrupt line. + +Examples: + eth0: eth@10000000 { + compatible = "cirrus,cs8900"; + reg = <0x10000000 0x400>; + interrupts = <10>; + }; diff --git a/Documentation/devicetree/bindings/net/cpsw.txt b/Documentation/devicetree/bindings/net/cpsw.txt index 0ae06491b430..5ad439f30135 100644 --- a/Documentation/devicetree/bindings/net/cpsw.txt +++ b/Documentation/devicetree/bindings/net/cpsw.txt @@ -15,7 +15,6 @@ Required properties: - cpdma_channels : Specifies number of channels in CPDMA - ale_entries : Specifies No of entries ALE can hold - bd_ram_size : Specifies internal descriptor RAM size -- rx_descs : Specifies number of Rx descriptors - mac_control : Specifies Default MAC control register content for the specific platform - slaves : Specifies number for slaves diff --git a/Documentation/devicetree/bindings/net/davinci-mdio.txt b/Documentation/devicetree/bindings/net/davinci-mdio.txt index 0369e25aabd2..621156ca4ffd 100644 --- a/Documentation/devicetree/bindings/net/davinci-mdio.txt +++ b/Documentation/devicetree/bindings/net/davinci-mdio.txt @@ -2,7 +2,10 @@ TI SoC Davinci/Keystone2 MDIO Controller Device Tree Bindings --------------------------------------------------- Required properties: -- compatible : Should be "ti,davinci_mdio" or "ti,keystone_mdio" +- compatible : Should be "ti,davinci_mdio" + and "ti,keystone_mdio" for Keystone 2 SoCs + and "ti,cpsw-mdio" for am335x, am472x, am57xx/dra7, dm814x SoCs + and "ti,am4372-mdio" for am472x SoC - reg : physical base address and size of the davinci mdio registers map - bus_freq : Mdio Bus frequency diff --git a/Documentation/devicetree/bindings/net/dsa/b53.txt b/Documentation/devicetree/bindings/net/dsa/b53.txt new file mode 100644 index 000000000000..d6c6e41648d4 --- /dev/null +++ b/Documentation/devicetree/bindings/net/dsa/b53.txt @@ -0,0 +1,97 @@ +Broadcom BCM53xx Ethernet switches +================================== + +Required properties: + +- compatible: For external switch chips, compatible string must be exactly one + of: "brcm,bcm5325" + "brcm,bcm53115" + "brcm,bcm53125" + "brcm,bcm53128" + "brcm,bcm5365" + "brcm,bcm5395" + "brcm,bcm5397" + "brcm,bcm5398" + + For the BCM5310x SoCs with an integrated switch, must be one of: + "brcm,bcm53010-srab" + "brcm,bcm53011-srab" + "brcm,bcm53012-srab" + "brcm,bcm53018-srab" + "brcm,bcm53019-srab" and the mandatory "brcm,bcm5301x-srab" string + + For the BCM585xx/586XX/88312 SoCs with an integrated switch, must be one of: + "brcm,bcm58522-srab" + "brcm,bcm58523-srab" + "brcm,bcm58525-srab" + "brcm,bcm58622-srab" + "brcm,bcm58623-srab" + "brcm,bcm58625-srab" + "brcm,bcm88312-srab" and the mandatory "brcm,nsp-srab string + + For the BCM63xx/33xx SoCs with an integrated switch, must be one of: + "brcm,bcm3384-switch" + "brcm,bcm6328-switch" + "brcm,bcm6368-switch" and the mandatory "brcm,bcm63xx-switch" + +See Documentation/devicetree/bindings/dsa/dsa.txt for a list of additional +required and optional properties. + +Examples: + +Ethernet switch connected via MDIO to the host, CPU port wired to eth0: + + eth0: ethernet@10001000 { + compatible = "brcm,unimac"; + reg = <0x10001000 0x1000>; + + fixed-link { + speed = <1000>; + duplex-full; + }; + }; + + mdio0: mdio@10000000 { + compatible = "brcm,unimac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + switch0: ethernet-switch@30 { + compatible = "brcm,bcm53125"; + #address-cells = <1>; + #size-cells = <0>; + + ports { + port0@0 { + reg = <0>; + label = "lan1"; + }; + + port1@1 { + reg = <1>; + label = "lan2"; + }; + + port5@5 { + reg = <5>; + label = "cable-modem"; + fixed-link { + speed = <1000>; + duplex-full; + }; + phy-mode = "rgmii-txid"; + }; + + port8@8 { + reg = <8>; + label = "cpu"; + fixed-link { + speed = <1000>; + duplex-full; + }; + phy-mode = "rgmii-txid"; + ethernet = <ð0>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/net/dsa/dsa.txt b/Documentation/devicetree/bindings/net/dsa/dsa.txt index 9f4807f90c31..9bbbe7f87d67 100644 --- a/Documentation/devicetree/bindings/net/dsa/dsa.txt +++ b/Documentation/devicetree/bindings/net/dsa/dsa.txt @@ -1,5 +1,279 @@ -Marvell Distributed Switch Architecture Device Tree Bindings ------------------------------------------------------------- +Distributed Switch Architecture Device Tree Bindings +---------------------------------------------------- + +Two bindings exist, one of which has been deprecated due to +limitations. + +Current Binding +--------------- + +Switches are true Linux devices and can be probes by any means. Once +probed, they register to the DSA framework, passing a node +pointer. This node is expected to fulfil the following binding, and +may contain additional properties as required by the device it is +embedded within. + +Required properties: + +- ports : A container for child nodes representing switch ports. + +Optional properties: + +- dsa,member : A two element list indicates which DSA cluster, and position + within the cluster a switch takes. <0 0> is cluster 0, + switch 0. <0 1> is cluster 0, switch 1. <1 0> is cluster 1, + switch 0. A switch not part of any cluster (single device + hanging off a CPU port) must not specify this property + +The ports container has the following properties + +Required properties: + +- #address-cells : Must be 1 +- #size-cells : Must be 0 + +Each port children node must have the following mandatory properties: +- reg : Describes the port address in the switch +- label : Describes the label associated with this port, which + will become the netdev name. Special labels are + "cpu" to indicate a CPU port and "dsa" to + indicate an uplink/downlink port between switches in + the cluster. + +A port labelled "dsa" has the following mandatory property: + +- link : Should be a list of phandles to other switch's DSA + port. This port is used as the outgoing port + towards the phandle ports. The full routing + information must be given, not just the one hop + routes to neighbouring switches. + +A port labelled "cpu" has the following mandatory property: + +- ethernet : Should be a phandle to a valid Ethernet device node. + This host device is what the switch port is + connected to. + +Port child nodes may also contain the following optional standardised +properties, described in binding documents: + +- phy-handle : Phandle to a PHY on an MDIO bus. See + Documentation/devicetree/bindings/net/ethernet.txt + for details. + +- phy-mode : See + Documentation/devicetree/bindings/net/ethernet.txt + for details. + +- fixed-link : Fixed-link subnode describing a link to a non-MDIO + managed entity. See + Documentation/devicetree/bindings/net/fixed-link.txt + for details. + +Example + +The following example shows three switches on three MDIO busses, +linked into one DSA cluster. + +&mdio1 { + #address-cells = <1>; + #size-cells = <0>; + + switch0: switch0@0 { + compatible = "marvell,mv88e6085"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + dsa,member = <0 0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + label = "lan0"; + }; + + port@1 { + reg = <1>; + label = "lan1"; + }; + + port@2 { + reg = <2>; + label = "lan2"; + }; + + switch0port5: port@5 { + reg = <5>; + label = "dsa"; + phy-mode = "rgmii-txid"; + link = <&switch1port6 + &switch2port9>; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; + + port@6 { + reg = <6>; + label = "cpu"; + ethernet = <&fec1>; + fixed-link { + speed = <100>; + full-duplex; + }; + }; + }; + }; +}; + +&mdio2 { + #address-cells = <1>; + #size-cells = <0>; + + switch1: switch1@0 { + compatible = "marvell,mv88e6085"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + dsa,member = <0 1>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + label = "lan3"; + phy-handle = <&switch1phy0>; + }; + + port@1 { + reg = <1>; + label = "lan4"; + phy-handle = <&switch1phy1>; + }; + + port@2 { + reg = <2>; + label = "lan5"; + phy-handle = <&switch1phy2>; + }; + + switch1port5: port@5 { + reg = <5>; + label = "dsa"; + link = <&switch2port9>; + phy-mode = "rgmii-txid"; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; + + switch1port6: port@6 { + reg = <6>; + label = "dsa"; + phy-mode = "rgmii-txid"; + link = <&switch0port5>; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; + }; + mdio-bus { + #address-cells = <1>; + #size-cells = <0>; + switch1phy0: switch1phy0@0 { + reg = <0>; + }; + switch1phy1: switch1phy0@1 { + reg = <1>; + }; + switch1phy2: switch1phy0@2 { + reg = <2>; + }; + }; + }; +}; + +&mdio4 { + #address-cells = <1>; + #size-cells = <0>; + + switch2: switch2@0 { + compatible = "marvell,mv88e6085"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + dsa,member = <0 2>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + label = "lan6"; + }; + + port@1 { + reg = <1>; + label = "lan7"; + }; + + port@2 { + reg = <2>; + label = "lan8"; + }; + + port@3 { + reg = <3>; + label = "optical3"; + fixed-link { + speed = <1000>; + full-duplex; + link-gpios = <&gpio6 2 + GPIO_ACTIVE_HIGH>; + }; + }; + + port@4 { + reg = <4>; + label = "optical4"; + fixed-link { + speed = <1000>; + full-duplex; + link-gpios = <&gpio6 3 + GPIO_ACTIVE_HIGH>; + }; + }; + + switch2port9: port@9 { + reg = <9>; + label = "dsa"; + phy-mode = "rgmii-txid"; + link = <&switch1port5 + &switch0port5>; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; + }; + }; +}; + +Deprecated Binding +------------------ + +The deprecated binding makes use of a platform device to represent the +switches. The switches themselves are not Linux devices, and make use +of an MDIO bus for management. Required properties: - compatible : Should be "marvell,dsa" diff --git a/Documentation/devicetree/bindings/net/hisilicon-femac-mdio.txt b/Documentation/devicetree/bindings/net/hisilicon-femac-mdio.txt new file mode 100644 index 000000000000..23a39a309d17 --- /dev/null +++ b/Documentation/devicetree/bindings/net/hisilicon-femac-mdio.txt @@ -0,0 +1,22 @@ +Hisilicon Fast Ethernet MDIO Controller interface + +Required properties: +- compatible: should be "hisilicon,hisi-femac-mdio". +- reg: address and length of the register set for the device. +- clocks: A phandle to the reference clock for this device. + +- PHY subnode: inherits from phy binding [1] +[1] Documentation/devicetree/bindings/net/phy.txt + +Example: +mdio: mdio@10091100 { + compatible = "hisilicon,hisi-femac-mdio"; + reg = <0x10091100 0x10>; + clocks = <&crg HI3516CV300_MDIO_CLK>; + #address-cells = <1>; + #size-cells = <0>; + + phy0: phy@1 { + reg = <1>; + }; +}; diff --git a/Documentation/devicetree/bindings/net/hisilicon-femac.txt b/Documentation/devicetree/bindings/net/hisilicon-femac.txt new file mode 100644 index 000000000000..d11af5ecace8 --- /dev/null +++ b/Documentation/devicetree/bindings/net/hisilicon-femac.txt @@ -0,0 +1,39 @@ +Hisilicon Fast Ethernet MAC controller + +Required properties: +- compatible: should contain one of the following version strings: + * "hisilicon,hisi-femac-v1" + * "hisilicon,hisi-femac-v2" + and the soc string "hisilicon,hi3516cv300-femac". +- reg: specifies base physical address(s) and size of the device registers. + The first region is the MAC core register base and size. + The second region is the global MAC control register. +- interrupts: should contain the MAC interrupt. +- clocks: A phandle to the MAC main clock. +- resets: should contain the phandle to the MAC reset signal(required) and + the PHY reset signal(optional). +- reset-names: should contain the reset signal name "mac"(required) + and "phy"(optional). +- mac-address: see ethernet.txt [1]. +- phy-mode: see ethernet.txt [1]. +- phy-handle: see ethernet.txt [1]. +- hisilicon,phy-reset-delays-us: triplet of delays if PHY reset signal given. + The 1st cell is reset pre-delay in micro seconds. + The 2nd cell is reset pulse in micro seconds. + The 3rd cell is reset post-delay in micro seconds. + +[1] Documentation/devicetree/bindings/net/ethernet.txt + +Example: + hisi_femac: ethernet@10090000 { + compatible = "hisilicon,hi3516cv300-femac","hisilicon,hisi-femac-v2"; + reg = <0x10090000 0x1000>,<0x10091300 0x200>; + interrupts = <12>; + clocks = <&crg HI3518EV200_ETH_CLK>; + resets = <&crg 0xec 0>,<&crg 0xec 3>; + reset-names = "mac","phy"; + mac-address = [00 00 00 00 00 00]; + phy-mode = "mii"; + phy-handle = <&phy0>; + hisilicon,phy-reset-delays-us = <10000 20000 20000>; + }; diff --git a/Documentation/devicetree/bindings/net/keystone-netcp.txt b/Documentation/devicetree/bindings/net/keystone-netcp.txt index b30ab6b5cbfa..04ba1dc34fd6 100644 --- a/Documentation/devicetree/bindings/net/keystone-netcp.txt +++ b/Documentation/devicetree/bindings/net/keystone-netcp.txt @@ -2,7 +2,7 @@ This document describes the device tree bindings associated with the keystone network coprocessor(NetCP) driver support. The network coprocessor (NetCP) is a hardware accelerator that processes -Ethernet packets. NetCP has a gigabit Ethernet (GbE) subsytem with a ethernet +Ethernet packets. NetCP has a gigabit Ethernet (GbE) subsystem with a ethernet switch sub-module to send and receive packets. NetCP also includes a packet accelerator (PA) module to perform packet classification operations such as header matching, and packet modification operations such as checksum diff --git a/Documentation/devicetree/bindings/net/mdio-mux.txt b/Documentation/devicetree/bindings/net/mdio-mux.txt index 491f5bd55203..f58571f36570 100644 --- a/Documentation/devicetree/bindings/net/mdio-mux.txt +++ b/Documentation/devicetree/bindings/net/mdio-mux.txt @@ -5,11 +5,12 @@ numbered uniquely in a device dependent manner. The nodes for an MDIO bus multiplexer/switch will have one child node for each child bus. Required properties: -- mdio-parent-bus : phandle to the parent MDIO bus. - #address-cells = <1>; - #size-cells = <0>; Optional properties: +- mdio-parent-bus : phandle to the parent MDIO bus. + - Other properties specific to the multiplexer/switch hardware. Required properties for child nodes: diff --git a/Documentation/devicetree/bindings/net/micrel.txt b/Documentation/devicetree/bindings/net/micrel.txt index 87496a8c64ab..8d157f0295a5 100644 --- a/Documentation/devicetree/bindings/net/micrel.txt +++ b/Documentation/devicetree/bindings/net/micrel.txt @@ -35,3 +35,13 @@ Optional properties: supported clocks: - KSZ8021, KSZ8031, KSZ8081, KSZ8091: "rmii-ref": The RMII reference input clock. Used to determine the XI input clock. + + - micrel,fiber-mode: If present the PHY is configured to operate in fiber mode + + Some PHYs, such as the KSZ8041FTL variant, support fiber mode, enabled + by the FXEN boot strapping pin. It can't be determined from the PHY + registers whether the PHY is in fiber mode, so this boolean device tree + property can be used to describe it. + + In fiber mode, auto-negotiation is disabled and the PHY can only work in + 100base-fx (full and half duplex) modes. diff --git a/Documentation/devicetree/bindings/net/rockchip-dwmac.txt b/Documentation/devicetree/bindings/net/rockchip-dwmac.txt index 93eac7ce1446..cccd945fc45b 100644 --- a/Documentation/devicetree/bindings/net/rockchip-dwmac.txt +++ b/Documentation/devicetree/bindings/net/rockchip-dwmac.txt @@ -3,7 +3,8 @@ Rockchip SoC RK3288 10/100/1000 Ethernet driver(GMAC) The device node has following properties. Required properties: - - compatible: Can be one of "rockchip,rk3288-gmac", "rockchip,rk3368-gmac" + - compatible: Can be one of "rockchip,rk3228-gmac", "rockchip,rk3288-gmac", + "rockchip,rk3368-gmac" - reg: addresses and length of the register sets for the device. - interrupts: Should contain the GMAC interrupts. - interrupt-names: Should contain the interrupt names "macirq". diff --git a/Documentation/devicetree/bindings/net/socfpga-dwmac.txt b/Documentation/devicetree/bindings/net/socfpga-dwmac.txt index 72d82d684342..2e68a3cd8513 100644 --- a/Documentation/devicetree/bindings/net/socfpga-dwmac.txt +++ b/Documentation/devicetree/bindings/net/socfpga-dwmac.txt @@ -17,9 +17,26 @@ Required properties: Optional properties: altr,emac-splitter: Should be the phandle to the emac splitter soft IP node if DWMAC controller is connected emac splitter. +phy-mode: The phy mode the ethernet operates in +altr,sgmii-to-sgmii-converter: phandle to the TSE SGMII converter + +This device node has additional phandle dependency, the sgmii converter: + +Required properties: + - compatible : Should be altr,gmii-to-sgmii-2.0 + - reg-names : Should be "eth_tse_control_port" Example: +gmii_to_sgmii_converter: phy@0x100000240 { + compatible = "altr,gmii-to-sgmii-2.0"; + reg = <0x00000001 0x00000240 0x00000008>, + <0x00000001 0x00000200 0x00000040>; + reg-names = "eth_tse_control_port"; + clocks = <&sgmii_1_clk_0 &emac1 1 &sgmii_clk_125 &sgmii_clk_125>; + clock-names = "tse_pcs_ref_clk_clock_connection", "tse_rx_cdr_refclk"; +}; + gmac0: ethernet@ff700000 { compatible = "altr,socfpga-stmmac", "snps,dwmac-3.70a", "snps,dwmac"; altr,sysmgr-syscon = <&sysmgr 0x60 0>; @@ -30,4 +47,6 @@ gmac0: ethernet@ff700000 { mac-address = [00 00 00 00 00 00];/* Filled in by U-Boot */ clocks = <&emac_0_clk>; clock-names = "stmmaceth"; + phy-mode = "sgmii"; + altr,gmii-to-sgmii-converter = <&gmii_to_sgmii_converter>; }; diff --git a/Documentation/devicetree/bindings/net/stmmac.txt b/Documentation/devicetree/bindings/net/stmmac.txt index 95816c5fc589..41b49e6075f5 100644 --- a/Documentation/devicetree/bindings/net/stmmac.txt +++ b/Documentation/devicetree/bindings/net/stmmac.txt @@ -47,6 +47,9 @@ Optional properties: supported by this device instance - snps,perfect-filter-entries: Number of perfect filter entries supported by this device instance +- snps,ps-speed: port selection speed that can be passed to the core when + PCS is supported. For example, this is used in case of SGMII + and MAC2MAC connection. - AXI BUS Mode parameters: below the list of all the parameters to program the AXI register inside the DMA module: - snps,lpi_en: enable Low Power Interface diff --git a/Documentation/devicetree/bindings/net/wireless/ti,wlcore,spi.txt b/Documentation/devicetree/bindings/net/wireless/ti,wlcore,spi.txt index 9180724e182c..8f9ced076fe1 100644 --- a/Documentation/devicetree/bindings/net/wireless/ti,wlcore,spi.txt +++ b/Documentation/devicetree/bindings/net/wireless/ti,wlcore,spi.txt @@ -1,19 +1,30 @@ -* Texas Instruments wl1271 wireless lan controller +* Texas Instruments wl12xx/wl18xx wireless lan controller -The wl1271 chip can be connected via SPI or via SDIO. This +The wl12xx/wl18xx chips can be connected via SPI or via SDIO. This document describes the binding for the SPI connected chip. Required properties: -- compatible : Should be "ti,wl1271" +- compatible : Should be one of the following: + * "ti,wl1271" + * "ti,wl1273" + * "ti,wl1281" + * "ti,wl1283" + * "ti,wl1801" + * "ti,wl1805" + * "ti,wl1807" + * "ti,wl1831" + * "ti,wl1835" + * "ti,wl1837" - reg : Chip select address of device - spi-max-frequency : Maximum SPI clocking speed of device in Hz -- ref-clock-frequency : Reference clock frequency - interrupt-parent, interrupts : Should contain parameters for 1 interrupt line. Interrupt parameters: parent, line number, type. -- vwlan-supply : Point the node of the regulator that powers/enable the wl1271 chip +- vwlan-supply : Point the node of the regulator that powers/enable the + wl12xx/wl18xx chip Optional properties: +- ref-clock-frequency : Reference clock frequency (should be set for wl12xx) - clock-xtal : boolean, clock is generated from XTAL - Please consult Documentation/devicetree/bindings/spi/spi-bus.txt @@ -21,16 +32,28 @@ Optional properties: Examples: +For wl12xx family: &spi1 { - wl1271@1 { + wlcore: wlcore@1 { compatible = "ti,wl1271"; - reg = <1>; spi-max-frequency = <48000000>; - clock-xtal; - ref-clock-frequency = <38400000>; interrupt-parent = <&gpio3>; interrupts = <8 IRQ_TYPE_LEVEL_HIGH>; vwlan-supply = <&vwlan_fixed>; + clock-xtal; + ref-clock-frequency = <38400000>; + }; +}; + +For wl18xx family: +&spi0 { + wlcore: wlcore@0 { + compatible = "ti,wl1835"; + reg = <0>; + spi-max-frequency = <48000000>; + interrupt-parent = <&gpio0>; + interrupts = <27 IRQ_TYPE_EDGE_RISING>; + vwlan-supply = <&vwlan_fixed>; }; }; diff --git a/Documentation/devicetree/bindings/phy/brcm,mdio-mux-bus-pci.txt b/Documentation/devicetree/bindings/phy/brcm,mdio-mux-bus-pci.txt new file mode 100644 index 000000000000..5b51007c6f24 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/brcm,mdio-mux-bus-pci.txt @@ -0,0 +1,27 @@ +* Broadcom NS2 PCIe PHY binding document + +Required bus properties: +- reg: MDIO Bus number for the MDIO interface +- #address-cells: must be 1 +- #size-cells: must be 0 + +Required PHY properties: +- compatible: should be "brcm,ns2-pcie-phy" +- reg: MDIO Phy ID for the MDIO interface +- #phy-cells: must be 0 + +This is a child bus node of "brcm,mdio-mux-iproc" node. + +Example: + +mdio@0 { + reg = <0x0>; + #address-cells = <1>; + #size-cells = <0>; + + pci_phy0: pci-phy@0 { + compatible = "brcm,ns2-pcie-phy"; + reg = <0x0>; + #phy-cells = <0>; + }; +}; diff --git a/Documentation/devicetree/bindings/phy/brcm-sata-phy.txt b/Documentation/devicetree/bindings/phy/brcm-sata-phy.txt index d0231209d846..6ccce09d8bbf 100644 --- a/Documentation/devicetree/bindings/phy/brcm-sata-phy.txt +++ b/Documentation/devicetree/bindings/phy/brcm-sata-phy.txt @@ -5,6 +5,7 @@ Required properties: "brcm,bcm7425-sata-phy" "brcm,bcm7445-sata-phy" "brcm,iproc-ns2-sata-phy" + "brcm,iproc-nsp-sata-phy" "brcm,phy-sata3" - address-cells: should be 1 - size-cells: should be 0 @@ -22,7 +23,8 @@ Sub-nodes required properties: Sub-nodes optional properties: - brcm,enable-ssc: use spread spectrum clocking (SSC) on this port - This property is not applicable for "brcm,iproc-ns2-sata-phy". + This property is not applicable for "brcm,iproc-ns2-sata-phy" and + "brcm,iproc-nsp-sata-phy". Example: sata-phy@f0458100 { diff --git a/Documentation/devicetree/bindings/phy/phy-da8xx-usb.txt b/Documentation/devicetree/bindings/phy/phy-da8xx-usb.txt new file mode 100644 index 000000000000..c26478be391b --- /dev/null +++ b/Documentation/devicetree/bindings/phy/phy-da8xx-usb.txt @@ -0,0 +1,40 @@ +TI DA8xx/OMAP-L1xx/AM18xx USB PHY + +Required properties: + - compatible: must be "ti,da830-usb-phy". + - #phy-cells: must be 1. + +This device controls the PHY for both the USB 1.1 OHCI and USB 2.0 OTG +controllers on DA8xx SoCs. Consumers of this device should use index 0 for +the USB 2.0 phy device and index 1 for the USB 1.1 phy device. + +It also requires a "syscon" node with compatible = "ti,da830-cfgchip", "syscon" +to access the CFGCHIP2 register. + +Example: + + cfgchip: cfgchip@1417c { + compatible = "ti,da830-cfgchip", "syscon"; + reg = <0x1417c 0x14>; + }; + + usb_phy: usb-phy { + compatible = "ti,da830-usb-phy"; + #phy-cells = <1>; + }; + + usb20: usb@200000 { + compatible = "ti,da830-musb"; + reg = <0x200000 0x1000>; + interrupts = <58>; + phys = <&usb_phy 0>; + phy-names = "usb-phy"; + }; + + usb11: usb@225000 { + compatible = "ti,da830-ohci"; + reg = <0x225000 0x1000>; + interrupts = <59>; + phys = <&usb_phy 1>; + phy-names = "usb-phy"; + }; diff --git a/Documentation/devicetree/bindings/phy/rockchip-usb-phy.txt b/Documentation/devicetree/bindings/phy/rockchip-usb-phy.txt index 68498d560354..cc6be9680a6d 100644 --- a/Documentation/devicetree/bindings/phy/rockchip-usb-phy.txt +++ b/Documentation/devicetree/bindings/phy/rockchip-usb-phy.txt @@ -5,11 +5,13 @@ Required properties: "rockchip,rk3066a-usb-phy" "rockchip,rk3188-usb-phy" "rockchip,rk3288-usb-phy" - - rockchip,grf : phandle to the syscon managing the "general - register files" - #address-cells: should be 1 - #size-cells: should be 0 +Deprecated properties: + - rockchip,grf : phandle to the syscon managing the "general + register files" - phy should be a child of the GRF instead + Sub-nodes: Each PHY should be represented as a sub-node. @@ -28,14 +30,19 @@ Optional Properties: Example: -usbphy: phy { - compatible = "rockchip,rk3288-usb-phy"; - rockchip,grf = <&grf>; - #address-cells = <1>; - #size-cells = <0>; +grf: syscon@ff770000 { + compatible = "rockchip,rk3288-grf", "syscon", "simple-mfd"; + +... + + usbphy: phy { + compatible = "rockchip,rk3288-usb-phy"; + #address-cells = <1>; + #size-cells = <0>; - usbphy0: usb-phy0 { - #phy-cells = <0>; - reg = <0x320>; + usbphy0: usb-phy0 { + #phy-cells = <0>; + reg = <0x320>; + }; }; }; diff --git a/Documentation/devicetree/bindings/power/max8903-charger.txt b/Documentation/devicetree/bindings/power/max8903-charger.txt new file mode 100644 index 000000000000..f0f4e12b076e --- /dev/null +++ b/Documentation/devicetree/bindings/power/max8903-charger.txt @@ -0,0 +1,25 @@ +Maxim Semiconductor MAX8903 Battery Charger bindings + +Required properties: +- compatible: "maxim,max8903" for MAX8903 Battery Charger +- dok-gpios: Valid DC power has been detected (active low, input), optional if uok-gpios is provided +- uok-gpios: Valid USB power has been detected (active low, input), optional if dok-gpios is provided + +Optional properties: +- cen-gpios: Charge enable pin (active low, output) +- chg-gpios: Charger status pin (active low, input) +- flt-gpios: Fault pin (active low, output) +- dcm-gpios: Current limit mode setting (DC=1 or USB=0, output) +- usus-gpios: USB suspend pin (active high, output) + + +Example: + + max8903-charger { + compatible = "maxim,max8903"; + dok-gpios = <&gpio2 3 GPIO_ACTIVE_LOW>; + flt-gpios = <&gpio2 2 GPIO_ACTIVE_LOW>; + chg-gpios = <&gpio3 15 GPIO_ACTIVE_LOW>; + cen-gpios = <&gpio2 5 GPIO_ACTIVE_LOW>; + status = "okay"; + }; diff --git a/Documentation/devicetree/bindings/reset/brcm,bcm21664-resetmgr.txt b/Documentation/devicetree/bindings/power/reset/brcm,bcm21664-resetmgr.txt similarity index 100% rename from Documentation/devicetree/bindings/reset/brcm,bcm21664-resetmgr.txt rename to Documentation/devicetree/bindings/power/reset/brcm,bcm21664-resetmgr.txt diff --git a/Documentation/devicetree/bindings/power/reset/reboot-mode.txt b/Documentation/devicetree/bindings/power/reset/reboot-mode.txt new file mode 100644 index 000000000000..de34f27d509e --- /dev/null +++ b/Documentation/devicetree/bindings/power/reset/reboot-mode.txt @@ -0,0 +1,25 @@ +Generic reboot mode core map driver + +This driver get reboot mode arguments and call the write +interface to store the magic value in special register +or ram. Then the bootloader can read it and take different +action according to the argument stored. + +All mode properties are vendor specific, it is a indication to tell +the bootloader what to do when the system reboots, and should be named +as mode-xxx = (xxx is mode name, magic should be a none-zero value). + +For example modes common on Android platform: +- mode-normal: Normal reboot mode, system reboot with command "reboot". +- mode-recovery: Android Recovery mode, it is a mode to format the device or update a new image. +- mode-bootloader: Android fastboot mode, it's a mode to re-flash partitions on the Android based device. +- mode-loader: A bootloader mode, it's a mode used to download image on Rockchip platform, + usually used in development. + +Example: + reboot-mode { + mode-normal = ; + mode-recovery = ; + mode-bootloader = ; + mode-loader = ; + } diff --git a/Documentation/devicetree/bindings/power/reset/syscon-reboot-mode.txt b/Documentation/devicetree/bindings/power/reset/syscon-reboot-mode.txt new file mode 100644 index 000000000000..f7ce1d8af04a --- /dev/null +++ b/Documentation/devicetree/bindings/power/reset/syscon-reboot-mode.txt @@ -0,0 +1,35 @@ +SYSCON reboot mode driver + +This driver gets reboot mode magic value form reboot-mode driver +and stores it in a SYSCON mapped register. Then the bootloader +can read it and take different action according to the magic +value stored. + +This DT node should be represented as a sub-node of a "syscon", "simple-mfd" +node. + +Required properties: +- compatible: should be "syscon-reboot-mode" +- offset: offset in the register map for the storage register (in bytes) + +Optional property: +- mask: bits mask of the bits in the register to store the reboot mode magic value, + default set to 0xffffffff if missing. + +The rest of the properties should follow the generic reboot-mode description +found in reboot-mode.txt + +Example: + pmu: pmu@20004000 { + compatible = "rockchip,rk3066-pmu", "syscon", "simple-mfd"; + reg = <0x20004000 0x100>; + + reboot-mode { + compatible = "syscon-reboot-mode"; + offset = <0x40>; + mode-normal = ; + mode-recovery = ; + mode-bootloader = ; + mode-loader = ; + }; + }; diff --git a/Documentation/devicetree/bindings/power_supply/axp20x_usb_power.txt b/Documentation/devicetree/bindings/power_supply/axp20x_usb_power.txt index 862f4a49dc49..f1d7beec45bf 100644 --- a/Documentation/devicetree/bindings/power_supply/axp20x_usb_power.txt +++ b/Documentation/devicetree/bindings/power_supply/axp20x_usb_power.txt @@ -1,7 +1,8 @@ AXP20x USB power supply Required Properties: --compatible: "x-powers,axp202-usb-power-supply" +-compatible: One of: "x-powers,axp202-usb-power-supply" + "x-powers,axp221-usb-power-supply" This node is a subnode of the axp20x PMIC. diff --git a/Documentation/devicetree/bindings/regulator/da9210.txt b/Documentation/devicetree/bindings/regulator/da9210.txt index 7aa9b1fa6b21..58065ca9e3b4 100644 --- a/Documentation/devicetree/bindings/regulator/da9210.txt +++ b/Documentation/devicetree/bindings/regulator/da9210.txt @@ -1,4 +1,4 @@ -* Dialog Semiconductor DA9210 Voltage Regulator +* Dialog Semiconductor DA9210 Multi-phase 12A DCDC BUCK Converter Required properties: @@ -18,8 +18,12 @@ Example: compatible = "dlg,da9210"; reg = <0x68>; - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <1000000>; + interrupt-parent = <...>; + interrupts = <...>; + + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1570000>; + regulator-min-microamp = <1600000>; + regulator-max-microamp = <4600000>; regulator-boot-on; - regulator-always-on; }; diff --git a/Documentation/devicetree/bindings/regulator/da9211.txt b/Documentation/devicetree/bindings/regulator/da9211.txt index c620493e8dbe..0f2a6f8fcafd 100644 --- a/Documentation/devicetree/bindings/regulator/da9211.txt +++ b/Documentation/devicetree/bindings/regulator/da9211.txt @@ -1,7 +1,8 @@ -* Dialog Semiconductor DA9211/DA9213/DA9215 Voltage Regulator +* Dialog Semiconductor DA9211/DA9212/DA9213/DA9214/DA9215 Voltage Regulator Required properties: -- compatible: "dlg,da9211" or "dlg,da9213" or "dlg,da9215" +- compatible: "dlg,da9211" or "dlg,da9212" or "dlg,da9213" + or "dlg,da9214" or "dlg,da9215" - reg: I2C slave address, usually 0x68. - interrupts: the interrupt outputs of the controller - regulators: A node that houses a sub-node for each regulator within the @@ -21,6 +22,25 @@ Example 1) DA9211 reg = <0x68>; interrupts = <3 27>; + regulators { + BUCKA { + regulator-name = "VBUCKA"; + regulator-min-microvolt = < 300000>; + regulator-max-microvolt = <1570000>; + regulator-min-microamp = <2000000>; + regulator-max-microamp = <5000000>; + enable-gpios = <&gpio 27 0>; + }; + }; + }; + +Example 2) DA9212 + + pmic: da9212@68 { + compatible = "dlg,da9212"; + reg = <0x68>; + interrupts = <3 27>; + regulators { BUCKA { regulator-name = "VBUCKA"; @@ -41,12 +61,30 @@ Example 1) DA9211 }; }; -Example 2) DA9213 +Example 3) DA9213 pmic: da9213@68 { compatible = "dlg,da9213"; reg = <0x68>; interrupts = <3 27>; + regulators { + BUCKA { + regulator-name = "VBUCKA"; + regulator-min-microvolt = < 300000>; + regulator-max-microvolt = <1570000>; + regulator-min-microamp = <3000000>; + regulator-max-microamp = <6000000>; + enable-gpios = <&gpio 27 0>; + }; + }; + }; + +Example 4) DA9214 + pmic: da9214@68 { + compatible = "dlg,da9214"; + reg = <0x68>; + interrupts = <3 27>; + regulators { BUCKA { regulator-name = "VBUCKA"; @@ -67,8 +105,7 @@ Example 2) DA9213 }; }; - -Example 3) DA9215 +Example 5) DA9215 pmic: da9215@68 { compatible = "dlg,da9215"; reg = <0x68>; diff --git a/Documentation/devicetree/bindings/regulator/mt6323-regulator.txt b/Documentation/devicetree/bindings/regulator/mt6323-regulator.txt new file mode 100644 index 000000000000..c35d878b0960 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/mt6323-regulator.txt @@ -0,0 +1,237 @@ +Mediatek MT6323 Regulator Driver + +All voltage regulators are defined as subnodes of the regulators node. A list +of regulators provided by this controller are defined as subnodes of the +PMIC's node. Each regulator is named according to its regulator type, +buck_ and ldo_. The definition for each of these nodes is defined +using the standard binding for regulators at +Documentation/devicetree/bindings/regulator/regulator.txt. + +The valid names for regulators are:: +BUCK: + buck_vproc, buck_vsys, buck_vpa +LDO: + ldo_vtcxo, ldo_vcn28, ldo_vcn33_bt, ldo_vcn33_wifi, ldo_va, ldo_vcama, + ldo_vio28, ldo_vusb, ldo_vmc, ldo_vmch, ldo_vemc3v3, ldo_vgp1, ldo_vgp2, + ldo_vgp3, ldo_vcn18, ldo_vsim1, ldo_vsim2, ldo_vrtc, ldo_vcamaf, ldo_vibr, + ldo_vrf18, ldo_vm, ldo_vio18, ldo_vcamd, ldo_vcamio + +Example: + + pmic: mt6323 { + mt6323regulator: regulators { + mt6323_vproc_reg: buck_vproc{ + regulator-name = "vproc"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <12500>; + regulator-always-on; + regulator-boot-on; + }; + + mt6323_vsys_reg: buck_vsys{ + regulator-name = "vsys"; + regulator-min-microvolt = <1400000>; + regulator-max-microvolt = <2987500>; + regulator-ramp-delay = <25000>; + regulator-always-on; + regulator-boot-on; + }; + + mt6323_vpa_reg: buck_vpa{ + regulator-name = "vpa"; + regulator-min-microvolt = < 500000>; + regulator-max-microvolt = <3650000>; + }; + + mt6323_vtcxo_reg: ldo_vtcxo{ + regulator-name = "vtcxo"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-enable-ramp-delay = <90>; + regulator-always-on; + regulator-boot-on; + }; + + mt6323_vcn28_reg: ldo_vcn28{ + regulator-name = "vcn28"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-enable-ramp-delay = <185>; + }; + + mt6323_vcn33_bt_reg: ldo_vcn33_bt{ + regulator-name = "vcn33_bt"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3600000>; + regulator-enable-ramp-delay = <185>; + }; + + mt6323_vcn33_wifi_reg: ldo_vcn33_wifi{ + regulator-name = "vcn33_wifi"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3600000>; + regulator-enable-ramp-delay = <185>; + }; + + mt6323_va_reg: ldo_va{ + regulator-name = "va"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-enable-ramp-delay = <216>; + regulator-always-on; + regulator-boot-on; + }; + + mt6323_vcama_reg: ldo_vcama{ + regulator-name = "vcama"; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <2800000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vio28_reg: ldo_vio28{ + regulator-name = "vio28"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-enable-ramp-delay = <216>; + regulator-always-on; + regulator-boot-on; + }; + + mt6323_vusb_reg: ldo_vusb{ + regulator-name = "vusb"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <216>; + regulator-boot-on; + }; + + mt6323_vmc_reg: ldo_vmc{ + regulator-name = "vmc"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <36>; + regulator-boot-on; + }; + + mt6323_vmch_reg: ldo_vmch{ + regulator-name = "vmch"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <36>; + regulator-boot-on; + }; + + mt6323_vemc3v3_reg: ldo_vemc3v3{ + regulator-name = "vemc3v3"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <36>; + regulator-boot-on; + }; + + mt6323_vgp1_reg: ldo_vgp1{ + regulator-name = "vgp1"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vgp2_reg: ldo_vgp2{ + regulator-name = "vgp2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3000000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vgp3_reg: ldo_vgp3{ + regulator-name = "vgp3"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1800000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vcn18_reg: ldo_vcn18{ + regulator-name = "vcn18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vsim1_reg: ldo_vsim1{ + regulator-name = "vsim1"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vsim2_reg: ldo_vsim2{ + regulator-name = "vsim2"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vrtc_reg: ldo_vrtc{ + regulator-name = "vrtc"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-always-on; + regulator-boot-on; + }; + + mt6323_vcamaf_reg: ldo_vcamaf{ + regulator-name = "vcamaf"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vibr_reg: ldo_vibr{ + regulator-name = "vibr"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <36>; + }; + + mt6323_vrf18_reg: ldo_vrf18{ + regulator-name = "vrf18"; + regulator-min-microvolt = <1825000>; + regulator-max-microvolt = <1825000>; + regulator-enable-ramp-delay = <187>; + }; + + mt6323_vm_reg: ldo_vm{ + regulator-name = "vm"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1800000>; + regulator-enable-ramp-delay = <216>; + regulator-always-on; + regulator-boot-on; + }; + + mt6323_vio18_reg: ldo_vio18{ + regulator-name = "vio18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-enable-ramp-delay = <216>; + regulator-always-on; + regulator-boot-on; + }; + + mt6323_vcamd_reg: ldo_vcamd{ + regulator-name = "vcamd"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1800000>; + regulator-enable-ramp-delay = <216>; + }; + + mt6323_vcamio_reg: ldo_vcamio{ + regulator-name = "vcamio"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-enable-ramp-delay = <216>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/regulator/pwm-regulator.txt b/Documentation/devicetree/bindings/regulator/pwm-regulator.txt index ed936f0f34f2..dd6f59cf1455 100644 --- a/Documentation/devicetree/bindings/regulator/pwm-regulator.txt +++ b/Documentation/devicetree/bindings/regulator/pwm-regulator.txt @@ -38,13 +38,18 @@ NB: To be clear, if voltage-table is provided, then the device will be used in Voltage Table Mode. If no voltage-table is provided, then the device will be used in Continuous Voltage Mode. +Optional properties: +-------------------- +- enable-gpios: GPIO to use to enable/disable the regulator + Any property defined as part of the core regulator binding can also be used. (See: ../regulator/regulator.txt) -Continuous Voltage Example: +Continuous Voltage With Enable GPIO Example: pwm_regulator { compatible = "pwm-regulator; pwms = <&pwm1 0 8448 0>; + enable-gpios = <&gpio0 23 GPIO_ACTIVE_HIGH>; regulator-min-microvolt = <1016000>; regulator-max-microvolt = <1114000>; regulator-name = "vdd_logic"; diff --git a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt index 46c6f3ed1a1c..0fa3b0fac129 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt +++ b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt @@ -113,9 +113,9 @@ pm8916: l14, l15, l16, l17, l18 pm8941: - s1, s2, s3, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, - l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, lvs1, lvs2, lvs3, - mvs1, mvs2 + s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, + l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, lvs1, lvs2, lvs3, + 5vs1, 5vs2 pm8994: s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, l1, l2, l3, l4, l5, diff --git a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt index 07ccdaa68324..26542690b578 100644 --- a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt +++ b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt @@ -2,7 +2,8 @@ BCM2835 Random number generator Required properties: -- compatible : should be "brcm,bcm2835-rng" +- compatible : should be "brcm,bcm2835-rng" or "brcm,bcm-nsp-rng" or + "brcm,bcm5301x-rng" - reg : Specifies base physical address and size of the registers. Example: @@ -11,3 +12,8 @@ rng { compatible = "brcm,bcm2835-rng"; reg = <0x7e104000 0x10>; }; + +rng@18033000 { + compatible = "brcm,bcm-nsp-rng"; + reg = <0x18033000 0x14>; +}; diff --git a/Documentation/devicetree/bindings/serial/8250.txt b/Documentation/devicetree/bindings/serial/8250.txt index 936ab5b87324..f5561ac7e17e 100644 --- a/Documentation/devicetree/bindings/serial/8250.txt +++ b/Documentation/devicetree/bindings/serial/8250.txt @@ -42,6 +42,9 @@ Optional properties: - auto-flow-control: one way to enable automatic flow control support. The driver is allowed to detect support for the capability even without this property. +- {rts,cts,dtr,dsr,rng,dcd}-gpios: specify a GPIO for RTS/CTS/DTR/DSR/RI/DCD + line respectively. It will use specified GPIO instead of the peripheral + function pin for the UART feature. If unsure, don't specify this property. Note: * fsl,ns16550: @@ -63,3 +66,19 @@ Example: interrupts = <10>; reg-shift = <2>; }; + +Example for OMAP UART using GPIO-based modem control signals: + + uart4: serial@49042000 { + compatible = "ti,omap3-uart"; + reg = <0x49042000 0x400>; + interrupts = <80>; + ti,hwmods = "uart4"; + clock-frequency = <48000000>; + cts-gpios = <&gpio3 5 GPIO_ACTIVE_LOW>; + rts-gpios = <&gpio3 6 GPIO_ACTIVE_LOW>; + dtr-gpios = <&gpio1 12 GPIO_ACTIVE_LOW>; + dsr-gpios = <&gpio1 13 GPIO_ACTIVE_LOW>; + dcd-gpios = <&gpio1 14 GPIO_ACTIVE_LOW>; + rng-gpios = <&gpio1 15 GPIO_ACTIVE_LOW>; + }; diff --git a/Documentation/devicetree/bindings/serial/renesas,sci-serial.txt b/Documentation/devicetree/bindings/serial/renesas,sci-serial.txt index 528c3b90f23c..1e4000d83aee 100644 --- a/Documentation/devicetree/bindings/serial/renesas,sci-serial.txt +++ b/Documentation/devicetree/bindings/serial/renesas,sci-serial.txt @@ -31,6 +31,8 @@ Required properties: - "renesas,hscif-r8a7794" for R8A7794 (R-Car E2) HSCIF compatible UART. - "renesas,scif-r8a7795" for R8A7795 (R-Car H3) SCIF compatible UART. - "renesas,hscif-r8a7795" for R8A7795 (R-Car H3) HSCIF compatible UART. + - "renesas,scif-r8a7796" for R8A7796 (R-Car M3-W) SCIF compatible UART. + - "renesas,hscif-r8a7796" for R8A7796 (R-Car M3-W) HSCIF compatible UART. - "renesas,scifa-sh73a0" for SH73A0 (SH-Mobile AG5) SCIFA compatible UART. - "renesas,scifb-sh73a0" for SH73A0 (SH-Mobile AG5) SCIFB compatible UART. - "renesas,rcar-gen1-scif" for R-Car Gen1 SCIF compatible UART, @@ -76,6 +78,10 @@ Optional properties: - dmas: Must contain a list of two references to DMA specifiers, one for transmission, and one for reception. - dma-names: Must contain a list of two DMA names, "tx" and "rx". + - {cts,dsr,dcd,rng,rts,dtr}-gpios: Specify GPIOs for modem lines, cfr. the + generic serial DT bindings in serial.txt. + - uart-has-rtscts: Indicates dedicated lines for RTS/CTS hardware flow + control, cfr. the generic serial DT bindings in serial.txt. Example: aliases { diff --git a/Documentation/devicetree/bindings/timer/oxsemi,rps-timer.txt b/Documentation/devicetree/bindings/timer/oxsemi,rps-timer.txt new file mode 100644 index 000000000000..3ca89cd1caef --- /dev/null +++ b/Documentation/devicetree/bindings/timer/oxsemi,rps-timer.txt @@ -0,0 +1,17 @@ +Oxford Semiconductor OXNAS SoCs Family RPS Timer +================================================ + +Required properties: +- compatible: Should be "oxsemi,ox810se-rps-timer" +- reg : Specifies base physical address and size of the registers. +- interrupts : The interrupts of the two timers +- clocks : The phandle of the timer clock source + +example: + +timer0: timer@200 { + compatible = "oxsemi,ox810se-rps-timer"; + reg = <0x200 0x40>; + clocks = <&rpsclk>; + interrupts = <4 5>; +}; diff --git a/Documentation/devicetree/bindings/timer/rockchip,rk3288-timer.txt b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.txt similarity index 75% rename from Documentation/devicetree/bindings/timer/rockchip,rk3288-timer.txt rename to Documentation/devicetree/bindings/timer/rockchip,rk-timer.txt index 87f0b0042bae..a41b184d5538 100644 --- a/Documentation/devicetree/bindings/timer/rockchip,rk3288-timer.txt +++ b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.txt @@ -1,7 +1,9 @@ -Rockchip rk3288 timer +Rockchip rk timer Required properties: -- compatible: shall be "rockchip,rk3288-timer" +- compatible: shall be one of: + "rockchip,rk3288-timer" - for rk3066, rk3036, rk3188, rk322x, rk3288, rk3368 + "rockchip,rk3399-timer" - for rk3399 - reg: base address of the timer register starting with TIMERS CONTROL register - interrupts: should contain the interrupts for Timer0 - clocks : must contain an entry for each entry in clock-names diff --git a/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.txt b/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.txt index 1084e2bcbe1c..341dc67f3472 100644 --- a/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.txt +++ b/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.txt @@ -93,7 +93,7 @@ Example: phys = <&usb_phy0>; phy-names = "usb-phy"; vbus-supply = <®_usb0_vbus>; - gadget-itc-setting = <0x4>; /* 4 micro-frames */ + itc-setting = <0x4>; /* 4 micro-frames */ /* Incremental burst of unspecified length */ ahb-burst-config = <0x0>; tx-burst-size-dword = <0x10>; /* 64 bytes */ diff --git a/Documentation/devicetree/bindings/usb/usb-ohci.txt b/Documentation/devicetree/bindings/usb/usb-ohci.txt index 19233b7365e1..9df456968596 100644 --- a/Documentation/devicetree/bindings/usb/usb-ohci.txt +++ b/Documentation/devicetree/bindings/usb/usb-ohci.txt @@ -14,7 +14,7 @@ Optional properties: - clocks : a list of phandle + clock specifier pairs - phys : phandle + phy specifier pair - phy-names : "usb" -- resets : phandle + reset specifier pair +- resets : a list of phandle + reset specifier pairs Example: diff --git a/Documentation/dmaengine/provider.txt b/Documentation/dmaengine/provider.txt index 122b7f4876bb..91ce82d5f0c4 100644 --- a/Documentation/dmaengine/provider.txt +++ b/Documentation/dmaengine/provider.txt @@ -323,7 +323,7 @@ supported. * device_resume - Resumes a transfer on the channel - This command should operate synchronously on the channel, - pausing right away the work of the given channel + resuming right away the work of the given channel * device_terminate_all - Aborts all the pending and ongoing transfers on the channel diff --git a/Documentation/filesystems/Locking b/Documentation/filesystems/Locking index 75eea7ce3d7c..5a7386e38e2d 100644 --- a/Documentation/filesystems/Locking +++ b/Documentation/filesystems/Locking @@ -195,7 +195,9 @@ prototypes: int (*releasepage) (struct page *, int); void (*freepage)(struct page *); int (*direct_IO)(struct kiocb *, struct iov_iter *iter); + bool (*isolate_page) (struct page *, isolate_mode_t); int (*migratepage)(struct address_space *, struct page *, struct page *); + void (*putback_page) (struct page *); int (*launder_page)(struct page *); int (*is_partially_uptodate)(struct page *, unsigned long, unsigned long); int (*error_remove_page)(struct address_space *, struct page *); @@ -219,7 +221,9 @@ invalidatepage: yes releasepage: yes freepage: yes direct_IO: +isolate_page: yes migratepage: yes (both) +putback_page: yes launder_page: yes is_partially_uptodate: yes error_remove_page: yes @@ -544,13 +548,13 @@ subsequent truncate), and then return with VM_FAULT_LOCKED, and the page locked. The VM will unlock the page. ->map_pages() is called when VM asks to map easy accessible pages. -Filesystem should find and map pages associated with offsets from "pgoff" -till "max_pgoff". ->map_pages() is called with page table locked and must +Filesystem should find and map pages associated with offsets from "start_pgoff" +till "end_pgoff". ->map_pages() is called with page table locked and must not block. If it's not possible to reach a page without blocking, filesystem should skip it. Filesystem should use do_set_pte() to setup -page table entry. Pointer to entry associated with offset "pgoff" is -passed in "pte" field in vm_fault structure. Pointers to entries for other -offsets should be calculated relative to "pte". +page table entry. Pointer to entry associated with the page is passed in +"pte" field in fault_env structure. Pointers to entries for other offsets +should be calculated relative to "pte". ->page_mkwrite() is called when a previously read-only pte is about to become writeable. The filesystem again must ensure that there are diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt index ce4587d257d2..0c16a22521a8 100644 --- a/Documentation/filesystems/dax.txt +++ b/Documentation/filesystems/dax.txt @@ -49,6 +49,7 @@ These block devices may be used for inspiration: - axonram: Axon DDR2 device driver - brd: RAM backed block device driver - dcssblk: s390 dcss block device driver +- pmem: NVDIMM persistent memory driver Implementation Tips for Filesystem Writers @@ -75,8 +76,9 @@ calls to get_block() (for example by a page-fault racing with a read() or a write()) work correctly. These filesystems may be used for inspiration: -- ext2: the second extended filesystem, see Documentation/filesystems/ext2.txt -- ext4: the fourth extended filesystem, see Documentation/filesystems/ext4.txt +- ext2: see Documentation/filesystems/ext2.txt +- ext4: see Documentation/filesystems/ext4.txt +- xfs: see Documentation/filesystems/xfs.txt Handling Media Errors diff --git a/Documentation/filesystems/f2fs.txt b/Documentation/filesystems/f2fs.txt index e1c9f0849da6..ecd808088362 100644 --- a/Documentation/filesystems/f2fs.txt +++ b/Documentation/filesystems/f2fs.txt @@ -109,7 +109,9 @@ background_gc=%s Turn on/off cleaning operations, namely garbage disable_roll_forward Disable the roll-forward recovery routine norecovery Disable the roll-forward recovery routine, mounted read- only (i.e., -o ro,disable_roll_forward) -discard Issue discard/TRIM commands when a segment is cleaned. +discard/nodiscard Enable/disable real-time discard in f2fs, if discard is + enabled, f2fs will issue discard/TRIM commands when a + segment is cleaned. no_heap Disable heap-style segment allocation which finds free segments for data from the beginning of main area, while for node from the end of main area. @@ -151,6 +153,9 @@ noinline_data Disable the inline data feature, inline data feature is enabled by default. data_flush Enable data flushing before checkpoint in order to persist data of regular and symlink. +mode=%s Control block allocation mode which supports "adaptive" + and "lfs". In "lfs" mode, there should be no random + writes towards main area. ================================================================================ DEBUGFS ENTRIES diff --git a/Documentation/filesystems/ocfs2-online-filecheck.txt b/Documentation/filesystems/ocfs2-online-filecheck.txt index 1ab07860430d..139fab175c8a 100644 --- a/Documentation/filesystems/ocfs2-online-filecheck.txt +++ b/Documentation/filesystems/ocfs2-online-filecheck.txt @@ -5,12 +5,12 @@ This document will describe OCFS2 online file check feature. Introduction ============ -OCFS2 is often used in high-availaibility systems. However, OCFS2 usually +OCFS2 is often used in high-availability systems. However, OCFS2 usually converts the filesystem to read-only when encounters an error. This may not be necessary, since turning the filesystem read-only would affect other running processes as well, decreasing availability. Then, a mount option (errors=continue) is introduced, which would return the --EIO errno to the calling process and terminate furhter processing so that the +-EIO errno to the calling process and terminate further processing so that the filesystem is not corrupted further. The filesystem is not converted to read-only, and the problematic file's inode number is reported in the kernel log. The user can try to check/fix this file via online filecheck feature. @@ -44,7 +44,7 @@ There is a sysfs directory for each OCFS2 file system mounting: /sys/fs/ocfs2//filecheck -Here, indicates the name of OCFS2 volumn device which has been already +Here, indicates the name of OCFS2 volume device which has been already mounted. The file above would accept inode numbers. This could be used to communicate with kernel space, tell which file(inode number) will be checked or fixed. Currently, three operations are supported, which includes checking @@ -76,14 +76,14 @@ The output is like this: This time, the column indicates whether this fix is successful or not. 3. The record cache is used to store the history of check/fix results. It's -defalut size is 10, and can be adjust between the range of 10 ~ 100. You can +default size is 10, and can be adjust between the range of 10 ~ 100. You can adjust the size like this: # echo "" > /sys/fs/ocfs2//filecheck/set Fixing stuff ============ -On receivng the inode, the filesystem would read the inode and the +On receiving the inode, the filesystem would read the inode and the file metadata. In case of errors, the filesystem would fix the errors and report the problems it fixed in the kernel log. As a precautionary measure, the inode must first be checked for errors before performing a final fix. diff --git a/Documentation/filesystems/proc.txt b/Documentation/filesystems/proc.txt index e8d00759bfa5..68080ad6a75e 100644 --- a/Documentation/filesystems/proc.txt +++ b/Documentation/filesystems/proc.txt @@ -436,6 +436,7 @@ Private_Dirty: 0 kB Referenced: 892 kB Anonymous: 0 kB AnonHugePages: 0 kB +ShmemPmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB @@ -464,6 +465,8 @@ accessed. a mapping associated with a file may contain anonymous pages: when MAP_PRIVATE and a page is modified, the file page is replaced by a private anonymous copy. "AnonHugePages" shows the ammount of memory backed by transparent hugepage. +"ShmemPmdMapped" shows the ammount of shared (shmem/tmpfs) memory backed by +huge pages. "Shared_Hugetlb" and "Private_Hugetlb" show the ammounts of memory backed by hugetlbfs page which is *not* counted in "RSS" or "PSS" field for historical reasons. And these are not included in {Shared,Private}_{Clean,Dirty} field. @@ -725,7 +728,7 @@ IRQ, you can set it by doing: > echo 1 > /proc/irq/10/smp_affinity This means that only the first CPU will handle the IRQ, but you can also echo -5 which means that only the first and fourth CPU can handle the IRQ. +5 which means that only the first and third CPU can handle the IRQ. The contents of each smp_affinity file is the same by default: @@ -868,6 +871,9 @@ VmallocTotal: 112216 kB VmallocUsed: 428 kB VmallocChunk: 111088 kB AnonHugePages: 49152 kB +ShmemHugePages: 0 kB +ShmemPmdMapped: 0 kB + MemTotal: Total usable ram (i.e. physical ram minus a few reserved bits and the kernel binary code) @@ -912,6 +918,9 @@ MemAvailable: An estimate of how much memory is available for starting new AnonHugePages: Non-file backed huge pages mapped into userspace page tables Mapped: files which have been mmaped, such as libraries Shmem: Total memory used by shared memory (shmem) and tmpfs +ShmemHugePages: Memory used by shared memory (shmem) and tmpfs allocated + with huge pages +ShmemPmdMapped: Shared memory mapped into userspace with huge pages Slab: in-kernel data structures cache SReclaimable: Part of Slab, that might be reclaimed, such as caches SUnreclaim: Part of Slab, that cannot be reclaimed on memory pressure diff --git a/Documentation/filesystems/vfs.txt b/Documentation/filesystems/vfs.txt index c61a223ef3ff..900360cbcdae 100644 --- a/Documentation/filesystems/vfs.txt +++ b/Documentation/filesystems/vfs.txt @@ -592,9 +592,14 @@ struct address_space_operations { int (*releasepage) (struct page *, int); void (*freepage)(struct page *); ssize_t (*direct_IO)(struct kiocb *, struct iov_iter *iter); + /* isolate a page for migration */ + bool (*isolate_page) (struct page *, isolate_mode_t); /* migrate the contents of a page to the specified target */ int (*migratepage) (struct page *, struct page *); + /* put migration-failed page back to right list */ + void (*putback_page) (struct page *); int (*launder_page) (struct page *); + int (*is_partially_uptodate) (struct page *, unsigned long, unsigned long); void (*is_dirty_writeback) (struct page *, bool *, bool *); @@ -747,6 +752,10 @@ struct address_space_operations { and transfer data directly between the storage and the application's address space. + isolate_page: Called by the VM when isolating a movable non-lru page. + If page is successfully isolated, VM marks the page as PG_isolated + via __SetPageIsolated. + migrate_page: This is used to compact the physical memory usage. If the VM wants to relocate a page (maybe off a memory card that is signalling imminent failure) it will pass a new page @@ -754,6 +763,8 @@ struct address_space_operations { transfer any private data across and update any references that it has to the page. + putback_page: Called by the VM when isolated page's migration fails. + launder_page: Called before freeing a page - it writes back the dirty page. To prevent redirtying the page, it is kept locked during the whole operation. diff --git a/Documentation/gdb-kernel-debugging.txt b/Documentation/gdb-kernel-debugging.txt index 4ab7d43d0754..7050ce8794b9 100644 --- a/Documentation/gdb-kernel-debugging.txt +++ b/Documentation/gdb-kernel-debugging.txt @@ -139,27 +139,6 @@ Examples of using the Linux-provided gdb helpers start_comm = "swapper/2\000\000\000\000\000\000" } - o Dig into a radix tree data structure, such as the IRQ descriptors: - (gdb) print (struct irq_desc)$lx_radix_tree_lookup(irq_desc_tree, 18) - $6 = { - irq_common_data = { - state_use_accessors = 67584, - handler_data = 0x0 <__vectors_start>, - msi_desc = 0x0 <__vectors_start>, - affinity = {{ - bits = {65535} - }} - }, - irq_data = { - mask = 0, - irq = 18, - hwirq = 27, - common = 0xee803d80, - chip = 0xc0eb0854 , - domain = 0xee808000, - parent_data = 0x0 <__vectors_start>, - chip_data = 0xc0eb0854 - } <... trimmed ...> List of commands and functions ------------------------------ diff --git a/Documentation/gpio/drivers-on-gpio.txt b/Documentation/gpio/drivers-on-gpio.txt index 14bf95a13bae..306513251713 100644 --- a/Documentation/gpio/drivers-on-gpio.txt +++ b/Documentation/gpio/drivers-on-gpio.txt @@ -37,15 +37,16 @@ hardware descriptions such as device tree or ACPI: external connector status, such as a headset line for an audio driver or an HDMI connector. It will provide a better userspace sysfs interface than GPIO. -- restart-gpio: drivers/power/gpio-restart.c is used to restart/reboot the - system by pulling a GPIO line and will register a restart handler so +- restart-gpio: drivers/power/reset/gpio-restart.c is used to restart/reboot + the system by pulling a GPIO line and will register a restart handler so userspace can issue the right system call to restart the system. -- poweroff-gpio: drivers/power/gpio-poweroff.c is used to power the system down - by pulling a GPIO line and will register a pm_power_off() callback so that - userspace can issue the right system call to power down the system. +- poweroff-gpio: drivers/power/reset/gpio-poweroff.c is used to power the + system down by pulling a GPIO line and will register a pm_power_off() + callback so that userspace can issue the right system call to power down the + system. -- gpio-gate-clock: drivers/clk/clk-gpio-gate.c is used to control a gated clock +- gpio-gate-clock: drivers/clk/clk-gpio.c is used to control a gated clock (off/on) that uses a GPIO, and integrated with the clock subsystem. - i2c-gpio: drivers/i2c/busses/i2c-gpio.c is used to drive an I2C bus diff --git a/Documentation/hwmon/abituguru b/Documentation/hwmon/abituguru index f1d4fe4c366c..44013d23b3f0 100644 --- a/Documentation/hwmon/abituguru +++ b/Documentation/hwmon/abituguru @@ -24,7 +24,7 @@ Supported chips: AW9D-MAX) (2) 1) For revisions 2 and 3 uGuru's the driver can autodetect the sensortype (Volt or Temp) for bank1 sensors, for revision 1 uGuru's - this doesnot always work. For these uGuru's the autodection can + this does not always work. For these uGuru's the autodetection can be overridden with the bank1_types module param. For all 3 known revison 1 motherboards the correct use of this param is: bank1_types=1,1,0,0,0,0,0,2,0,0,0,0,2,0,0,1 diff --git a/Documentation/hwmon/ftsteutates b/Documentation/hwmon/ftsteutates new file mode 100644 index 000000000000..2a1bf69c6a26 --- /dev/null +++ b/Documentation/hwmon/ftsteutates @@ -0,0 +1,23 @@ +Kernel driver ftsteutates +===================== + +Supported chips: + * FTS Teutates + Prefix: 'ftsteutates' + Addresses scanned: I2C 0x73 (7-Bit) + +Author: Thilo Cestonaro + + +Description +----------- +The BMC Teutates is the Eleventh generation of Superior System +monitoring and thermal management solution. It is builds on the basic +functionality of the BMC Theseus and contains several new features and +enhancements. It can monitor up to 4 voltages, 16 temperatures and +8 fans. It also contains an integrated watchdog which is currently +implemented in this driver. + +Specification of the chip can be found here: +ftp:///pub/Mainboard-OEM-Sales/Services/Software&Tools/Linux_SystemMonitoring&Watchdog&GPIO/BMC-Teutates_Specification_V1.21.pdf +ftp:///pub/Mainboard-OEM-Sales/Services/Software&Tools/Linux_SystemMonitoring&Watchdog&GPIO/Fujitsu_mainboards-1-Sensors_HowTo-en-US.pdf diff --git a/Documentation/hwmon/ina3221 b/Documentation/hwmon/ina3221 new file mode 100644 index 000000000000..0ff74854cb2e --- /dev/null +++ b/Documentation/hwmon/ina3221 @@ -0,0 +1,35 @@ +Kernel driver ina3221 +===================== + +Supported chips: + * Texas Instruments INA3221 + Prefix: 'ina3221' + Addresses: I2C 0x40 - 0x43 + Datasheet: Publicly available at the Texas Instruments website + http://www.ti.com/ + +Author: Andrew F. Davis + +Description +----------- + +The Texas Instruments INA3221 monitors voltage, current, and power on the high +side of up to three D.C. power supplies. The INA3221 monitors both shunt drop +and supply voltage, with programmable conversion times and averaging, current +and power are calculated host-side from these. + +Sysfs entries +------------- + +in[123]_input Bus voltage(mV) channels +curr[123]_input Current(mA) measurement channels +shunt[123]_resistor Shunt resistance(uOhm) channels +curr[123]_crit Critical alert current(mA) setting, activates the + corresponding alarm when the respective current + is above this value +curr[123]_crit_alarm Critical alert current limit exceeded +curr[123]_max Warning alert current(mA) setting, activates the + corresponding alarm when the respective current + average is above this value. +curr[123]_max_alarm Warning alert current limit exceeded +in[456]_input Shunt voltage(uV) for channels 1, 2, and 3 respectively diff --git a/Documentation/hwmon/jc42 b/Documentation/hwmon/jc42 index f7f1830a2566..b4b671f22453 100644 --- a/Documentation/hwmon/jc42 +++ b/Documentation/hwmon/jc42 @@ -18,10 +18,11 @@ Supported chips: * Maxim MAX6604 Datasheets: http://datasheets.maxim-ic.com/en/ds/MAX6604.pdf - * Microchip MCP9804, MCP9805, MCP98242, MCP98243, MCP98244, MCP9843 + * Microchip MCP9804, MCP9805, MCP9808, MCP98242, MCP98243, MCP98244, MCP9843 Datasheets: http://ww1.microchip.com/downloads/en/DeviceDoc/22203C.pdf http://ww1.microchip.com/downloads/en/DeviceDoc/21977b.pdf + http://ww1.microchip.com/downloads/en/DeviceDoc/25095A.pdf http://ww1.microchip.com/downloads/en/DeviceDoc/21996a.pdf http://ww1.microchip.com/downloads/en/DeviceDoc/22153c.pdf http://ww1.microchip.com/downloads/en/DeviceDoc/22327A.pdf diff --git a/Documentation/hwmon/max1668 b/Documentation/hwmon/max1668 index 0616ed9758df..8f9d570dbfec 100644 --- a/Documentation/hwmon/max1668 +++ b/Documentation/hwmon/max1668 @@ -17,7 +17,7 @@ This driver implements support for the Maxim MAX1668, MAX1805 and MAX1989 chips. The three devices are very similar, but the MAX1805 has a reduced feature -set; only two remote temperature inputs vs the four avaible on the other +set; only two remote temperature inputs vs the four available on the other two ICs. The driver is able to distinguish between the devices and creates sysfs diff --git a/Documentation/hwmon/sht3x b/Documentation/hwmon/sht3x new file mode 100644 index 000000000000..b0d88184f48e --- /dev/null +++ b/Documentation/hwmon/sht3x @@ -0,0 +1,76 @@ +Kernel driver sht3x +=================== + +Supported chips: + * Sensirion SHT3x-DIS + Prefix: 'sht3x' + Addresses scanned: none + Datasheet: http://www.sensirion.com/fileadmin/user_upload/customers/sensirion/Dokumente/Humidity/Sensirion_Humidity_Datasheet_SHT3x_DIS.pdf + +Author: + David Frey + Pascal Sachs + +Description +----------- + +This driver implements support for the Sensirion SHT3x-DIS chip, a humidity +and temperature sensor. Temperature is measured in degrees celsius, relative +humidity is expressed as a percentage. In the sysfs interface, all values are +scaled by 1000, i.e. the value for 31.5 degrees celsius is 31500. + +The device communicates with the I2C protocol. Sensors can have the I2C +addresses 0x44 or 0x45, depending on the wiring. See +Documentation/i2c/instantiating-devices for methods to instantiate the device. + +There are two options configurable by means of sht3x_platform_data: +1. blocking (pull the I2C clock line down while performing the measurement) or + non-blocking mode. Blocking mode will guarantee the fastest result but + the I2C bus will be busy during that time. By default, non-blocking mode + is used. Make sure clock-stretching works properly on your device if you + want to use blocking mode. +2. high or low accuracy. High accuracy is used by default and using it is + strongly recommended. + +The sht3x sensor supports a single shot mode as well as 5 periodic measure +modes, which can be controlled with the update_interval sysfs interface. +The allowed update_interval in milliseconds are as follows: + * 0 single shot mode + * 2000 0.5 Hz periodic measurement + * 1000 1 Hz periodic measurement + * 500 2 Hz periodic measurement + * 250 4 Hz periodic measurement + * 100 10 Hz periodic measurement + +In the periodic measure mode, the sensor automatically triggers a measurement +with the configured update interval on the chip. When a temperature or humidity +reading exceeds the configured limits, the alert attribute is set to 1 and +the alert pin on the sensor is set to high. +When the temperature and humidity readings move back between the hysteresis +values, the alert bit is set to 0 and the alert pin on the sensor is set to +low. + +sysfs-Interface +--------------- + +temp1_input: temperature input +humidity1_input: humidity input +temp1_max: temperature max value +temp1_max_hyst: temperature hysteresis value for max limit +humidity1_max: humidity max value +humidity1_max_hyst: humidity hysteresis value for max limit +temp1_min: temperature min value +temp1_min_hyst: temperature hysteresis value for min limit +humidity1_min: humidity min value +humidity1_min_hyst: humidity hysteresis value for min limit +temp1_alarm: alarm flag is set to 1 if the temperature is outside the + configured limits. Alarm only works in periodic measure mode +humidity1_alarm: alarm flag is set to 1 if the humidity is outside the + configured limits. Alarm only works in periodic measure mode +heater_enable: heater enable, heating element removes excess humidity from + sensor + 0: turned off + 1: turned on +update_interval: update interval, 0 for single shot, interval in msec + for periodic measurement. If the interval is not supported + by the sensor, the next faster interval is chosen diff --git a/Documentation/hwmon/submitting-patches b/Documentation/hwmon/submitting-patches index d201828d202f..57f60307accc 100644 --- a/Documentation/hwmon/submitting-patches +++ b/Documentation/hwmon/submitting-patches @@ -15,10 +15,15 @@ increase the chances of your change being accepted. Documentation/SubmittingPatches Documentation/CodingStyle -* If your patch generates checkpatch warnings, please refrain from explanations - such as "I don't like that coding style". Keep in mind that each unnecessary - warning helps hiding a real problem. If you don't like the kernel coding - style, don't write kernel drivers. +* Please run your patch through 'checkpatch --strict'. There should be no + errors, no warnings, and few if any check messages. If there are any + messages, please be prepared to explain. + +* If your patch generates checkpatch errors, warnings, or check messages, + please refrain from explanations such as "I prefer that coding style". + Keep in mind that each unnecessary message helps hiding a real problem, + and a consistent coding style makes it easier for others to understand + and review the code. * Please test your patch thoroughly. We are not your test group. Sometimes a patch can not or not completely be tested because of missing @@ -61,15 +66,30 @@ increase the chances of your change being accepted. * Make sure that all dependencies are listed in Kconfig. +* Please list include files in alphabetic order. + +* Please align continuation lines with '(' on the previous line. + * Avoid forward declarations if you can. Rearrange the code if necessary. +* Avoid macros to generate groups of sensor attributes. It not only confuses + checkpatch, but also makes it more difficult to review the code. + * Avoid calculations in macros and macro-generated functions. While such macros may save a line or so in the source, it obfuscates the code and makes code review more difficult. It may also result in code which is more complicated than necessary. Use inline functions or just regular functions instead. +* Limit the number of kernel log messages. In general, your driver should not + generate an error message just because a runtime operation failed. Report + errors to user space instead, using an appropriate error code. Keep in mind + that kernel error log messages not only fill up the kernel log, but also are + printed synchronously, most likely with interrupt disabled, often to a serial + console. Excessive logging can seriously affect system performance. + * Use devres functions whenever possible to allocate resources. For rationale and supported functions, please see Documentation/driver-model/devres.txt. + If a function is not supported by devres, consider using devm_add_action(). * If the driver has a detect function, make sure it is silent. Debug messages and messages printed after a successful detection are acceptable, but it @@ -96,8 +116,16 @@ increase the chances of your change being accepted. writing to it might cause a bad misconfiguration. * Make sure there are no race conditions in the probe function. Specifically, - completely initialize your chip first, then create sysfs entries and register - with the hwmon subsystem. + completely initialize your chip and your driver first, then register with + the hwmon subsystem. + +* Use devm_hwmon_device_register_with_groups() or, if your driver needs a remove + function, hwmon_device_register_with_groups() to register your driver with the + hwmon subsystem. Try using devm_add_action() instead of a remove function if + possible. Do not use hwmon_device_register(). + +* Your driver should be buildable as module. If not, please be prepared to + explain why it has to be built into the kernel. * Do not provide support for deprecated sysfs attributes. diff --git a/Documentation/hwmon/tmp401 b/Documentation/hwmon/tmp401 index 711f75e189eb..2d9ca42213cf 100644 --- a/Documentation/hwmon/tmp401 +++ b/Documentation/hwmon/tmp401 @@ -22,6 +22,9 @@ Supported chips: Prefix: 'tmp435' Addresses scanned: I2C 0x48 - 0x4f Datasheet: http://focus.ti.com/docs/prod/folders/print/tmp435.html + * Texas Instruments TMP461 + Prefix: 'tmp461' + Datasheet: http://www.ti.com/product/tmp461 Authors: Hans de Goede @@ -31,8 +34,8 @@ Description ----------- This driver implements support for Texas Instruments TMP401, TMP411, -TMP431, TMP432 and TMP435 chips. These chips implement one or two remote -and one local temperature sensors. Temperature is measured in degrees +TMP431, TMP432, TMP435, and TMP461 chips. These chips implement one or two +remote and one local temperature sensors. Temperature is measured in degrees Celsius. Resolution of the remote sensor is 0.0625 degree. Local sensor resolution can be set to 0.5, 0.25, 0.125 or 0.0625 degree (not supported by the driver so far, so using the default resolution of 0.5 @@ -55,3 +58,10 @@ some additional features. TMP432 is compatible with TMP401 and TMP431. It supports two external temperature sensors. + +TMP461 is compatible with TMP401. It supports offset correction +that is applied to the remote sensor. + +* Sensor offset values are temperature values + + Exported via sysfs attribute tempX_offset diff --git a/Documentation/index.rst b/Documentation/index.rst new file mode 100644 index 000000000000..f92854f01773 --- /dev/null +++ b/Documentation/index.rst @@ -0,0 +1,22 @@ +.. The Linux Kernel documentation master file, created by + sphinx-quickstart on Fri Feb 12 13:51:46 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to The Linux Kernel's documentation! +============================================ + +Nothing for you to see here *yet*. Please move along. + +Contents: + +.. toctree:: + :maxdepth: 2 + + kernel-documentation + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/Documentation/ioctl/cdrom.txt b/Documentation/ioctl/cdrom.txt index 59df81c8da2b..a4d62a9d6771 100644 --- a/Documentation/ioctl/cdrom.txt +++ b/Documentation/ioctl/cdrom.txt @@ -340,7 +340,8 @@ CDROMSUBCHNL Read subchannel data (struct cdrom_subchnl) EINVAL format not CDROM_MSF or CDROM_LBA notes: - Format is converted to CDROM_MSF on return + Format is converted to CDROM_MSF or CDROM_LBA + as per user request on return diff --git a/Documentation/kbuild/makefiles.txt b/Documentation/kbuild/makefiles.txt index 13f888a02a3d..385a5ef41c17 100644 --- a/Documentation/kbuild/makefiles.txt +++ b/Documentation/kbuild/makefiles.txt @@ -47,6 +47,7 @@ This document describes the Linux kernel Makefiles. --- 7.2 genhdr-y --- 7.3 destination-y --- 7.4 generic-y + --- 7.5 generated-y === 8 Kbuild Variables === 9 Makefile language @@ -1319,6 +1320,19 @@ See subsequent chapter for the syntax of the Kbuild file. Example: termios.h #include + --- 7.5 generated-y + + If an architecture generates other header files alongside generic-y + wrappers, and not included in genhdr-y, then generated-y specifies + them. + + This prevents them being treated as stale asm-generic wrappers and + removed. + + Example: + #arch/x86/include/asm/Kbuild + generated-y += syscalls_32.h + === 8 Kbuild Variables The top Makefile exports the following variables: diff --git a/Documentation/kernel-doc-nano-HOWTO.txt b/Documentation/kernel-doc-nano-HOWTO.txt index 78f69cdc9b3f..062e3af271b7 100644 --- a/Documentation/kernel-doc-nano-HOWTO.txt +++ b/Documentation/kernel-doc-nano-HOWTO.txt @@ -1,3 +1,6 @@ +NOTE: this document is outdated and will eventually be removed. See +Documentation/kernel-documentation.rst for current information. + kernel-doc nano-HOWTO ===================== diff --git a/Documentation/kernel-documentation.rst b/Documentation/kernel-documentation.rst new file mode 100644 index 000000000000..c4eb5049da39 --- /dev/null +++ b/Documentation/kernel-documentation.rst @@ -0,0 +1,654 @@ +========================== +Linux Kernel Documentation +========================== + +Introduction +============ + +The Linux kernel uses `Sphinx`_ to generate pretty documentation from +`reStructuredText`_ files under ``Documentation``. To build the documentation in +HTML or PDF formats, use ``make htmldocs`` or ``make pdfdocs``. The generated +documentation is placed in ``Documentation/output``. + +.. _Sphinx: http://www.sphinx-doc.org/ +.. _reStructuredText: http://docutils.sourceforge.net/rst.html + +The reStructuredText files may contain directives to include structured +documentation comments, or kernel-doc comments, from source files. Usually these +are used to describe the functions and types and design of the code. The +kernel-doc comments have some special structure and formatting, but beyond that +they are also treated as reStructuredText. + +There is also the deprecated DocBook toolchain to generate documentation from +DocBook XML template files under ``Documentation/DocBook``. The DocBook files +are to be converted to reStructuredText, and the toolchain is slated to be +removed. + +Finally, there are thousands of plain text documentation files scattered around +``Documentation``. Some of these will likely be converted to reStructuredText +over time, but the bulk of them will remain in plain text. + +Sphinx Build +============ + +The usual way to generate the documentation is to run ``make htmldocs`` or +``make pdfdocs``. There are also other formats available, see the documentation +section of ``make help``. The generated documentation is placed in +format-specific subdirectories under ``Documentation/output``. + +To generate documentation, Sphinx (``sphinx-build``) must obviously be +installed. For prettier HTML output, the Read the Docs Sphinx theme +(``sphinx_rtd_theme``) is used if available. For PDF output, ``rst2pdf`` is also +needed. All of these are widely available and packaged in distributions. + +To pass extra options to Sphinx, you can use the ``SPHINXOPTS`` make +variable. For example, use ``make SPHINXOPTS=-v htmldocs`` to get more verbose +output. + +To remove the generated documentation, run ``make cleandocs``. + +Writing Documentation +===================== + +Adding new documentation can be as simple as: + +1. Add a new ``.rst`` file somewhere under ``Documentation``. +2. Refer to it from the Sphinx main `TOC tree`_ in ``Documentation/index.rst``. + +.. _TOC tree: http://www.sphinx-doc.org/en/stable/markup/toctree.html + +This is usually good enough for simple documentation (like the one you're +reading right now), but for larger documents it may be advisable to create a +subdirectory (or use an existing one). For example, the graphics subsystem +documentation is under ``Documentation/gpu``, split to several ``.rst`` files, +and has a separate ``index.rst`` (with a ``toctree`` of its own) referenced from +the main index. + +See the documentation for `Sphinx`_ and `reStructuredText`_ on what you can do +with them. In particular, the Sphinx `reStructuredText Primer`_ is a good place +to get started with reStructuredText. There are also some `Sphinx specific +markup constructs`_. + +.. _reStructuredText Primer: http://www.sphinx-doc.org/en/stable/rest.html +.. _Sphinx specific markup constructs: http://www.sphinx-doc.org/en/stable/markup/index.html + +Specific guidelines for the kernel documentation +------------------------------------------------ + +Here are some specific guidelines for the kernel documentation: + +* Please don't go overboard with reStructuredText markup. Keep it simple. + +* Please stick to this order of heading adornments: + + 1. ``=`` with overline for document title:: + + ============== + Document title + ============== + + 2. ``=`` for chapters:: + + Chapters + ======== + + 3. ``-`` for sections:: + + Section + ------- + + 4. ``~`` for subsections:: + + Subsection + ~~~~~~~~~~ + + Although RST doesn't mandate a specific order ("Rather than imposing a fixed + number and order of section title adornment styles, the order enforced will be + the order as encountered."), having the higher levels the same overall makes + it easier to follow the documents. + +list tables +----------- + +We recommend the use of *list table* formats. The *list table* formats are +double-stage lists. Compared to the ASCII-art they might not be as +comfortable for +readers of the text files. Their advantage is that they are easy to +create or modify and that the diff of a modification is much more meaningful, +because it is limited to the modified content. + +The ``flat-table`` is a double-stage list similar to the ``list-table`` with +some additional features: + +* column-span: with the role ``cspan`` a cell can be extended through + additional columns + +* row-span: with the role ``rspan`` a cell can be extended through + additional rows + +* auto span rightmost cell of a table row over the missing cells on the right + side of that table-row. With Option ``:fill-cells:`` this behavior can + changed from *auto span* to *auto fill*, which automatically inserts (empty) + cells instead of spanning the last cell. + +options: + +* ``:header-rows:`` [int] count of header rows +* ``:stub-columns:`` [int] count of stub columns +* ``:widths:`` [[int] [int] ... ] widths of columns +* ``:fill-cells:`` instead of auto-spanning missing cells, insert missing cells + +roles: + +* ``:cspan:`` [int] additional columns (*morecols*) +* ``:rspan:`` [int] additional rows (*morerows*) + +The example below shows how to use this markup. The first level of the staged +list is the *table-row*. In the *table-row* there is only one markup allowed, +the list of the cells in this *table-row*. Exceptions are *comments* ( ``..`` ) +and *targets* (e.g. a ref to ``:ref:`last row ``` / :ref:`last row +`). + +.. code-block:: rst + + .. flat-table:: table title + :widths: 2 1 1 3 + + * - head col 1 + - head col 2 + - head col 3 + - head col 4 + + * - column 1 + - field 1.1 + - field 1.2 with autospan + + * - column 2 + - field 2.1 + - :rspan:`1` :cspan:`1` field 2.2 - 3.3 + + * .. _`last row`: + + - column 3 + +Rendered as: + + .. flat-table:: table title + :widths: 2 1 1 3 + + * - head col 1 + - head col 2 + - head col 3 + - head col 4 + + * - column 1 + - field 1.1 + - field 1.2 with autospan + + * - column 2 + - field 2.1 + - :rspan:`1` :cspan:`1` field 2.2 - 3.3 + + * .. _`last row`: + + - column 3 + + +Including kernel-doc comments +============================= + +The Linux kernel source files may contain structured documentation comments, or +kernel-doc comments to describe the functions and types and design of the +code. The documentation comments may be included to any of the reStructuredText +documents using a dedicated kernel-doc Sphinx directive extension. + +The kernel-doc directive is of the format:: + + .. kernel-doc:: source + :option: + +The *source* is the path to a source file, relative to the kernel source +tree. The following directive options are supported: + +export: *[source-pattern ...]* + Include documentation for all functions in *source* that have been exported + using ``EXPORT_SYMBOL`` or ``EXPORT_SYMBOL_GPL`` either in *source* or in any + of the files specified by *source-pattern*. + + The *source-pattern* is useful when the kernel-doc comments have been placed + in header files, while ``EXPORT_SYMBOL`` and ``EXPORT_SYMBOL_GPL`` are next to + the function definitions. + + Examples:: + + .. kernel-doc:: lib/bitmap.c + :export: + + .. kernel-doc:: include/net/mac80211.h + :export: net/mac80211/*.c + +internal: *[source-pattern ...]* + Include documentation for all functions and types in *source* that have + **not** been exported using ``EXPORT_SYMBOL`` or ``EXPORT_SYMBOL_GPL`` either + in *source* or in any of the files specified by *source-pattern*. + + Example:: + + .. kernel-doc:: drivers/gpu/drm/i915/intel_audio.c + :internal: + +doc: *title* + Include documentation for the ``DOC:`` paragraph identified by *title* in + *source*. Spaces are allowed in *title*; do not quote the *title*. The *title* + is only used as an identifier for the paragraph, and is not included in the + output. Please make sure to have an appropriate heading in the enclosing + reStructuredText document. + + Example:: + + .. kernel-doc:: drivers/gpu/drm/i915/intel_audio.c + :doc: High Definition Audio over HDMI and Display Port + +functions: *function* *[...]* + Include documentation for each *function* in *source*. + + Example:: + + .. kernel-doc:: lib/bitmap.c + :functions: bitmap_parselist bitmap_parselist_user + +Without options, the kernel-doc directive includes all documentation comments +from the source file. + +The kernel-doc extension is included in the kernel source tree, at +``Documentation/sphinx/kernel-doc.py``. Internally, it uses the +``scripts/kernel-doc`` script to extract the documentation comments from the +source. + +Writing kernel-doc comments +=========================== + +In order to provide embedded, "C" friendly, easy to maintain, but consistent and +extractable overview, function and type documentation, the Linux kernel has +adopted a consistent style for documentation comments. The format for this +documentation is called the kernel-doc format, described below. This style +embeds the documentation within the source files, using a few simple conventions +for adding documentation paragraphs and documenting functions and their +parameters, structures and unions and their members, enumerations, and typedefs. + +.. note:: The kernel-doc format is deceptively similar to gtk-doc or Doxygen, + yet distinctively different, for historical reasons. The kernel source + contains tens of thousands of kernel-doc comments. Please stick to the style + described here. + +The ``scripts/kernel-doc`` script is used by the Sphinx kernel-doc extension in +the documentation build to extract this embedded documentation into the various +HTML, PDF, and other format documents. + +In order to provide good documentation of kernel functions and data structures, +please use the following conventions to format your kernel-doc comments in the +Linux kernel source. + +How to format kernel-doc comments +--------------------------------- + +The opening comment mark ``/**`` is reserved for kernel-doc comments. Only +comments so marked will be considered by the ``kernel-doc`` tool. Use it only +for comment blocks that contain kernel-doc formatted comments. The usual ``*/`` +should be used as the closing comment marker. The lines in between should be +prefixed by `` * `` (space star space). + +The function and type kernel-doc comments should be placed just before the +function or type being described. The overview kernel-doc comments may be freely +placed at the top indentation level. + +Example kernel-doc function comment:: + + /** + * foobar() - Brief description of foobar. + * @arg: Description of argument of foobar. + * + * Longer description of foobar. + * + * Return: Description of return value of foobar. + */ + int foobar(int arg) + +The format is similar for documentation for structures, enums, paragraphs, +etc. See the sections below for details. + +The kernel-doc structure is extracted from the comments, and proper `Sphinx C +Domain`_ function and type descriptions with anchors are generated for them. The +descriptions are filtered for special kernel-doc highlights and +cross-references. See below for details. + +.. _Sphinx C Domain: http://www.sphinx-doc.org/en/stable/domains.html + +Highlights and cross-references +------------------------------- + +The following special patterns are recognized in the kernel-doc comment +descriptive text and converted to proper reStructuredText markup and `Sphinx C +Domain`_ references. + +.. attention:: The below are **only** recognized within kernel-doc comments, + **not** within normal reStructuredText documents. + +``funcname()`` + Function reference. + +``@parameter`` + Name of a function parameter. (No cross-referencing, just formatting.) + +``%CONST`` + Name of a constant. (No cross-referencing, just formatting.) + +``$ENVVAR`` + Name of an environment variable. (No cross-referencing, just formatting.) + +``&struct name`` + Structure reference. + +``&enum name`` + Enum reference. + +``&typedef name`` + Typedef reference. + +``&struct_name->member`` or ``&struct_name.member`` + Structure or union member reference. The cross-reference will be to the struct + or union definition, not the member directly. + +``&name`` + A generic type reference. Prefer using the full reference described above + instead. This is mostly for legacy comments. + +Cross-referencing from reStructuredText +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. highlight:: none + +To cross-reference the functions and types defined in the kernel-doc comments +from reStructuredText documents, please use the `Sphinx C Domain`_ +references. For example:: + + See function :c:func:`foo` and struct/union/enum/typedef :c:type:`bar`. + +While the type reference works with just the type name, without the +struct/union/enum/typedef part in front, you may want to use:: + + See :c:type:`struct foo `. + See :c:type:`union bar `. + See :c:type:`enum baz `. + See :c:type:`typedef meh `. + +This will produce prettier links, and is in line with how kernel-doc does the +cross-references. + +For further details, please refer to the `Sphinx C Domain`_ documentation. + +Function documentation +---------------------- + +.. highlight:: c + +The general format of a function and function-like macro kernel-doc comment is:: + + /** + * function_name() - Brief description of function. + * @arg1: Describe the first argument. + * @arg2: Describe the second argument. + * One can provide multiple line descriptions + * for arguments. + * + * A longer description, with more discussion of the function function_name() + * that might be useful to those using or modifying it. Begins with an + * empty comment line, and may include additional embedded empty + * comment lines. + * + * The longer description may have multiple paragraphs. + * + * Return: Describe the return value of foobar. + * + * The return value description can also have multiple paragraphs, and should + * be placed at the end of the comment block. + */ + +The brief description following the function name may span multiple lines, and +ends with an ``@argument:`` description, a blank comment line, or the end of the +comment block. + +The kernel-doc function comments describe each parameter to the function, in +order, with the ``@argument:`` descriptions. The ``@argument:`` descriptions +must begin on the very next line following the opening brief function +description line, with no intervening blank comment lines. The ``@argument:`` +descriptions may span multiple lines. The continuation lines may contain +indentation. If a function parameter is ``...`` (varargs), it should be listed +in kernel-doc notation as: ``@...:``. + +The return value, if any, should be described in a dedicated section at the end +of the comment starting with "Return:". + +Structure, union, and enumeration documentation +----------------------------------------------- + +The general format of a struct, union, and enum kernel-doc comment is:: + + /** + * struct struct_name - Brief description. + * @member_name: Description of member member_name. + * + * Description of the structure. + */ + +Below, "struct" is used to mean structs, unions and enums, and "member" is used +to mean struct and union members as well as enumerations in an enum. + +The brief description following the structure name may span multiple lines, and +ends with a ``@member:`` description, a blank comment line, or the end of the +comment block. + +The kernel-doc data structure comments describe each member of the structure, in +order, with the ``@member:`` descriptions. The ``@member:`` descriptions must +begin on the very next line following the opening brief function description +line, with no intervening blank comment lines. The ``@member:`` descriptions may +span multiple lines. The continuation lines may contain indentation. + +In-line member documentation comments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The structure members may also be documented in-line within the definition:: + + /** + * struct foo - Brief description. + * @foo: The Foo member. + */ + struct foo { + int foo; + /** + * @bar: The Bar member. + */ + int bar; + /** + * @baz: The Baz member. + * + * Here, the member description may contain several paragraphs. + */ + int baz; + } + +Private members +~~~~~~~~~~~~~~~ + +Inside a struct description, you can use the "private:" and "public:" comment +tags. Structure fields that are inside a "private:" area are not listed in the +generated output documentation. The "private:" and "public:" tags must begin +immediately following a ``/*`` comment marker. They may optionally include +comments between the ``:`` and the ending ``*/`` marker. + +Example:: + + /** + * struct my_struct - short description + * @a: first member + * @b: second member + * + * Longer description + */ + struct my_struct { + int a; + int b; + /* private: internal use only */ + int c; + }; + + +Typedef documentation +--------------------- + +The general format of a typedef kernel-doc comment is:: + + /** + * typedef type_name - Brief description. + * + * Description of the type. + */ + +Overview documentation comments +------------------------------- + +To facilitate having source code and comments close together, you can include +kernel-doc documentation blocks that are free-form comments instead of being +kernel-doc for functions, structures, unions, enums, or typedefs. This could be +used for something like a theory of operation for a driver or library code, for +example. + +This is done by using a ``DOC:`` section keyword with a section title. + +The general format of an overview or high-level documentation comment is:: + + /** + * DOC: Theory of Operation + * + * The whizbang foobar is a dilly of a gizmo. It can do whatever you + * want it to do, at any time. It reads your mind. Here's how it works. + * + * foo bar splat + * + * The only drawback to this gizmo is that is can sometimes damage + * hardware, software, or its subject(s). + */ + +The title following ``DOC:`` acts as a heading within the source file, but also +as an identifier for extracting the documentation comment. Thus, the title must +be unique within the file. + +Recommendations +--------------- + +We definitely need kernel-doc formatted documentation for functions that are +exported to loadable modules using ``EXPORT_SYMBOL`` or ``EXPORT_SYMBOL_GPL``. + +We also look to provide kernel-doc formatted documentation for functions +externally visible to other kernel files (not marked "static"). + +We also recommend providing kernel-doc formatted documentation for private (file +"static") routines, for consistency of kernel source code layout. But this is +lower priority and at the discretion of the MAINTAINER of that kernel source +file. + +Data structures visible in kernel include files should also be documented using +kernel-doc formatted comments. + +DocBook XML [DEPRECATED] +======================== + +.. attention:: + + This section describes the deprecated DocBook XML toolchain. Please do not + create new DocBook XML template files. Please consider converting existing + DocBook XML templates files to Sphinx/reStructuredText. + +Converting DocBook to Sphinx +---------------------------- + +.. highlight:: none + +Over time, we expect all of the documents under ``Documentation/DocBook`` to be +converted to Sphinx and reStructuredText. For most DocBook XML documents, a good +enough solution is to use the simple ``Documentation/sphinx/tmplcvt`` script, +which uses ``pandoc`` under the hood. For example:: + + $ cd Documentation/sphinx + $ ./tmplcvt ../DocBook/in.tmpl ../out.rst + +Then edit the resulting rst files to fix any remaining issues, and add the +document in the ``toctree`` in ``Documentation/index.rst``. + +Components of the kernel-doc system +----------------------------------- + +Many places in the source tree have extractable documentation in the form of +block comments above functions. The components of this system are: + +- ``scripts/kernel-doc`` + + This is a perl script that hunts for the block comments and can mark them up + directly into reStructuredText, DocBook, man, text, and HTML. (No, not + texinfo.) + +- ``Documentation/DocBook/*.tmpl`` + + These are XML template files, which are normal XML files with special + place-holders for where the extracted documentation should go. + +- ``scripts/docproc.c`` + + This is a program for converting XML template files into XML files. When a + file is referenced it is searched for symbols exported (EXPORT_SYMBOL), to be + able to distinguish between internal and external functions. + + It invokes kernel-doc, giving it the list of functions that are to be + documented. + + Additionally it is used to scan the XML template files to locate all the files + referenced herein. This is used to generate dependency information as used by + make. + +- ``Makefile`` + + The targets 'xmldocs', 'psdocs', 'pdfdocs', and 'htmldocs' are used to build + DocBook XML files, PostScript files, PDF files, and html files in + Documentation/DocBook. The older target 'sgmldocs' is equivalent to 'xmldocs'. + +- ``Documentation/DocBook/Makefile`` + + This is where C files are associated with SGML templates. + +How to use kernel-doc comments in DocBook XML template files +------------------------------------------------------------ + +DocBook XML template files (\*.tmpl) are like normal XML files, except that they +can contain escape sequences where extracted documentation should be inserted. + +``!E`` is replaced by the documentation, in ````, for +functions that are exported using ``EXPORT_SYMBOL``: the function list is +collected from files listed in ``Documentation/DocBook/Makefile``. + +``!I`` is replaced by the documentation for functions that are **not** +exported using ``EXPORT_SYMBOL``. + +``!D`` is used to name additional files to search for functions +exported using ``EXPORT_SYMBOL``. + +``!F `` is replaced by the documentation, in +````, for the functions listed. + +``!P

`` is replaced by the contents of the ``DOC:`` +section titled ``
`` from ````. Spaces are allowed in +``
``; do not quote the ``
``. + +``!C`` is replaced by nothing, but makes the tools check that all DOC: +sections and documented functions, symbols, etc. are used. This makes sense to +use when you use ``!F`` or ``!P`` only and want to verify that all documentation +is included. diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 82b42c958d1c..769db8399ac8 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -582,6 +582,9 @@ bytes respectively. Such letter suffixes can also be entirely omitted. bootmem_debug [KNL] Enable bootmem allocator debug messages. + bert_disable [ACPI] + Disable BERT OS support on buggy BIOSes. + bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) bttv.radio= Most important insmod options are available as kernel args too. @@ -687,6 +690,14 @@ bytes respectively. Such letter suffixes can also be entirely omitted. [SPARC64] tick [X86-64] hpet,tsc + clocksource.arm_arch_timer.evtstrm= + [ARM,ARM64] + Format: + Enable/disable the eventstream feature of the ARM + architected timer so that code using WFE-based polling + loops can be debugged more effectively on production + systems. + clearcpuid=BITNUM [X86] Disable CPUID feature X for the kernel. See arch/x86/include/asm/cpufeatures.h for the valid bit @@ -1185,6 +1196,13 @@ bytes respectively. Such letter suffixes can also be entirely omitted. Address Range Mirroring feature even if your box doesn't support it. + efivar_ssdt= [EFI; X86] Name of an EFI variable that contains an SSDT + that is to be dynamically loaded by Linux. If there are + multiple variables with the same name but with different + vendor GUIDs, all of them will be loaded. See + Documentation/acpi/ssdt-overlays.txt for details. + + eisa_irq_edge= [PARISC,HW] See header of drivers/parisc/eisa.c. @@ -1803,12 +1821,10 @@ bytes respectively. Such letter suffixes can also be entirely omitted. js= [HW,JOY] Analog joystick See Documentation/input/joystick.txt. - kaslr/nokaslr [X86] - Enable/disable kernel and module base offset ASLR - (Address Space Layout Randomization) if built into - the kernel. When CONFIG_HIBERNATION is selected, - kASLR is disabled by default. When kASLR is enabled, - hibernation will be disabled. + nokaslr [KNL] + When CONFIG_RANDOMIZE_BASE is set, this disables + kernel and module base offset ASLR (Address Space + Layout Randomization). keepinitrd [HW,ARM] @@ -2788,8 +2804,6 @@ bytes respectively. Such letter suffixes can also be entirely omitted. timer: [X86] Force use of architectural NMI timer mode (see also oprofile.timer for generic hr timer mode) - [s390] Force legacy basic mode sampling - (report cpu_type "timer") oops=panic Always panic on oopses. Default is to just kill the process, but there is a small probability of @@ -3594,6 +3608,9 @@ bytes respectively. Such letter suffixes can also be entirely omitted. present during boot. nocompress Don't compress/decompress hibernation images. no Disable hibernation and resume. + protect_image Turn on image protection during restoration + (that will set all pages holding image data + during restoration read-only). retain_initrd [RAM] Keep initrd memory after extraction @@ -3992,8 +4009,9 @@ bytes respectively. Such letter suffixes can also be entirely omitted. trace_event=[event-list] [FTRACE] Set and start specified trace events in order - to facilitate early boot debugging. - See also Documentation/trace/events.txt + to facilitate early boot debugging. The event-list is a + comma separated list of trace events to enable. See + also Documentation/trace/events.txt trace_options=[option-list] [FTRACE] Enable or disable tracer options at boot. diff --git a/Documentation/md.txt b/Documentation/md.txt index 1a2ada46aaed..d6e2fcf27337 100644 --- a/Documentation/md.txt +++ b/Documentation/md.txt @@ -602,7 +602,7 @@ These currently include stripe_cache_size (currently raid5 only) number of entries in the stripe cache. This is writable, but - there are upper and lower limits (32768, 16). Default is 128. + there are upper and lower limits (32768, 17). Default is 256. strip_cache_active (currently raid5 only) number of active entries in the stripe cache preread_bypass_threshold (currently raid5 only) diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt index 147ae8ec836f..a4d0a99de04d 100644 --- a/Documentation/memory-barriers.txt +++ b/Documentation/memory-barriers.txt @@ -806,6 +806,41 @@ out-guess your code. More generally, although READ_ONCE() does force the compiler to actually emit code for a given load, it does not force the compiler to use the results. +In addition, control dependencies apply only to the then-clause and +else-clause of the if-statement in question. In particular, it does +not necessarily apply to code following the if-statement: + + q = READ_ONCE(a); + if (q) { + WRITE_ONCE(b, p); + } else { + WRITE_ONCE(b, r); + } + WRITE_ONCE(c, 1); /* BUG: No ordering against the read from "a". */ + +It is tempting to argue that there in fact is ordering because the +compiler cannot reorder volatile accesses and also cannot reorder +the writes to "b" with the condition. Unfortunately for this line +of reasoning, the compiler might compile the two writes to "b" as +conditional-move instructions, as in this fanciful pseudo-assembly +language: + + ld r1,a + ld r2,p + ld r3,r + cmp r1,$0 + cmov,ne r4,r2 + cmov,eq r4,r3 + st r4,b + st $1,c + +A weakly ordered CPU would have no dependency of any sort between the load +from "a" and the store to "c". The control dependencies would extend +only to the pair of cmov instructions and the store depending on them. +In short, control dependencies apply only to the stores in the then-clause +and else-clause of the if-statement in question (including functions +invoked by those two clauses), not to code following that if-statement. + Finally, control dependencies do -not- provide transitivity. This is demonstrated by two related examples, with the initial values of x and y both being zero: @@ -869,6 +904,12 @@ In summary: atomic{,64}_read() can help to preserve your control dependency. Please see the COMPILER BARRIER section for more information. + (*) Control dependencies apply only to the then-clause and else-clause + of the if-statement containing the control dependency, including + any functions that these two clauses call. Control dependencies + do -not- apply to code following the if-statement containing the + control dependency. + (*) Control dependencies pair normally with other types of barriers. (*) Control dependencies do -not- provide transitivity. If you diff --git a/Documentation/mic/mpssd/mpssd.c b/Documentation/mic/mpssd/mpssd.c index 30fb842a976d..49db1def1721 100644 --- a/Documentation/mic/mpssd/mpssd.c +++ b/Documentation/mic/mpssd/mpssd.c @@ -1538,9 +1538,9 @@ set_cmdline(struct mic_info *mic) len = snprintf(buffer, PATH_MAX, "clocksource=tsc highres=off nohz=off "); - len += snprintf(buffer + len, PATH_MAX, + len += snprintf(buffer + len, PATH_MAX - len, "cpufreq_on;corec6_off;pc3_off;pc6_off "); - len += snprintf(buffer + len, PATH_MAX, + len += snprintf(buffer + len, PATH_MAX - len, "ifcfg=static;address,172.31.%d.1;netmask,255.255.255.0", mic->id + 1); diff --git a/Documentation/networking/can.txt b/Documentation/networking/can.txt index d58ff8467953..aa15b9ee2e70 100644 --- a/Documentation/networking/can.txt +++ b/Documentation/networking/can.txt @@ -31,6 +31,7 @@ This file contains 4.2.4 Broadcast Manager message sequence transmission 4.2.5 Broadcast Manager receive filter timers 4.2.6 Broadcast Manager multiplex message receive filter + 4.2.7 Broadcast Manager CAN FD support 4.3 connected transport protocols (SOCK_SEQPACKET) 4.4 unconnected transport protocols (SOCK_DGRAM) @@ -799,7 +800,7 @@ solution for a couple of reasons: } mytxmsg; (..) - mytxmsg.nframes = 4; + mytxmsg.msg_head.nframes = 4; (..) write(s, &mytxmsg, sizeof(mytxmsg)); @@ -852,6 +853,28 @@ solution for a couple of reasons: write(s, &msg, sizeof(msg)); + 4.2.7 Broadcast Manager CAN FD support + + The programming API of the CAN_BCM depends on struct can_frame which is + given as array directly behind the bcm_msg_head structure. To follow this + schema for the CAN FD frames a new flag 'CAN_FD_FRAME' in the bcm_msg_head + flags indicates that the concatenated CAN frame structures behind the + bcm_msg_head are defined as struct canfd_frame. + + struct { + struct bcm_msg_head msg_head; + struct canfd_frame frame[5]; + } msg; + + msg.msg_head.opcode = RX_SETUP; + msg.msg_head.can_id = 0x42; + msg.msg_head.flags = CAN_FD_FRAME; + msg.msg_head.nframes = 5; + (..) + + When using CAN FD frames for multiplex filtering the MUX mask is still + expected in the first 64 bit of the struct canfd_frame data section. + 4.3 connected transport protocols (SOCK_SEQPACKET) 4.4 unconnected transport protocols (SOCK_DGRAM) diff --git a/Documentation/networking/gen_stats.txt b/Documentation/networking/gen_stats.txt index ff630a87b511..179b18ce45ff 100644 --- a/Documentation/networking/gen_stats.txt +++ b/Documentation/networking/gen_stats.txt @@ -21,7 +21,7 @@ struct mystruct { ... }; -Update statistics: +Update statistics, in dequeue() methods only, (while owning qdisc->running) mystruct->tstats.packet++; mystruct->qstats.backlog += skb->pkt_len; diff --git a/Documentation/networking/nf_conntrack-sysctl.txt b/Documentation/networking/nf_conntrack-sysctl.txt index f55599c62c9d..4fb51d32fccc 100644 --- a/Documentation/networking/nf_conntrack-sysctl.txt +++ b/Documentation/networking/nf_conntrack-sysctl.txt @@ -7,12 +7,13 @@ nf_conntrack_acct - BOOLEAN Enable connection tracking flow accounting. 64-bit byte and packet counters per flow are added. -nf_conntrack_buckets - INTEGER (read-only) +nf_conntrack_buckets - INTEGER Size of hash table. If not specified as parameter during module loading, the default size is calculated by dividing total memory by 16384 to determine the number of buckets but the hash table will never have fewer than 32 and limited to 16384 buckets. For systems with more than 4GB of memory it will be 65536 buckets. + This sysctl is only writeable in the initial net namespace. nf_conntrack_checksum - BOOLEAN 0 - disabled diff --git a/Documentation/networking/rds.txt b/Documentation/networking/rds.txt index 9d219d856d46..0235ae69af2a 100644 --- a/Documentation/networking/rds.txt +++ b/Documentation/networking/rds.txt @@ -85,7 +85,8 @@ Socket Interface bind(fd, &sockaddr_in, ...) This binds the socket to a local IP address and port, and a - transport. + transport, if one has not already been selected via the + SO_RDS_TRANSPORT socket option sendmsg(fd, ...) Sends a message to the indicated recipient. The kernel will @@ -146,6 +147,20 @@ Socket Interface operation. In this case, it would use RDS_CANCEL_SENT_TO to nuke any pending messages. + setsockopt(fd, SOL_RDS, SO_RDS_TRANSPORT, (int *)&transport ..) + getsockopt(fd, SOL_RDS, SO_RDS_TRANSPORT, (int *)&transport ..) + Set or read an integer defining the underlying + encapsulating transport to be used for RDS packets on the + socket. When setting the option, integer argument may be + one of RDS_TRANS_TCP or RDS_TRANS_IB. When retrieving the + value, RDS_TRANS_NONE will be returned on an unbound socket. + This socket option may only be set exactly once on the socket, + prior to binding it via the bind(2) system call. Attempts to + set SO_RDS_TRANSPORT on a socket for which the transport has + been previously attached explicitly (by SO_RDS_TRANSPORT) or + implicitly (via bind(2)) will return an error of EOPNOTSUPP. + An attempt to set SO_RDS_TRANSPPORT to RDS_TRANS_NONE will + always return EINVAL. RDMA for RDS ============ @@ -350,4 +365,59 @@ The recv path handle CMSGs return to application +Multipath RDS (mprds) +===================== + Mprds is multipathed-RDS, primarily intended for RDS-over-TCP + (though the concept can be extended to other transports). The classical + implementation of RDS-over-TCP is implemented by demultiplexing multiple + PF_RDS sockets between any 2 endpoints (where endpoint == [IP address, + port]) over a single TCP socket between the 2 IP addresses involved. This + has the limitation that it ends up funneling multiple RDS flows over a + single TCP flow, thus it is + (a) upper-bounded to the single-flow bandwidth, + (b) suffers from head-of-line blocking for all the RDS sockets. + + Better throughput (for a fixed small packet size, MTU) can be achieved + by having multiple TCP/IP flows per rds/tcp connection, i.e., multipathed + RDS (mprds). Each such TCP/IP flow constitutes a path for the rds/tcp + connection. RDS sockets will be attached to a path based on some hash + (e.g., of local address and RDS port number) and packets for that RDS + socket will be sent over the attached path using TCP to segment/reassemble + RDS datagrams on that path. + + Multipathed RDS is implemented by splitting the struct rds_connection into + a common (to all paths) part, and a per-path struct rds_conn_path. All + I/O workqs and reconnect threads are driven from the rds_conn_path. + Transports such as TCP that are multipath capable may then set up a + TPC socket per rds_conn_path, and this is managed by the transport via + the transport privatee cp_transport_data pointer. + + Transports announce themselves as multipath capable by setting the + t_mp_capable bit during registration with the rds core module. When the + transport is multipath-capable, rds_sendmsg() hashes outgoing traffic + across multiple paths. The outgoing hash is computed based on the + local address and port that the PF_RDS socket is bound to. + + Additionally, even if the transport is MP capable, we may be + peering with some node that does not support mprds, or supports + a different number of paths. As a result, the peering nodes need + to agree on the number of paths to be used for the connection. + This is done by sending out a control packet exchange before the + first data packet. The control packet exchange must have completed + prior to outgoing hash completion in rds_sendmsg() when the transport + is mutlipath capable. + + The control packet is an RDS ping packet (i.e., packet to rds dest + port 0) with the ping packet having a rds extension header option of + type RDS_EXTHDR_NPATHS, length 2 bytes, and the value is the + number of paths supported by the sender. The "probe" ping packet will + get sent from some reserved port, RDS_FLAG_PROBE_PORT (in ) + The receiver of a ping from RDS_FLAG_PROBE_PORT will thus immediately + be able to compute the min(sender_paths, rcvr_paths). The pong + sent in response to a probe-ping should contain the rcvr's npaths + when the rcvr is mprds-capable. + + If the rcvr is not mprds-capable, the exthdr in the ping will be + ignored. In this case the pong will not have any exthdrs, so the sender + of the probe-ping can default to single-path mprds. diff --git a/Documentation/networking/stmmac.txt b/Documentation/networking/stmmac.txt index 671fe3dd56d3..e226f8925c9e 100644 --- a/Documentation/networking/stmmac.txt +++ b/Documentation/networking/stmmac.txt @@ -285,6 +285,7 @@ Please see the following document: o mmc_core.c/mmc.h: Management MAC Counters; o stmmac_hwtstamp.c: HW timestamp support for PTP; o stmmac_ptp.c: PTP 1588 clock; + o stmmac_pcs.h: Physical Coding Sublayer common implementation; o dwmac-.c: these are for the platform glue-logic file; e.g. dwmac-sti.c for STMicroelectronics SoCs. diff --git a/Documentation/networking/vrf.txt b/Documentation/networking/vrf.txt index 5da679c573d2..755dab856392 100644 --- a/Documentation/networking/vrf.txt +++ b/Documentation/networking/vrf.txt @@ -15,9 +15,9 @@ the use of higher priority ip rules (Policy Based Routing, PBR) to take precedence over the VRF device rules directing specific traffic as desired. In addition, VRF devices allow VRFs to be nested within namespaces. For -example network namespaces provide separation of network interfaces at L1 -(Layer 1 separation), VLANs on the interfaces within a namespace provide -L2 separation and then VRF devices provide L3 separation. +example network namespaces provide separation of network interfaces at the +device layer, VLANs on the interfaces within a namespace provide L2 separation +and then VRF devices provide L3 separation. Design ------ @@ -37,21 +37,22 @@ are then enslaved to a VRF device: +------+ +------+ Packets received on an enslaved device and are switched to the VRF device -using an rx_handler which gives the impression that packets flow through -the VRF device. Similarly on egress routing rules are used to send packets -to the VRF device driver before getting sent out the actual interface. This -allows tcpdump on a VRF device to capture all packets into and out of the -VRF as a whole.[1] Similarly, netfilter [2] and tc rules can be applied -using the VRF device to specify rules that apply to the VRF domain as a whole. +in the IPv4 and IPv6 processing stacks giving the impression that packets +flow through the VRF device. Similarly on egress routing rules are used to +send packets to the VRF device driver before getting sent out the actual +interface. This allows tcpdump on a VRF device to capture all packets into +and out of the VRF as a whole.[1] Similarly, netfilter[2] and tc rules can be +applied using the VRF device to specify rules that apply to the VRF domain +as a whole. [1] Packets in the forwarded state do not flow through the device, so those packets are not seen by tcpdump. Will revisit this limitation in a future release. -[2] Iptables on ingress is limited to NF_INET_PRE_ROUTING only with skb->dev - set to real ingress device and egress is limited to NF_INET_POST_ROUTING. - Will revisit this limitation in a future release. - +[2] Iptables on ingress supports PREROUTING with skb->dev set to the real + ingress device and both INPUT and PREROUTING rules with skb->dev set to + the VRF device. For egress POSTROUTING and OUTPUT rules can be written + using either the VRF device or real egress device. Setup ----- @@ -59,23 +60,33 @@ Setup e.g, ip link add vrf-blue type vrf table 10 ip link set dev vrf-blue up -2. Rules are added that send lookups to the associated FIB table when the - iif or oif is the VRF device. e.g., +2. An l3mdev FIB rule directs lookups to the table associated with the device. + A single l3mdev rule is sufficient for all VRFs. The VRF device adds the + l3mdev rule for IPv4 and IPv6 when the first device is created with a + default preference of 1000. Users may delete the rule if desired and add + with a different priority or install per-VRF rules. + + Prior to the v4.8 kernel iif and oif rules are needed for each VRF device: ip ru add oif vrf-blue table 10 ip ru add iif vrf-blue table 10 - Set the default route for the table (and hence default route for the VRF). - e.g, ip route add table 10 prohibit default +3. Set the default route for the table (and hence default route for the VRF). + ip route add table 10 unreachable default -3. Enslave L3 interfaces to a VRF device. - e.g, ip link set dev eth1 master vrf-blue +4. Enslave L3 interfaces to a VRF device. + ip link set dev eth1 master vrf-blue Local and connected routes for enslaved devices are automatically moved to the table associated with VRF device. Any additional routes depending on - the enslaved device will need to be reinserted following the enslavement. + the enslaved device are dropped and will need to be reinserted to the VRF + FIB table following the enslavement. + + The IPv6 sysctl option keep_addr_on_down can be enabled to keep IPv6 global + addresses as VRF enslavement changes. + sysctl -w net.ipv6.conf.all.keep_addr_on_down=1 -4. Additional VRF routes are added to associated table. - e.g., ip route add table 10 ... +5. Additional VRF routes are added to associated table. + ip route add table 10 ... Applications @@ -87,39 +98,34 @@ VRF device: or to specify the output device using cmsg and IP_PKTINFO. +TCP services running in the default VRF context (ie., not bound to any VRF +device) can work across all VRF domains by enabling the tcp_l3mdev_accept +sysctl option: + sysctl -w net.ipv4.tcp_l3mdev_accept=1 -Limitations ------------ -Index of original ingress interface is not available via cmsg. Will address -soon. +netfilter rules on the VRF device can be used to limit access to services +running in the default VRF context as well. + +The default VRF does not have limited scope with respect to port bindings. +That is, if a process does a wildcard bind to a port in the default VRF it +owns the port across all VRF domains within the network namespace. ################################################################################ Using iproute2 for VRFs ======================= -VRF devices do *not* have to start with 'vrf-'. That is a convention used here -for emphasis of the device type, similar to use of 'br' in bridge names. +iproute2 supports the vrf keyword as of v4.7. For backwards compatibility this +section lists both commands where appropriate -- with the vrf keyword and the +older form without it. 1. Create a VRF To instantiate a VRF device and associate it with a table: $ ip link add dev NAME type vrf table ID - Remember to add the ip rules as well: - $ ip ru add oif NAME table 10 - $ ip ru add iif NAME table 10 - $ ip -6 ru add oif NAME table 10 - $ ip -6 ru add iif NAME table 10 - - Without the rules route lookups are not directed to the table. - - For example: - $ ip link add dev vrf-blue type vrf table 10 - $ ip ru add pref 200 oif vrf-blue table 10 - $ ip ru add pref 200 iif vrf-blue table 10 - $ ip -6 ru add pref 200 oif vrf-blue table 10 - $ ip -6 ru add pref 200 iif vrf-blue table 10 - + As of v4.8 the kernel supports the l3mdev FIB rule where a single rule + covers all VRFs. The l3mdev rule is created for IPv4 and IPv6 on first + device create. 2. List VRFs @@ -129,16 +135,16 @@ for emphasis of the device type, similar to use of 'br' in bridge names. For example: $ ip -d link show type vrf - 11: vrf-mgmt: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 + 11: mgmt: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether 72:b3:ba:91:e2:24 brd ff:ff:ff:ff:ff:ff promiscuity 0 vrf table 1 addrgenmode eui64 - 12: vrf-red: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 + 12: red: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether b6:6f:6e:f6:da:73 brd ff:ff:ff:ff:ff:ff promiscuity 0 vrf table 10 addrgenmode eui64 - 13: vrf-blue: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 + 13: blue: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether 36:62:e8:7d:bb:8c brd ff:ff:ff:ff:ff:ff promiscuity 0 vrf table 66 addrgenmode eui64 - 14: vrf-green: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 + 14: green: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether e6:28:b8:63:70:bb brd ff:ff:ff:ff:ff:ff promiscuity 0 vrf table 81 addrgenmode eui64 @@ -146,43 +152,44 @@ for emphasis of the device type, similar to use of 'br' in bridge names. Or in brief output: $ ip -br link show type vrf - vrf-mgmt UP 72:b3:ba:91:e2:24 - vrf-red UP b6:6f:6e:f6:da:73 - vrf-blue UP 36:62:e8:7d:bb:8c - vrf-green UP e6:28:b8:63:70:bb + mgmt UP 72:b3:ba:91:e2:24 + red UP b6:6f:6e:f6:da:73 + blue UP 36:62:e8:7d:bb:8c + green UP e6:28:b8:63:70:bb 3. Assign a Network Interface to a VRF Network interfaces are assigned to a VRF by enslaving the netdevice to a VRF device: - $ ip link set dev NAME master VRF-NAME + $ ip link set dev NAME master NAME On enslavement connected and local routes are automatically moved to the table associated with the VRF device. For example: - $ ip link set dev eth0 master vrf-mgmt + $ ip link set dev eth0 master mgmt 4. Show Devices Assigned to a VRF To show devices that have been assigned to a specific VRF add the master option to the ip command: - $ ip link show master VRF-NAME + $ ip link show vrf NAME + $ ip link show master NAME For example: - $ ip link show master vrf-red - 3: eth1: mtu 1500 qdisc pfifo_fast master vrf-red state UP mode DEFAULT group default qlen 1000 + $ ip link show vrf red + 3: eth1: mtu 1500 qdisc pfifo_fast master red state UP mode DEFAULT group default qlen 1000 link/ether 02:00:00:00:02:02 brd ff:ff:ff:ff:ff:ff - 4: eth2: mtu 1500 qdisc pfifo_fast master vrf-red state UP mode DEFAULT group default qlen 1000 + 4: eth2: mtu 1500 qdisc pfifo_fast master red state UP mode DEFAULT group default qlen 1000 link/ether 02:00:00:00:02:03 brd ff:ff:ff:ff:ff:ff - 7: eth5: mtu 1500 qdisc noop master vrf-red state DOWN mode DEFAULT group default qlen 1000 + 7: eth5: mtu 1500 qdisc noop master red state DOWN mode DEFAULT group default qlen 1000 link/ether 02:00:00:00:02:06 brd ff:ff:ff:ff:ff:ff Or using the brief output: - $ ip -br link show master vrf-red + $ ip -br link show vrf red eth1 UP 02:00:00:00:02:02 eth2 UP 02:00:00:00:02:03 eth5 DOWN 02:00:00:00:02:06 @@ -192,26 +199,28 @@ for emphasis of the device type, similar to use of 'br' in bridge names. To list neighbor entries associated with devices enslaved to a VRF device add the master option to the ip command: - $ ip [-6] neigh show master VRF-NAME + $ ip [-6] neigh show vrf NAME + $ ip [-6] neigh show master NAME For example: - $ ip neigh show master vrf-red + $ ip neigh show vrf red 10.2.1.254 dev eth1 lladdr a6:d9:c7:4f:06:23 REACHABLE 10.2.2.254 dev eth2 lladdr 5e:54:01:6a:ee:80 REACHABLE - $ ip -6 neigh show master vrf-red - 2002:1::64 dev eth1 lladdr a6:d9:c7:4f:06:23 REACHABLE + $ ip -6 neigh show vrf red + 2002:1::64 dev eth1 lladdr a6:d9:c7:4f:06:23 REACHABLE 6. Show Addresses for a VRF To show addresses for interfaces associated with a VRF add the master option to the ip command: - $ ip addr show master VRF-NAME + $ ip addr show vrf NAME + $ ip addr show master NAME For example: - $ ip addr show master vrf-red - 3: eth1: mtu 1500 qdisc pfifo_fast master vrf-red state UP group default qlen 1000 + $ ip addr show vrf red + 3: eth1: mtu 1500 qdisc pfifo_fast master red state UP group default qlen 1000 link/ether 02:00:00:00:02:02 brd ff:ff:ff:ff:ff:ff inet 10.2.1.2/24 brd 10.2.1.255 scope global eth1 valid_lft forever preferred_lft forever @@ -219,7 +228,7 @@ for emphasis of the device type, similar to use of 'br' in bridge names. valid_lft forever preferred_lft forever inet6 fe80::ff:fe00:202/64 scope link valid_lft forever preferred_lft forever - 4: eth2: mtu 1500 qdisc pfifo_fast master vrf-red state UP group default qlen 1000 + 4: eth2: mtu 1500 qdisc pfifo_fast master red state UP group default qlen 1000 link/ether 02:00:00:00:02:03 brd ff:ff:ff:ff:ff:ff inet 10.2.2.2/24 brd 10.2.2.255 scope global eth2 valid_lft forever preferred_lft forever @@ -227,11 +236,11 @@ for emphasis of the device type, similar to use of 'br' in bridge names. valid_lft forever preferred_lft forever inet6 fe80::ff:fe00:203/64 scope link valid_lft forever preferred_lft forever - 7: eth5: mtu 1500 qdisc noop master vrf-red state DOWN group default qlen 1000 + 7: eth5: mtu 1500 qdisc noop master red state DOWN group default qlen 1000 link/ether 02:00:00:00:02:06 brd ff:ff:ff:ff:ff:ff Or in brief format: - $ ip -br addr show master vrf-red + $ ip -br addr show vrf red eth1 UP 10.2.1.2/24 2002:1::2/120 fe80::ff:fe00:202/64 eth2 UP 10.2.2.2/24 2002:2::2/120 fe80::ff:fe00:203/64 eth5 DOWN @@ -241,10 +250,11 @@ for emphasis of the device type, similar to use of 'br' in bridge names. To show routes for a VRF use the ip command to display the table associated with the VRF device: + $ ip [-6] route show vrf NAME $ ip [-6] route show table ID For example: - $ ip route show table vrf-red + $ ip route show vrf red prohibit default broadcast 10.2.1.0 dev eth1 proto kernel scope link src 10.2.1.2 10.2.1.0/24 dev eth1 proto kernel scope link src 10.2.1.2 @@ -255,7 +265,7 @@ for emphasis of the device type, similar to use of 'br' in bridge names. local 10.2.2.2 dev eth2 proto kernel scope host src 10.2.2.2 broadcast 10.2.2.255 dev eth2 proto kernel scope link src 10.2.2.2 - $ ip -6 route show table vrf-red + $ ip -6 route show vrf red local 2002:1:: dev lo proto none metric 0 pref medium local 2002:1::2 dev lo proto none metric 0 pref medium 2002:1::/120 dev eth1 proto kernel metric 256 pref medium @@ -268,23 +278,24 @@ for emphasis of the device type, similar to use of 'br' in bridge names. local fe80::ff:fe00:203 dev lo proto none metric 0 pref medium fe80::/64 dev eth1 proto kernel metric 256 pref medium fe80::/64 dev eth2 proto kernel metric 256 pref medium - ff00::/8 dev vrf-red metric 256 pref medium + ff00::/8 dev red metric 256 pref medium ff00::/8 dev eth1 metric 256 pref medium ff00::/8 dev eth2 metric 256 pref medium 8. Route Lookup for a VRF - A test route lookup can be done for a VRF by adding the oif option to ip: - $ ip [-6] route get oif VRF-NAME ADDRESS + A test route lookup can be done for a VRF: + $ ip [-6] route get vrf NAME ADDRESS + $ ip [-6] route get oif NAME ADDRESS For example: - $ ip route get 10.2.1.40 oif vrf-red - 10.2.1.40 dev eth1 table vrf-red src 10.2.1.2 + $ ip route get 10.2.1.40 vrf red + 10.2.1.40 dev eth1 table red src 10.2.1.2 cache - $ ip -6 route get 2002:1::32 oif vrf-red - 2002:1::32 from :: dev eth1 table vrf-red proto kernel src 2002:1::2 metric 256 pref medium + $ ip -6 route get 2002:1::32 vrf red + 2002:1::32 from :: dev eth1 table red proto kernel src 2002:1::2 metric 256 pref medium 9. Removing Network Interface from a VRF @@ -303,46 +314,40 @@ for emphasis of the device type, similar to use of 'br' in bridge names. Commands used in this example: -cat >> /etc/iproute2/rt_tables <> /etc/iproute2/rt_tables.d/vrf.conf < diff --git a/Documentation/s390/s390dbf.txt b/Documentation/s390/s390dbf.txt index 3da163383c93..61329fd62e89 100644 --- a/Documentation/s390/s390dbf.txt +++ b/Documentation/s390/s390dbf.txt @@ -405,7 +405,7 @@ Example: > ls /sys/kernel/debug/s390dbf/dasd flush hex_ascii level pages raw -> cat /sys/kernel/debug/s390dbf/dasd/hex_ascii | sort +1 +> cat /sys/kernel/debug/s390dbf/dasd/hex_ascii | sort -k2,2 -s 00 00974733272:680099 2 - 02 0006ad7e 07 ea 4a 90 | .... 00 00974733272:682210 2 - 02 0006ade6 46 52 45 45 | FREE 00 00974733272:682213 2 - 02 0006adf6 07 ea 4a 90 | .... diff --git a/Documentation/scsi/scsi_eh.txt b/Documentation/scsi/scsi_eh.txt index 8638f61c8c9d..37eca00796ee 100644 --- a/Documentation/scsi/scsi_eh.txt +++ b/Documentation/scsi/scsi_eh.txt @@ -263,19 +263,23 @@ scmd->allowed. 3. scmd recovered ACTION: scsi_eh_finish_cmd() is invoked to EH-finish scmd - - shost->host_failed-- - clear scmd->eh_eflags - scsi_setup_cmd_retry() - move from local eh_work_q to local eh_done_q LOCKING: none + CONCURRENCY: at most one thread per separate eh_work_q to + keep queue manipulation lockless 4. EH completes ACTION: scsi_eh_flush_done_q() retries scmds or notifies upper - layer of failure. + layer of failure. May be called concurrently but must have + a no more than one thread per separate eh_work_q to + manipulate the queue locklessly - scmd is removed from eh_done_q and scmd->eh_entry is cleared - if retry is necessary, scmd is requeued using scsi_queue_insert() - otherwise, scsi_finish_command() is invoked for scmd + - zero shost->host_failed LOCKING: queue or finish function performs appropriate locking diff --git a/Documentation/security/self-protection.txt b/Documentation/security/self-protection.txt index babd6378ec05..3010576c9fca 100644 --- a/Documentation/security/self-protection.txt +++ b/Documentation/security/self-protection.txt @@ -183,8 +183,9 @@ provide meaningful defenses. ### Canaries, blinding, and other secrets It should be noted that things like the stack canary discussed earlier -are technically statistical defenses, since they rely on a (leakable) -secret value. +are technically statistical defenses, since they rely on a secret value, +and such values may become discoverable through an information exposure +flaw. Blinding literal values for things like JITs, where the executable contents may be partially under the control of userspace, need a similar @@ -199,8 +200,8 @@ working?) in order to maximize their success. Since the location of kernel memory is almost always instrumental in mounting a successful attack, making the location non-deterministic raises the difficulty of an exploit. (Note that this in turn makes -the value of leaks higher, since they may be used to discover desired -memory locations.) +the value of information exposures higher, since they may be used to +discover desired memory locations.) #### Text and module base @@ -222,14 +223,21 @@ become more difficult to locate. Much of the kernel's dynamic memory (e.g. kmalloc, vmalloc, etc) ends up being relatively deterministic in layout due to the order of early-boot initializations. If the base address of these areas is not the same -between boots, targeting them is frustrated, requiring a leak specific -to the region. +between boots, targeting them is frustrated, requiring an information +exposure specific to the region. + +#### Structure layout + +By performing a per-build randomization of the layout of sensitive +structures, attacks must either be tuned to known kernel builds or expose +enough kernel memory to determine structure layouts before manipulating +them. -## Preventing Leaks +## Preventing Information Exposures Since the locations of sensitive structures are the primary target for -attacks, it is important to defend against leaks of both kernel memory +attacks, it is important to defend against exposure of both kernel memory addresses and kernel memory contents (since they may contain kernel addresses or other sensitive things like canary values). @@ -250,8 +258,8 @@ sure structure holes are cleared. When releasing memory, it is best to poison the contents (clear stack on syscall return, wipe heap memory on a free), to avoid reuse attacks that rely on the old contents of memory. This frustrates many uninitialized -variable attacks, stack info leaks, heap info leaks, and use-after-free -attacks. +variable attacks, stack content exposures, heap content exposures, and +use-after-free attacks. ### Destination tracking diff --git a/Documentation/sphinx/convert_template.sed b/Documentation/sphinx/convert_template.sed new file mode 100644 index 000000000000..c1503fcca4ec --- /dev/null +++ b/Documentation/sphinx/convert_template.sed @@ -0,0 +1,18 @@ +# +# Pandoc doesn't grok or , so convert them +# ahead of time. +# +# Use the following escapes to pass through pandoc: +# $bq = "`" +# $lt = "<" +# $gt = ">" +# +s%\([^<(]\+\)()%:c:func:$bq\1()$bq%g +s%\([^<(]\+\)%:c:func:$bq\1()$bq%g +s%struct *\([^<]\+\)%:c:type:$bqstruct \1 $lt\1$gt$bq%g +s%struct \([^<]\+\)%:c:type:$bqstruct \1 $lt\1$gt$bq%g +s%\([^<]\+\)%:c:type:$bqstruct \1 $lt\1$gt$bq%g +# +# Wrap docproc directives in para and code blocks. +# +s%^\(!.*\)$%DOCPROC: \1% diff --git a/Documentation/sphinx/kernel-doc.py b/Documentation/sphinx/kernel-doc.py new file mode 100644 index 000000000000..f6920c0af6ee --- /dev/null +++ b/Documentation/sphinx/kernel-doc.py @@ -0,0 +1,141 @@ +# coding=utf-8 +# +# Copyright © 2016 Intel Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice (including the next +# paragraph) shall be included in all copies or substantial portions of the +# Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# Authors: +# Jani Nikula +# +# Please make sure this works on both python2 and python3. +# + +import os +import subprocess +import sys +import re +import glob + +from docutils import nodes, statemachine +from docutils.statemachine import ViewList +from docutils.parsers.rst import directives +from sphinx.util.compat import Directive +from sphinx.ext.autodoc import AutodocReporter + +class KernelDocDirective(Directive): + """Extract kernel-doc comments from the specified file""" + required_argument = 1 + optional_arguments = 4 + option_spec = { + 'doc': directives.unchanged_required, + 'functions': directives.unchanged_required, + 'export': directives.unchanged, + 'internal': directives.unchanged, + } + has_content = False + + def run(self): + env = self.state.document.settings.env + cmd = [env.config.kerneldoc_bin, '-rst', '-enable-lineno'] + + filename = env.config.kerneldoc_srctree + '/' + self.arguments[0] + export_file_patterns = [] + + # Tell sphinx of the dependency + env.note_dependency(os.path.abspath(filename)) + + tab_width = self.options.get('tab-width', self.state.document.settings.tab_width) + + # FIXME: make this nicer and more robust against errors + if 'export' in self.options: + cmd += ['-export'] + export_file_patterns = str(self.options.get('export')).split() + elif 'internal' in self.options: + cmd += ['-internal'] + export_file_patterns = str(self.options.get('internal')).split() + elif 'doc' in self.options: + cmd += ['-function', str(self.options.get('doc'))] + elif 'functions' in self.options: + for f in str(self.options.get('functions')).split(): + cmd += ['-function', f] + + for pattern in export_file_patterns: + for f in glob.glob(env.config.kerneldoc_srctree + '/' + pattern): + env.note_dependency(os.path.abspath(f)) + cmd += ['-export-file', f] + + cmd += [filename] + + try: + env.app.verbose('calling kernel-doc \'%s\'' % (" ".join(cmd))) + + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + out, err = p.communicate() + + # python2 needs conversion to unicode. + # python3 with universal_newlines=True returns strings. + if sys.version_info.major < 3: + out, err = unicode(out, 'utf-8'), unicode(err, 'utf-8') + + if p.returncode != 0: + sys.stderr.write(err) + + env.app.warn('kernel-doc \'%s\' failed with return code %d' % (" ".join(cmd), p.returncode)) + return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] + elif env.config.kerneldoc_verbosity > 0: + sys.stderr.write(err) + + lines = statemachine.string2lines(out, tab_width, convert_whitespace=True) + result = ViewList() + + lineoffset = 0; + line_regex = re.compile("^#define LINENO ([0-9]+)$") + for line in lines: + match = line_regex.search(line) + if match: + # sphinx counts lines from 0 + lineoffset = int(match.group(1)) - 1 + # we must eat our comments since the upset the markup + else: + result.append(line, filename, lineoffset) + lineoffset += 1 + + node = nodes.section() + buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter + self.state.memo.reporter = AutodocReporter(result, self.state.memo.reporter) + self.state.memo.title_styles, self.state.memo.section_level = [], 0 + try: + self.state.nested_parse(result, 0, node, match_titles=1) + finally: + self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + + return node.children + + except Exception as e: # pylint: disable=W0703 + env.app.warn('kernel-doc \'%s\' processing failed with: %s' % + (" ".join(cmd), str(e))) + return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] + +def setup(app): + app.add_config_value('kerneldoc_bin', None, 'env') + app.add_config_value('kerneldoc_srctree', None, 'env') + app.add_config_value('kerneldoc_verbosity', 1, 'env') + + app.add_directive('kernel-doc', KernelDocDirective) diff --git a/Documentation/sphinx/post_convert.sed b/Documentation/sphinx/post_convert.sed new file mode 100644 index 000000000000..392770bac53b --- /dev/null +++ b/Documentation/sphinx/post_convert.sed @@ -0,0 +1,23 @@ +# +# Unescape. +# +s/$bq/`/g +s/$lt//g +# +# pandoc thinks that both "_" needs to be escaped. Remove the extra +# backslashes. +# +s/\\_/_/g +# +# Unwrap docproc directives. +# +s/^``DOCPROC: !E\(.*\)``$/.. kernel-doc:: \1\n :export:/ +s/^``DOCPROC: !I\(.*\)``$/.. kernel-doc:: \1\n :internal:/ +s/^``DOCPROC: !F\([^ ]*\) \(.*\)``$/.. kernel-doc:: \1\n :functions: \2/ +s/^``DOCPROC: !P\([^ ]*\) \(.*\)``$/.. kernel-doc:: \1\n :doc: \2/ +s/^``DOCPROC: \(!.*\)``$/.. WARNING: DOCPROC directive not supported: \1/ +# +# Trim trailing whitespace. +# +s/[[:space:]]*$// diff --git a/Documentation/sphinx/rstFlatTable.py b/Documentation/sphinx/rstFlatTable.py new file mode 100644 index 000000000000..26db852e3c74 --- /dev/null +++ b/Documentation/sphinx/rstFlatTable.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; mode: python -*- +# pylint: disable=C0330, R0903, R0912 + +u""" + flat-table + ~~~~~~~~~~ + + Implementation of the ``flat-table`` reST-directive. + + :copyright: Copyright (C) 2016 Markus Heiser + :license: GPL Version 2, June 1991 see linux/COPYING for details. + + The ``flat-table`` (:py:class:`FlatTable`) is a double-stage list similar to + the ``list-table`` with some additional features: + + * *column-span*: with the role ``cspan`` a cell can be extended through + additional columns + + * *row-span*: with the role ``rspan`` a cell can be extended through + additional rows + + * *auto span* rightmost cell of a table row over the missing cells on the + right side of that table-row. With Option ``:fill-cells:`` this behavior + can changed from *auto span* to *auto fill*, which automaticly inserts + (empty) cells instead of spanning the last cell. + + Options: + + * header-rows: [int] count of header rows + * stub-columns: [int] count of stub columns + * widths: [[int] [int] ... ] widths of columns + * fill-cells: instead of autospann missing cells, insert missing cells + + roles: + + * cspan: [int] additionale columns (*morecols*) + * rspan: [int] additionale rows (*morerows*) +""" + +# ============================================================================== +# imports +# ============================================================================== + +import sys + +from docutils import nodes +from docutils.parsers.rst import directives, roles +from docutils.parsers.rst.directives.tables import Table +from docutils.utils import SystemMessagePropagation + +# ============================================================================== +# common globals +# ============================================================================== + +# The version numbering follows numbering of the specification +# (Documentation/books/kernel-doc-HOWTO). +__version__ = '1.0' + +PY3 = sys.version_info[0] == 3 +PY2 = sys.version_info[0] == 2 + +if PY3: + # pylint: disable=C0103, W0622 + unicode = str + basestring = str + +# ============================================================================== +def setup(app): +# ============================================================================== + + app.add_directive("flat-table", FlatTable) + roles.register_local_role('cspan', c_span) + roles.register_local_role('rspan', r_span) + +# ============================================================================== +def c_span(name, rawtext, text, lineno, inliner, options=None, content=None): +# ============================================================================== + # pylint: disable=W0613 + + options = options if options is not None else {} + content = content if content is not None else [] + nodelist = [colSpan(span=int(text))] + msglist = [] + return nodelist, msglist + +# ============================================================================== +def r_span(name, rawtext, text, lineno, inliner, options=None, content=None): +# ============================================================================== + # pylint: disable=W0613 + + options = options if options is not None else {} + content = content if content is not None else [] + nodelist = [rowSpan(span=int(text))] + msglist = [] + return nodelist, msglist + + +# ============================================================================== +class rowSpan(nodes.General, nodes.Element): pass # pylint: disable=C0103,C0321 +class colSpan(nodes.General, nodes.Element): pass # pylint: disable=C0103,C0321 +# ============================================================================== + +# ============================================================================== +class FlatTable(Table): +# ============================================================================== + + u"""FlatTable (``flat-table``) directive""" + + option_spec = { + 'name': directives.unchanged + , 'class': directives.class_option + , 'header-rows': directives.nonnegative_int + , 'stub-columns': directives.nonnegative_int + , 'widths': directives.positive_int_list + , 'fill-cells' : directives.flag } + + def run(self): + + if not self.content: + error = self.state_machine.reporter.error( + 'The "%s" directive is empty; content required.' % self.name, + nodes.literal_block(self.block_text, self.block_text), + line=self.lineno) + return [error] + + title, messages = self.make_title() + node = nodes.Element() # anonymous container for parsing + self.state.nested_parse(self.content, self.content_offset, node) + + tableBuilder = ListTableBuilder(self) + tableBuilder.parseFlatTableNode(node) + tableNode = tableBuilder.buildTableNode() + # SDK.CONSOLE() # print --> tableNode.asdom().toprettyxml() + if title: + tableNode.insert(0, title) + return [tableNode] + messages + + +# ============================================================================== +class ListTableBuilder(object): +# ============================================================================== + + u"""Builds a table from a double-stage list""" + + def __init__(self, directive): + self.directive = directive + self.rows = [] + self.max_cols = 0 + + def buildTableNode(self): + + colwidths = self.directive.get_column_widths(self.max_cols) + stub_columns = self.directive.options.get('stub-columns', 0) + header_rows = self.directive.options.get('header-rows', 0) + + table = nodes.table() + tgroup = nodes.tgroup(cols=len(colwidths)) + table += tgroup + + + for colwidth in colwidths: + colspec = nodes.colspec(colwidth=colwidth) + # FIXME: It seems, that the stub method only works well in the + # absence of rowspan (observed by the html buidler, the docutils-xml + # build seems OK). This is not extraordinary, because there exists + # no table directive (except *this* flat-table) which allows to + # define coexistent of rowspan and stubs (there was no use-case + # before flat-table). This should be reviewed (later). + if stub_columns: + colspec.attributes['stub'] = 1 + stub_columns -= 1 + tgroup += colspec + stub_columns = self.directive.options.get('stub-columns', 0) + + if header_rows: + thead = nodes.thead() + tgroup += thead + for row in self.rows[:header_rows]: + thead += self.buildTableRowNode(row) + + tbody = nodes.tbody() + tgroup += tbody + + for row in self.rows[header_rows:]: + tbody += self.buildTableRowNode(row) + return table + + def buildTableRowNode(self, row_data, classes=None): + classes = [] if classes is None else classes + row = nodes.row() + for cell in row_data: + if cell is None: + continue + cspan, rspan, cellElements = cell + + attributes = {"classes" : classes} + if rspan: + attributes['morerows'] = rspan + if cspan: + attributes['morecols'] = cspan + entry = nodes.entry(**attributes) + entry.extend(cellElements) + row += entry + return row + + def raiseError(self, msg): + error = self.directive.state_machine.reporter.error( + msg + , nodes.literal_block(self.directive.block_text + , self.directive.block_text) + , line = self.directive.lineno ) + raise SystemMessagePropagation(error) + + def parseFlatTableNode(self, node): + u"""parses the node from a :py:class:`FlatTable` directive's body""" + + if len(node) != 1 or not isinstance(node[0], nodes.bullet_list): + self.raiseError( + 'Error parsing content block for the "%s" directive: ' + 'exactly one bullet list expected.' % self.directive.name ) + + for rowNum, rowItem in enumerate(node[0]): + row = self.parseRowItem(rowItem, rowNum) + self.rows.append(row) + self.roundOffTableDefinition() + + def roundOffTableDefinition(self): + u"""Round off the table definition. + + This method rounds off the table definition in :py:member:`rows`. + + * This method inserts the needed ``None`` values for the missing cells + arising from spanning cells over rows and/or columns. + + * recount the :py:member:`max_cols` + + * Autospan or fill (option ``fill-cells``) missing cells on the right + side of the table-row + """ + + y = 0 + while y < len(self.rows): + x = 0 + + while x < len(self.rows[y]): + cell = self.rows[y][x] + if cell is None: + x += 1 + continue + cspan, rspan = cell[:2] + # handle colspan in current row + for c in range(cspan): + try: + self.rows[y].insert(x+c+1, None) + except: # pylint: disable=W0702 + # the user sets ambiguous rowspans + pass # SDK.CONSOLE() + # handle colspan in spanned rows + for r in range(rspan): + for c in range(cspan + 1): + try: + self.rows[y+r+1].insert(x+c, None) + except: # pylint: disable=W0702 + # the user sets ambiguous rowspans + pass # SDK.CONSOLE() + x += 1 + y += 1 + + # Insert the missing cells on the right side. For this, first + # re-calculate the max columns. + + for row in self.rows: + if self.max_cols < len(row): + self.max_cols = len(row) + + # fill with empty cells or cellspan? + + fill_cells = False + if 'fill-cells' in self.directive.options: + fill_cells = True + + for row in self.rows: + x = self.max_cols - len(row) + if x and not fill_cells: + if row[-1] is None: + row.append( ( x - 1, 0, []) ) + else: + cspan, rspan, content = row[-1] + row[-1] = (cspan + x, rspan, content) + elif x and fill_cells: + for i in range(x): + row.append( (0, 0, nodes.comment()) ) + + def pprint(self): + # for debugging + retVal = "[ " + for row in self.rows: + retVal += "[ " + for col in row: + if col is None: + retVal += ('%r' % col) + retVal += "\n , " + else: + content = col[2][0].astext() + if len (content) > 30: + content = content[:30] + "..." + retVal += ('(cspan=%s, rspan=%s, %r)' + % (col[0], col[1], content)) + retVal += "]\n , " + retVal = retVal[:-2] + retVal += "]\n , " + retVal = retVal[:-2] + return retVal + "]" + + def parseRowItem(self, rowItem, rowNum): + row = [] + childNo = 0 + error = False + cell = None + target = None + + for child in rowItem: + if (isinstance(child , nodes.comment) + or isinstance(child, nodes.system_message)): + pass + elif isinstance(child , nodes.target): + target = child + elif isinstance(child, nodes.bullet_list): + childNo += 1 + cell = child + else: + error = True + break + + if childNo != 1 or error: + self.raiseError( + 'Error parsing content block for the "%s" directive: ' + 'two-level bullet list expected, but row %s does not ' + 'contain a second-level bullet list.' + % (self.directive.name, rowNum + 1)) + + for cellItem in cell: + cspan, rspan, cellElements = self.parseCellItem(cellItem) + if target is not None: + cellElements.insert(0, target) + row.append( (cspan, rspan, cellElements) ) + return row + + def parseCellItem(self, cellItem): + # search and remove cspan, rspan colspec from the first element in + # this listItem (field). + cspan = rspan = 0 + if not len(cellItem): + return cspan, rspan, [] + for elem in cellItem[0]: + if isinstance(elem, colSpan): + cspan = elem.get("span") + elem.parent.remove(elem) + continue + if isinstance(elem, rowSpan): + rspan = elem.get("span") + elem.parent.remove(elem) + continue + return cspan, rspan, cellItem[:] diff --git a/Documentation/sphinx/tmplcvt b/Documentation/sphinx/tmplcvt new file mode 100755 index 000000000000..909a73065e0a --- /dev/null +++ b/Documentation/sphinx/tmplcvt @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Convert a template file into something like RST +# +# fix +# feed to pandoc +# fix \_ +# title line? +# + +in=$1 +rst=$2 +tmp=$rst.tmp + +cp $in $tmp +sed --in-place -f convert_template.sed $tmp +pandoc -s -S -f docbook -t rst -o $rst $tmp +sed --in-place -f post_convert.sed $rst +rm $tmp diff --git a/Documentation/sync_file.txt b/Documentation/sync_file.txt index eaf8297dbca2..e8e2ebafe5fa 100644 --- a/Documentation/sync_file.txt +++ b/Documentation/sync_file.txt @@ -6,8 +6,8 @@ This document serves as a guide for device drivers writers on what the sync_file API is, and how drivers can support it. Sync file is the carrier of -the fences(struct fence) that needs to synchronized between drivers or across -process boundaries. +the fences(struct fence) that are needed to synchronize between drivers or +across process boundaries. The sync_file API is meant to be used to send and receive fence information to/from userspace. It enables userspace to do explicit fencing, where instead @@ -32,7 +32,7 @@ in-fences and out-fences Sync files can go either to or from userspace. When a sync_file is sent from the driver to userspace we call the fences it contains 'out-fences'. They are related to a buffer that the driver is processing or is going to process, so -the driver an create out-fence to be able to notify, through fence_signal(), +the driver creates an out-fence to be able to notify, through fence_signal(), when it has finished using (or processing) that buffer. Out-fences are fences that the driver creates. diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt index a3683ce2a2f3..33204604de6c 100644 --- a/Documentation/sysctl/kernel.txt +++ b/Documentation/sysctl/kernel.txt @@ -58,6 +58,7 @@ show up in /proc/sys/kernel: - panic_on_stackoverflow - panic_on_unrecovered_nmi - panic_on_warn +- panic_on_rcu_stall - perf_cpu_time_max_percent - perf_event_paranoid - perf_event_max_stack @@ -618,6 +619,17 @@ a kernel rebuild when attempting to kdump at the location of a WARN(). ============================================================== +panic_on_rcu_stall: + +When set to 1, calls panic() after RCU stall detection messages. This +is useful to define the root cause of RCU stalls using a vmcore. + +0: do not panic() when RCU stall takes place, default behavior. + +1: panic() after printing RCU stall messages. + +============================================================== + perf_cpu_time_max_percent: Hints to the kernel how much CPU time it should be allowed to diff --git a/Documentation/sysctl/vm.txt b/Documentation/sysctl/vm.txt index 720355cbdf45..95ccbe6d79ce 100644 --- a/Documentation/sysctl/vm.txt +++ b/Documentation/sysctl/vm.txt @@ -61,6 +61,7 @@ Currently, these files are in /proc/sys/vm: - swappiness - user_reserve_kbytes - vfs_cache_pressure +- watermark_scale_factor - zone_reclaim_mode ============================================================== diff --git a/Documentation/thermal/intel_powerclamp.txt b/Documentation/thermal/intel_powerclamp.txt index 332de4a39b5a..60073dc9f748 100644 --- a/Documentation/thermal/intel_powerclamp.txt +++ b/Documentation/thermal/intel_powerclamp.txt @@ -121,7 +121,7 @@ time is considered largely as a non-causal system where its behavior cannot be based on the past or current input. Therefore, the intel_powerclamp driver attempts to enforce the desired idle time instantly as given input (target idle ratio). After injection, -powerclamp moniors the actual idle for a given time window and adjust +powerclamp monitors the actual idle for a given time window and adjust the next injection accordingly to avoid over/under correction. When used in a causal control system, such as a temperature control, diff --git a/Documentation/usb/gadget_multi.txt b/Documentation/usb/gadget_multi.txt index 5faf514047e9..b3146dd7aa43 100644 --- a/Documentation/usb/gadget_multi.txt +++ b/Documentation/usb/gadget_multi.txt @@ -36,7 +36,7 @@ configuration with CDC ECM which should work better under Linux. ** Windows host drivers -For the gadget two work under Windows two conditions have to be met: +For the gadget to work under Windows two conditions have to be met: *** Detecting as composite gadget diff --git a/Documentation/video4linux/CARDLIST.cx23885 b/Documentation/video4linux/CARDLIST.cx23885 index 85a8fdcfcdaa..c9b4959fd04e 100644 --- a/Documentation/video4linux/CARDLIST.cx23885 +++ b/Documentation/video4linux/CARDLIST.cx23885 @@ -54,3 +54,4 @@ 53 -> Hauppauge WinTV Starburst [0070:c12a] 54 -> ViewCast 260e [1576:0260] 55 -> ViewCast 460e [1576:0460] + 56 -> Hauppauge WinTV-quadHD (DVB) [0070:6a28,0070:6b28] diff --git a/Documentation/video4linux/v4l2-controls.txt b/Documentation/video4linux/v4l2-controls.txt index 5e759cab4538..f930b80e9111 100644 --- a/Documentation/video4linux/v4l2-controls.txt +++ b/Documentation/video4linux/v4l2-controls.txt @@ -96,21 +96,6 @@ Basic usage for V4L2 and sub-device drivers Where foo->sd is of type struct v4l2_subdev. - And set all core control ops in your struct v4l2_subdev_core_ops to these - helpers: - - .queryctrl = v4l2_subdev_queryctrl, - .querymenu = v4l2_subdev_querymenu, - .g_ctrl = v4l2_subdev_g_ctrl, - .s_ctrl = v4l2_subdev_s_ctrl, - .g_ext_ctrls = v4l2_subdev_g_ext_ctrls, - .try_ext_ctrls = v4l2_subdev_try_ext_ctrls, - .s_ext_ctrls = v4l2_subdev_s_ext_ctrls, - - Note: this is a temporary solution only. Once all V4L2 drivers that depend - on subdev drivers are converted to the control framework these helpers will - no longer be needed. - 1.4) Clean up the handler at the end: v4l2_ctrl_handler_free(&foo->ctrl_handler); diff --git a/Documentation/video4linux/vivid.txt b/Documentation/video4linux/vivid.txt index 8da5d2a576bc..1b26519c6ddc 100644 --- a/Documentation/video4linux/vivid.txt +++ b/Documentation/video4linux/vivid.txt @@ -74,7 +74,8 @@ Section 11: Cropping, Composing, Scaling Section 12: Formats Section 13: Capture Overlay Section 14: Output Overlay -Section 15: Some Future Improvements +Section 15: CEC (Consumer Electronics Control) +Section 16: Some Future Improvements Section 1: Configuring the driver @@ -364,7 +365,11 @@ For HDMI inputs it is possible to set the EDID. By default a simple EDID is provided. You can only set the EDID for HDMI inputs. Internally, however, the EDID is shared between all HDMI inputs. -No interpretation is done of the EDID data. +No interpretation is done of the EDID data with the exception of the +physical address. See the CEC section for more details. + +There is a maximum of 15 HDMI inputs (if there are more, then they will be +reduced to 15) since that's the limitation of the EDID physical address. Section 3: Video Output @@ -409,6 +414,9 @@ standard, and for all others a 1:1 pixel aspect ratio is returned. An HDMI output has a valid EDID which can be obtained through VIDIOC_G_EDID. +There is a maximum of 15 HDMI outputs (if there are more, then they will be +reduced to 15) since that's the limitation of the EDID physical address. See +also the CEC section for more details. Section 4: VBI Capture ---------------------- @@ -1108,7 +1116,26 @@ capabilities will slow down the video loop considerably as a lot of checks have to be done per pixel. -Section 15: Some Future Improvements +Section 15: CEC (Consumer Electronics Control) +---------------------------------------------- + +If there are HDMI inputs then a CEC adapter will be created that has +the same number of input ports. This is the equivalent of e.g. a TV that +has that number of inputs. Each HDMI output will also create a +CEC adapter that is hooked up to the corresponding input port, or (if there +are more outputs than inputs) is not hooked up at all. In other words, +this is the equivalent of hooking up each output device to an input port of +the TV. Any remaining output devices remain unconnected. + +The EDID that each output reads reports a unique CEC physical address that is +based on the physical address of the EDID of the input. So if the EDID of the +receiver has physical address A.B.0.0, then each output will see an EDID +containing physical address A.B.C.0 where C is 1 to the number of inputs. If +there are more outputs than inputs then the remaining outputs have a CEC adapter +that is disabled and reports an invalid physical address. + + +Section 16: Some Future Improvements ------------------------------------ Just as a reminder and in no particular order: @@ -1121,8 +1148,6 @@ Just as a reminder and in no particular order: - Fix sequence/field numbering when looping of video with alternate fields - Add support for V4L2_CID_BG_COLOR for video outputs - Add ARGB888 overlay support: better testing of the alpha channel -- Add custom DV timings support -- Add support for V4L2_DV_FL_REDUCED_FPS - Improve pixel aspect support in the tpg code by passing a real v4l2_fract - Use per-queue locks and/or per-device locks to improve throughput - Add support to loop from a specific output to a specific input across @@ -1133,3 +1158,4 @@ Just as a reminder and in no particular order: - Make a thread for the RDS generation, that would help in particular for the "Controls" RDS Rx I/O Mode as the read-only RDS controls could be updated in real-time. +- Changing the EDID should cause hotplug detect emulation to happen. diff --git a/Documentation/vm/page_migration b/Documentation/vm/page_migration index fea5c0864170..94bd9c11c4e0 100644 --- a/Documentation/vm/page_migration +++ b/Documentation/vm/page_migration @@ -142,5 +142,111 @@ Steps: 20. The new page is moved to the LRU and can be scanned by the swapper etc again. -Christoph Lameter, May 8, 2006. +C. Non-LRU page migration +------------------------- + +Although original migration aimed for reducing the latency of memory access +for NUMA, compaction who want to create high-order page is also main customer. + +Current problem of the implementation is that it is designed to migrate only +*LRU* pages. However, there are potential non-lru pages which can be migrated +in drivers, for example, zsmalloc, virtio-balloon pages. + +For virtio-balloon pages, some parts of migration code path have been hooked +up and added virtio-balloon specific functions to intercept migration logics. +It's too specific to a driver so other drivers who want to make their pages +movable would have to add own specific hooks in migration path. + +To overclome the problem, VM supports non-LRU page migration which provides +generic functions for non-LRU movable pages without driver specific hooks +migration path. + +If a driver want to make own pages movable, it should define three functions +which are function pointers of struct address_space_operations. + +1. bool (*isolate_page) (struct page *page, isolate_mode_t mode); + +What VM expects on isolate_page function of driver is to return *true* +if driver isolates page successfully. On returing true, VM marks the page +as PG_isolated so concurrent isolation in several CPUs skip the page +for isolation. If a driver cannot isolate the page, it should return *false*. + +Once page is successfully isolated, VM uses page.lru fields so driver +shouldn't expect to preserve values in that fields. + +2. int (*migratepage) (struct address_space *mapping, + struct page *newpage, struct page *oldpage, enum migrate_mode); + +After isolation, VM calls migratepage of driver with isolated page. +The function of migratepage is to move content of the old page to new page +and set up fields of struct page newpage. Keep in mind that you should +indicate to the VM the oldpage is no longer movable via __ClearPageMovable() +under page_lock if you migrated the oldpage successfully and returns +MIGRATEPAGE_SUCCESS. If driver cannot migrate the page at the moment, driver +can return -EAGAIN. On -EAGAIN, VM will retry page migration in a short time +because VM interprets -EAGAIN as "temporal migration failure". On returning +any error except -EAGAIN, VM will give up the page migration without retrying +in this time. + +Driver shouldn't touch page.lru field VM using in the functions. + +3. void (*putback_page)(struct page *); + +If migration fails on isolated page, VM should return the isolated page +to the driver so VM calls driver's putback_page with migration failed page. +In this function, driver should put the isolated page back to the own data +structure. +4. non-lru movable page flags + +There are two page flags for supporting non-lru movable page. + +* PG_movable + +Driver should use the below function to make page movable under page_lock. + + void __SetPageMovable(struct page *page, struct address_space *mapping) + +It needs argument of address_space for registering migration family functions +which will be called by VM. Exactly speaking, PG_movable is not a real flag of +struct page. Rather than, VM reuses page->mapping's lower bits to represent it. + + #define PAGE_MAPPING_MOVABLE 0x2 + page->mapping = page->mapping | PAGE_MAPPING_MOVABLE; + +so driver shouldn't access page->mapping directly. Instead, driver should +use page_mapping which mask off the low two bits of page->mapping under +page lock so it can get right struct address_space. + +For testing of non-lru movable page, VM supports __PageMovable function. +However, it doesn't guarantee to identify non-lru movable page because +page->mapping field is unified with other variables in struct page. +As well, if driver releases the page after isolation by VM, page->mapping +doesn't have stable value although it has PAGE_MAPPING_MOVABLE +(Look at __ClearPageMovable). But __PageMovable is cheap to catch whether +page is LRU or non-lru movable once the page has been isolated. Because +LRU pages never can have PAGE_MAPPING_MOVABLE in page->mapping. It is also +good for just peeking to test non-lru movable pages before more expensive +checking with lock_page in pfn scanning to select victim. + +For guaranteeing non-lru movable page, VM provides PageMovable function. +Unlike __PageMovable, PageMovable functions validates page->mapping and +mapping->a_ops->isolate_page under lock_page. The lock_page prevents sudden +destroying of page->mapping. + +Driver using __SetPageMovable should clear the flag via __ClearMovablePage +under page_lock before the releasing the page. + +* PG_isolated + +To prevent concurrent isolation among several CPUs, VM marks isolated page +as PG_isolated under lock_page. So if a CPU encounters PG_isolated non-lru +movable page, it can skip it. Driver doesn't need to manipulate the flag +because VM will set/clear it automatically. Keep in mind that if driver +sees PG_isolated page, it means the page have been isolated by VM so it +shouldn't touch page.lru field. +PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag +for own purpose. + +Christoph Lameter, May 8, 2006. +Minchan Kim, Mar 28, 2016. diff --git a/Documentation/vm/transhuge.txt b/Documentation/vm/transhuge.txt index 7c871d6beb63..2ec6adb5a4ce 100644 --- a/Documentation/vm/transhuge.txt +++ b/Documentation/vm/transhuge.txt @@ -9,8 +9,8 @@ using huge pages for the backing of virtual memory with huge pages that supports the automatic promotion and demotion of page sizes and without the shortcomings of hugetlbfs. -Currently it only works for anonymous memory mappings but in the -future it can expand over the pagecache layer starting with tmpfs. +Currently it only works for anonymous memory mappings and tmpfs/shmem. +But in the future it can expand to other filesystems. The reason applications are running faster is because of two factors. The first factor is almost completely irrelevant and it's not @@ -57,10 +57,6 @@ miss is going to run faster. feature that applies to all dynamic high order allocations in the kernel) -- this initial support only offers the feature in the anonymous memory - regions but it'd be ideal to move it to tmpfs and the pagecache - later - Transparent Hugepage Support maximizes the usefulness of free memory if compared to the reservation approach of hugetlbfs by allowing all unused memory to be used as cache or other movable (or even unmovable @@ -94,21 +90,21 @@ madvise(MADV_HUGEPAGE) on their critical mmapped regions. == sysfs == -Transparent Hugepage Support can be entirely disabled (mostly for -debugging purposes) or only enabled inside MADV_HUGEPAGE regions (to -avoid the risk of consuming more memory resources) or enabled system -wide. This can be achieved with one of: +Transparent Hugepage Support for anonymous memory can be entirely disabled +(mostly for debugging purposes) or only enabled inside MADV_HUGEPAGE +regions (to avoid the risk of consuming more memory resources) or enabled +system wide. This can be achieved with one of: echo always >/sys/kernel/mm/transparent_hugepage/enabled echo madvise >/sys/kernel/mm/transparent_hugepage/enabled echo never >/sys/kernel/mm/transparent_hugepage/enabled It's also possible to limit defrag efforts in the VM to generate -hugepages in case they're not immediately free to madvise regions or -to never try to defrag memory and simply fallback to regular pages -unless hugepages are immediately available. Clearly if we spend CPU -time to defrag memory, we would expect to gain even more by the fact -we use hugepages later instead of regular pages. This isn't always +anonymous hugepages in case they're not immediately free to madvise +regions or to never try to defrag memory and simply fallback to regular +pages unless hugepages are immediately available. Clearly if we spend CPU +time to defrag memory, we would expect to gain even more by the fact we +use hugepages later instead of regular pages. This isn't always guaranteed, but it may be more likely in case the allocation is for a MADV_HUGEPAGE region. @@ -133,9 +129,9 @@ that are have used madvise(MADV_HUGEPAGE). This is the default behaviour. "never" should be self-explanatory. -By default kernel tries to use huge zero page on read page fault. -It's possible to disable huge zero page by writing 0 or enable it -back by writing 1: +By default kernel tries to use huge zero page on read page fault to +anonymous mapping. It's possible to disable huge zero page by writing 0 +or enable it back by writing 1: echo 0 >/sys/kernel/mm/transparent_hugepage/use_zero_page echo 1 >/sys/kernel/mm/transparent_hugepage/use_zero_page @@ -204,21 +200,67 @@ Support by passing the parameter "transparent_hugepage=always" or "transparent_hugepage=madvise" or "transparent_hugepage=never" (without "") to the kernel command line. +== Hugepages in tmpfs/shmem == + +You can control hugepage allocation policy in tmpfs with mount option +"huge=". It can have following values: + + - "always": + Attempt to allocate huge pages every time we need a new page; + + - "never": + Do not allocate huge pages; + + - "within_size": + Only allocate huge page if it will be fully within i_size. + Also respect fadvise()/madvise() hints; + + - "advise: + Only allocate huge pages if requested with fadvise()/madvise(); + +The default policy is "never". + +"mount -o remount,huge= /mountpoint" works fine after mount: remounting +huge=never will not attempt to break up huge pages at all, just stop more +from being allocated. + +There's also sysfs knob to control hugepage allocation policy for internal +shmem mount: /sys/kernel/mm/transparent_hugepage/shmem_enabled. The mount +is used for SysV SHM, memfds, shared anonymous mmaps (of /dev/zero or +MAP_ANONYMOUS), GPU drivers' DRM objects, Ashmem. + +In addition to policies listed above, shmem_enabled allows two further +values: + + - "deny": + For use in emergencies, to force the huge option off from + all mounts; + - "force": + Force the huge option on for all - very useful for testing; + == Need of application restart == -The transparent_hugepage/enabled values only affect future -behavior. So to make them effective you need to restart any -application that could have been using hugepages. This also applies to -the regions registered in khugepaged. +The transparent_hugepage/enabled values and tmpfs mount option only affect +future behavior. So to make them effective you need to restart any +application that could have been using hugepages. This also applies to the +regions registered in khugepaged. == Monitoring usage == -The number of transparent huge pages currently used by the system is -available by reading the AnonHugePages field in /proc/meminfo. To -identify what applications are using transparent huge pages, it is -necessary to read /proc/PID/smaps and count the AnonHugePages fields -for each mapping. Note that reading the smaps file is expensive and -reading it frequently will incur overhead. +The number of anonymous transparent huge pages currently used by the +system is available by reading the AnonHugePages field in /proc/meminfo. +To identify what applications are using anonymous transparent huge pages, +it is necessary to read /proc/PID/smaps and count the AnonHugePages fields +for each mapping. + +The number of file transparent huge pages mapped to userspace is available +by reading ShmemPmdMapped and ShmemHugePages fields in /proc/meminfo. +To identify what applications are mapping file transparent huge pages, it +is necessary to read /proc/PID/smaps and count the FileHugeMapped fields +for each mapping. + +Note that reading the smaps file is expensive and reading it +frequently will incur overhead. There are a number of counters in /proc/vmstat that may be used to monitor how successfully the system is providing huge pages for use. @@ -238,6 +280,12 @@ thp_collapse_alloc_failed is incremented if khugepaged found a range of pages that should be collapsed into one huge page but failed the allocation. +thp_file_alloc is incremented every time a file huge page is successfully +i allocated. + +thp_file_mapped is incremented every time a file huge page is mapped into + user address space. + thp_split_page is incremented every time a huge page is split into base pages. This can happen for a variety of reasons but a common reason is that a huge page is old and is being reclaimed. @@ -403,19 +451,27 @@ pages: on relevant sub-page of the compound page. - map/unmap of the whole compound page accounted in compound_mapcount - (stored in first tail page). + (stored in first tail page). For file huge pages, we also increment + ->_mapcount of all sub-pages in order to have race-free detection of + last unmap of subpages. -PageDoubleMap() indicates that ->_mapcount in all subpages is offset up by one. -This additional reference is required to get race-free detection of unmap of -subpages when we have them mapped with both PMDs and PTEs. +PageDoubleMap() indicates that the page is *possibly* mapped with PTEs. + +For anonymous pages PageDoubleMap() also indicates ->_mapcount in all +subpages is offset up by one. This additional reference is required to +get race-free detection of unmap of subpages when we have them mapped with +both PMDs and PTEs. This is optimization required to lower overhead of per-subpage mapcount tracking. The alternative is alter ->_mapcount in all subpages on each map/unmap of the whole compound page. -We set PG_double_map when a PMD of the page got split for the first time, -but still have PMD mapping. The additional references go away with last -compound_mapcount. +For anonymous pages, we set PG_double_map when a PMD of the page got split +for the first time, but still have PMD mapping. The additional references +go away with last compound_mapcount. + +File pages get PG_double_map set on first map of the page with PTE and +goes away when the page gets evicted from page cache. split_huge_page internally has to distribute the refcounts in the head page to the tail pages before clearing all PG_head/tail bits from the page @@ -427,7 +483,7 @@ sum of mapcount of all sub-pages plus one (split_huge_page caller must have reference for head page). split_huge_page uses migration entries to stabilize page->_refcount and -page->_mapcount. +page->_mapcount of anonymous pages. File pages just got unmapped. We safe against physical memory scanners too: the only legitimate way scanner can get reference to a page is get_page_unless_zero(). diff --git a/Documentation/vm/unevictable-lru.txt b/Documentation/vm/unevictable-lru.txt index fa3b527086fa..0026a8d33fc0 100644 --- a/Documentation/vm/unevictable-lru.txt +++ b/Documentation/vm/unevictable-lru.txt @@ -461,6 +461,27 @@ unevictable LRU is enabled, the work of compaction is mostly handled by the page migration code and the same work flow as described in MIGRATING MLOCKED PAGES will apply. +MLOCKING TRANSPARENT HUGE PAGES +------------------------------- + +A transparent huge page is represented by a single entry on an LRU list. +Therefore, we can only make unevictable an entire compound page, not +individual subpages. + +If a user tries to mlock() part of a huge page, we want the rest of the +page to be reclaimable. + +We cannot just split the page on partial mlock() as split_huge_page() can +fail and new intermittent failure mode for the syscall is undesirable. + +We handle this by keeping PTE-mapped huge pages on normal LRU lists: the +PMD on border of VM_LOCKED VMA will be split into PTE table. + +This way the huge page is accessible for vmscan. Under memory pressure the +page will be split, subpages which belong to VM_LOCKED VMAs will be moved +to unevictable LRU and the rest can be reclaimed. + +See also comment in follow_trans_huge_pmd(). mmap(MAP_LOCKED) SYSTEM CALL HANDLING ------------------------------------- diff --git a/Documentation/workqueue.txt b/Documentation/workqueue.txt index 5e0e05c5183e..c49e3178178d 100644 --- a/Documentation/workqueue.txt +++ b/Documentation/workqueue.txt @@ -169,7 +169,7 @@ resources, scheduled and executed. WQ_UNBOUND Work items queued to an unbound wq are served by the special - woker-pools which host workers which are not bound to any + worker-pools which host workers which are not bound to any specific CPU. This makes the wq behave as a simple execution context provider without concurrency management. The unbound worker-pools try to start execution of work items as soon as diff --git a/Documentation/x86/intel_mpx.txt b/Documentation/x86/intel_mpx.txt index 1a5a12184a35..85d0549ad846 100644 --- a/Documentation/x86/intel_mpx.txt +++ b/Documentation/x86/intel_mpx.txt @@ -45,7 +45,7 @@ is how we expect the compiler, application and kernel to work together. MPX-instrumented. 3) The kernel detects that the CPU has MPX, allows the new prctl() to succeed, and notes the location of the bounds directory. Userspace is - expected to keep the bounds directory at that locationWe note it + expected to keep the bounds directory at that location. We note it instead of reading it each time because the 'xsave' operation needed to access the bounds directory register is an expensive operation. 4) If the application needs to spill bounds out of the 4 registers, it @@ -167,7 +167,7 @@ If a #BR is generated due to a bounds violation caused by MPX. We need to decode MPX instructions to get violation address and set this address into extended struct siginfo. -The _sigfault feild of struct siginfo is extended as follow: +The _sigfault field of struct siginfo is extended as follow: 87 /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ 88 struct { @@ -240,5 +240,5 @@ them at the same bounds table. This is allowed architecturally. See more information "Intel(R) Architecture Instruction Set Extensions Programming Reference" (9.3.4). -However, if users did this, the kernel might be fooled in to unmaping an +However, if users did this, the kernel might be fooled in to unmapping an in-use bounds table since it does not recognize sharing. diff --git a/Documentation/x86/tlb.txt b/Documentation/x86/tlb.txt index 39d172326703..6a0607b99ed8 100644 --- a/Documentation/x86/tlb.txt +++ b/Documentation/x86/tlb.txt @@ -5,7 +5,7 @@ memory, it has two choices: from areas other than the one we are trying to flush will be destroyed and must be refilled later, at some cost. 2. Use the invlpg instruction to invalidate a single page at a - time. This could potentialy cost many more instructions, but + time. This could potentially cost many more instructions, but it is a much more precise operation, causing no collateral damage to other TLB entries. @@ -19,7 +19,7 @@ Which method to do depends on a few things: work. 3. The size of the TLB. The larger the TLB, the more collateral damage we do with a full flush. So, the larger the TLB, the - more attrative an individual flush looks. Data and + more attractive an individual flush looks. Data and instructions have separate TLBs, as do different page sizes. 4. The microarchitecture. The TLB has become a multi-level cache on modern CPUs, and the global flushes have become more diff --git a/Documentation/x86/x86_64/machinecheck b/Documentation/x86/x86_64/machinecheck index b1fb30273286..d0648a74fceb 100644 --- a/Documentation/x86/x86_64/machinecheck +++ b/Documentation/x86/x86_64/machinecheck @@ -36,7 +36,7 @@ between all CPUs. check_interval How often to poll for corrected machine check errors, in seconds - (Note output is hexademical). Default 5 minutes. When the poller + (Note output is hexadecimal). Default 5 minutes. When the poller finds MCEs it triggers an exponential speedup (poll more often) on the polling interval. When the poller stops finding MCEs, it triggers an exponential backoff (poll less often) on the polling diff --git a/Documentation/x86/x86_64/mm.txt b/Documentation/x86/x86_64/mm.txt index 5aa738346062..8c7dd5957ae1 100644 --- a/Documentation/x86/x86_64/mm.txt +++ b/Documentation/x86/x86_64/mm.txt @@ -39,4 +39,8 @@ memory window (this size is arbitrary, it can be raised later if needed). The mappings are not part of any other kernel PGD and are only available during EFI runtime calls. +Note that if CONFIG_RANDOMIZE_MEMORY is enabled, the direct mapping of all +physical memory, vmalloc/ioremap space and virtual memory map are randomized. +Their order is preserved but their base will be offset early at boot time. + -Andi Kleen, Jul 2004 diff --git a/Documentation/zh_CN/CodingStyle b/Documentation/zh_CN/CodingStyle index 654afd72eb24..12717791baac 100644 --- a/Documentation/zh_CN/CodingStyle +++ b/Documentation/zh_CN/CodingStyle @@ -24,34 +24,33 @@ Documentation/CodingStyle的中文翻译 Linux内核代码风格 -这是一个简短的文档，描述了linux内核的首选代码风格。代码风格是因人而异的，而且我 -不愿意把我的观点强加给任何人，不过这里所讲述的是我必须要维护的代码所遵守的风格， -并且我也希望绝大多数其他代码也能遵守这个风格。请在写代码时至少考虑一下本文所述的 -风格。 +这是一个简短的文档，描述了 linux 内核的首选代码风格。代码风格是因人而异的，而且我 +不愿意把自己的观点强加给任何人，但这就像我去做任何事情都必须遵循的原则那样，我也 +希望在绝大多数事上保持这种的态度。请（在写代码时）至少考虑一下这里的代码风格。 -首先，我建议你打印一份GNU代码规范，然后不要读它。烧了它，这是一个具有重大象征性 -意义的动作。 +首先，我建议你打印一份 GNU 代码规范，然后不要读。烧了它，这是一个具有重大象征性意义 +的动作。 不管怎样，现在我们开始： - 第一章：缩进 + 第一章：缩进 -制表符是8个字符，所以缩进也是8个字符。有些异端运动试图将缩进变为4（乃至2）个字符 -深，这几乎相当于尝试将圆周率的值定义为3。 +制表符是 8 个字符，所以缩进也是 8 个字符。有些异端运动试图将缩进变为 4（甚至 2！） +个字符深，这几乎相当于尝试将圆周率的值定义为 3。 理由：缩进的全部意义就在于清楚的定义一个控制块起止于何处。尤其是当你盯着你的屏幕 -连续看了20小时之后，你将会发现大一点的缩进会使你更容易分辨缩进。 +连续看了 20 小时之后，你将会发现大一点的缩进会使你更容易分辨缩进。 -现在，有些人会抱怨8个字符的缩进会使代码向右边移动的太远，在80个字符的终端屏幕上 -就很难读这样的代码。这个问题的答案是，如果你需要3级以上的缩进，不管用何种方式你 +现在，有些人会抱怨 8 个字符的缩进会使代码向右边移动的太远，在 80 个字符的终端屏幕上 +就很难读这样的代码。这个问题的答案是，如果你需要 3 级以上的缩进，不管用何种方式你 的代码已经有问题了，应该修正你的程序。 -简而言之，8个字符的缩进可以让代码更容易阅读，还有一个好处是当你的函数嵌套太深的 +简而言之，8 个字符的缩进可以让代码更容易阅读，还有一个好处是当你的函数嵌套太深的 时候可以给你警告。留心这个警告。 -在switch语句中消除多级缩进的首选的方式是让“switch”和从属于它的“case”标签对齐于同 -一列，而不要“两次缩进”“case”标签。比如： +在 switch 语句中消除多级缩进的首选的方式是让 “switch” 和从属于它的 “case” 标签 +对齐于同一列，而不要 “两次缩进” “case” 标签。比如： switch (suffix) { case 'G': @@ -70,7 +69,6 @@ Documentation/CodingStyle的中文翻译 break; } - 不要把多个语句放在一行里，除非你有什么东西要隐藏： if (condition) do_this; @@ -79,7 +77,7 @@ Documentation/CodingStyle的中文翻译 也不要在一行里放多个赋值语句。内核代码风格超级简单。就是避免可能导致别人误读的表 达式。 -除了注释、文档和Kconfig之外，不要使用空格来缩进，前面的例子是例外，是有意为之。 +除了注释、文档和 Kconfig 之外，不要使用空格来缩进，前面的例子是例外，是有意为之。 选用一个好的编辑器，不要在行尾留空格。 @@ -88,27 +86,18 @@ Documentation/CodingStyle的中文翻译 代码风格的意义就在于使用平常使用的工具来维持代码的可读性和可维护性。 -每一行的长度的限制是80列，我们强烈建议您遵守这个惯例。 +每一行的长度的限制是 80 列，我们强烈建议您遵守这个惯例。 -长于80列的语句要打散成有意义的片段。每个片段要明显短于原来的语句，而且放置的位置 -也明显的靠右。同样的规则也适用于有很长参数列表的函数头。长字符串也要打散成较短的 -字符串。唯一的例外是超过80列可以大幅度提高可读性并且不会隐藏信息的情况。 - -void fun(int a, int b, int c) -{ - if (condition) - printk(KERN_WARNING "Warning this is a long printk with " - "3 parameters a: %u b: %u " - "c: %u \n", a, b, c); - else - next_statement; -} +长于 80 列的语句要打散成有意义的片段。除非超过 80 列能显著增加可读性，并且不会隐藏 +信息。子片段要明显短于母片段，并明显靠右。这同样适用于有着很长参数列表的函数头。 +然而，绝对不要打散对用户可见的字符串，例如 printk 信息，因为这将导致无法 grep 这些 +信息。 第三章：大括号和空格的放置 C语言风格中另外一个常见问题是大括号的放置。和缩进大小不同，选择或弃用某种放置策 -略并没有多少技术上的原因，不过首选的方式，就像Kernighan和Ritchie展示给我们的，是 -把起始大括号放在行尾，而把结束大括号放在行首，所以： +略并没有多少技术上的原因，不过首选的方式，就像 Kernighan 和 Ritchie 展示给我们的， +是把起始大括号放在行尾，而把结束大括号放在行首，所以： if (x is true) { we do y @@ -134,12 +123,12 @@ C语言风格中另外一个常见问题是大括号的放置。和缩进大小 body of function } -全世界的异端可能会抱怨这个不一致性是……呃……不一致的，不过所有思维健全的人都知道（ -a）K&R是_正确的_，并且（b）K&R是正确的。此外，不管怎样函数都是特殊的（在C语言中 -，函数是不能嵌套的）。 +全世界的异端可能会抱怨这个不一致性是……呃……不一致的，不过所有思维健全的人都知道 +(a) K&R 是 _正确的_，并且 (b) K&R 是正确的。此外，不管怎样函数都是特殊的（C +函数是不能嵌套的）。 -注意结束大括号独自占据一行，除非它后面跟着同一个语句的剩余部分，也就是do语句中的 -“while”或者if语句中的“else”，像这样： +注意结束大括号独自占据一行，除非它后面跟着同一个语句的剩余部分，也就是 do 语句中的 +“while” 或者 if 语句中的 “else”，像这样： do { body of do-loop @@ -158,41 +147,50 @@ a）K&R是_正确的_，并且（b）K&R是正确的。此外，不管怎样函 理由：K&R。 也请注意这种大括号的放置方式也能使空（或者差不多空的）行的数量最小化，同时不失可 -读性。因此，由于你的屏幕上的新行是不可再生资源（想想25行的终端屏幕），你将会有更 +读性。因此，由于你的屏幕上的新行是不可再生资源（想想 25 行的终端屏幕），你将会有更 多的空行来放置注释。 当只有一个单独的语句的时候，不用加不必要的大括号。 -if (condition) - action(); + if (condition) + action(); + +和 + + if (condition) + do_this(); + else + do_that(); -这点不适用于本身为某个条件语句的一个分支的单独语句。这时需要在两个分支里都使用大 -括号。 +这并不适用于只有一个条件分支是单语句的情况；这时所有分支都要使用大括号： -if (condition) { - do_this(); - do_that(); -} else { - otherwise(); -} + if (condition) { + do_this(); + do_that(); + } else { + otherwise(); + } 3.1：空格 -Linux内核的空格使用方式（主要）取决于它是用于函数还是关键字。（大多数）关键字后 -要加一个空格。值得注意的例外是sizeof、typeof、alignof和__attribute__，这些关键字 -某些程度上看起来更像函数（它们在Linux里也常常伴随小括号而使用，尽管在C语言里这样 -的小括号不是必需的，就像“struct fileinfo info”声明过后的“sizeof info”）。 +Linux 内核的空格使用方式（主要）取决于它是用于函数还是关键字。（大多数）关键字后 +要加一个空格。值得注意的例外是 sizeof、typeof、alignof 和 __attribute__，这些 +关键字某些程度上看起来更像函数（它们在 Linux 里也常常伴随小括号而使用，尽管在 C 里 +这样的小括号不是必需的，就像 “struct fileinfo info” 声明过后的 “sizeof info”）。 所以在这些关键字之后放一个空格： + if, switch, case, for, do, while -但是不要在sizeof、typeof、alignof或者__attribute__这些关键字之后放空格。例如， + +但是不要在 sizeof、typeof、alignof 或者 __attribute__ 这些关键字之后放空格。例如， + s = sizeof(struct file); 不要在小括号里的表达式两侧加空格。这是一个反例： s = sizeof( struct file ); -当声明指针类型或者返回指针类型的函数时，“*”的首选使用方式是使之靠近变量名或者函 +当声明指针类型或者返回指针类型的函数时，“*” 的首选使用方式是使之靠近变量名或者函 数名，而不是靠近类型名。例子： char *linux_banner; @@ -204,15 +202,18 @@ Linux内核的空格使用方式（主要）取决于它是用于函数还是关 = + - < > * / % | & ^ <= >= == != ? : 但是一元操作符后不要加空格： + & * + - ~ ! sizeof typeof alignof __attribute__ defined 后缀自加和自减一元操作符前不加空格： + ++ -- 前缀自加和自减一元操作符后不加空格： + ++ -- -“.”和“->”结构体成员操作符前后不加空格。 +‘.’ 和 “->” 结构体成员操作符前后不加空格。 不要在行尾留空白。有些可以自动缩进的编辑器会在新行的行首加入适量的空白，然后你 就可以直接在那一行输入代码。不过假如你最后没有在那一行输入代码，有些编辑器就不 @@ -225,23 +226,23 @@ Linux内核的空格使用方式（主要）取决于它是用于函数还是关 第四章：命名 -C是一个简朴的语言，你的命名也应该这样。和Modula-2和Pascal程序员不同，C程序员不使 -用类似ThisVariableIsATemporaryCounter这样华丽的名字。C程序员会称那个变量为“tmp” -，这样写起来会更容易，而且至少不会令其难于理解。 +C是一个简朴的语言，你的命名也应该这样。和 Modula-2 和 Pascal 程序员不同，C 程序员 +不使用类似 ThisVariableIsATemporaryCounter 这样华丽的名字。C 程序员会称那个变量 +为 “tmp”，这样写起来会更容易，而且至少不会令其难于理解。 不过，虽然混用大小写的名字是不提倡使用的，但是全局变量还是需要一个具描述性的名字 -。称一个全局函数为“foo”是一个难以饶恕的错误。 +。称一个全局函数为 “foo” 是一个难以饶恕的错误。 全局变量（只有当你真正需要它们的时候再用它）需要有一个具描述性的名字，就像全局函 -数。如果你有一个可以计算活动用户数量的函数，你应该叫它“count_active_users()”或者 -类似的名字，你不应该叫它“cntuser()”。 +数。如果你有一个可以计算活动用户数量的函数，你应该叫它 “count_active_users()” +或者类似的名字，你不应该叫它 “cntuser()”。 在函数名中包含函数类型（所谓的匈牙利命名法）是脑子出了问题——编译器知道那些类型而 且能够检查那些类型，这样做只能把程序员弄糊涂了。难怪微软总是制造出有问题的程序。 本地变量名应该简短，而且能够表达相关的含义。如果你有一些随机的整数型的循环计数器 -，它应该被称为“i”。叫它“loop_counter”并无益处，如果它没有被误解的可能的话。类似 -的，“tmp”可以用来称呼任意类型的临时变量。 +，它应该被称为 “i”。叫它 “loop_counter” 并无益处，如果它没有被误解的可能的话。 +类似的，“tmp” 可以用来称呼任意类型的临时变量。 如果你怕混淆了你的本地变量名，你就遇到另一个问题了，叫做函数增长荷尔蒙失衡综合症 。请看第六章（函数）。 @@ -249,9 +250,9 @@ C是一个简朴的语言，你的命名也应该这样。和Modula-2和Pascal 第五章：Typedef -不要使用类似“vps_t”之类的东西。 +不要使用类似 “vps_t” 之类的东西。 -对结构体和指针使用typedef是一个错误。当你在代码里看到： +对结构体和指针使用 typedef 是一个错误。当你在代码里看到： vps_t a; @@ -261,91 +262,91 @@ C是一个简朴的语言，你的命名也应该这样。和Modula-2和Pascal struct virtual_container *a; -你就知道“a”是什么了。 +你就知道 “a” 是什么了。 -很多人认为typedef“能提高可读性”。实际不是这样的。它们只在下列情况下有用： +很多人认为 typedef “能提高可读性”。实际不是这样的。它们只在下列情况下有用： - (a) 完全不透明的对象（这种情况下要主动使用typedef来隐藏这个对象实际上是什么）。 + (a) 完全不透明的对象（这种情况下要主动使用 typedef 来隐藏这个对象实际上是什么）。 - 例如：“pte_t”等不透明对象，你只能用合适的访问函数来访问它们。 + 例如：“pte_t” 等不透明对象，你只能用合适的访问函数来访问它们。 - 注意！不透明性和“访问函数”本身是不好的。我们使用pte_t等类型的原因在于真的是 + 注意！不透明性和“访问函数”本身是不好的。我们使用 pte_t 等类型的原因在于真的是 完全没有任何共用的可访问信息。 - (b) 清楚的整数类型，如此，这层抽象就可以帮助消除到底是“int”还是“long”的混淆。 + (b) 清楚的整数类型，如此，这层抽象就可以帮助消除到底是 “int” 还是 “long” 的混淆。 - u8/u16/u32是完全没有问题的typedef，不过它们更符合类别(d)而不是这里。 + u8/u16/u32 是完全没有问题的 typedef，不过它们更符合类别 (d) 而不是这里。 - 再次注意！要这样做，必须事出有因。如果某个变量是“unsigned long“，那么没有必要 + 再次注意！要这样做，必须事出有因。如果某个变量是 “unsigned long“，那么没有必要 typedef unsigned long myflags_t; - 不过如果有一个明确的原因，比如它在某种情况下可能会是一个“unsigned int”而在 - 其他情况下可能为“unsigned long”，那么就不要犹豫，请务必使用typedef。 + 不过如果有一个明确的原因，比如它在某种情况下可能会是一个 “unsigned int” 而在 + 其他情况下可能为 “unsigned long”，那么就不要犹豫，请务必使用 typedef。 (c) 当你使用sparse按字面的创建一个新类型来做类型检查的时候。 (d) 和标准C99类型相同的类型，在某些例外的情况下。 - 虽然让眼睛和脑筋来适应新的标准类型比如“uint32_t”不需要花很多时间，可是有些 + 虽然让眼睛和脑筋来适应新的标准类型比如 “uint32_t” 不需要花很多时间，可是有些 人仍然拒绝使用它们。 - 因此，Linux特有的等同于标准类型的“u8/u16/u32/u64”类型和它们的有符号类型是被 + 因此，Linux 特有的等同于标准类型的 “u8/u16/u32/u64” 类型和它们的有符号类型是被 允许的——尽管在你自己的新代码中，它们不是强制要求要使用的。 当编辑已经使用了某个类型集的已有代码时，你应该遵循那些代码中已经做出的选择。 (e) 可以在用户空间安全使用的类型。 - 在某些用户空间可见的结构体里，我们不能要求C99类型而且不能用上面提到的“u32” - 类型。因此，我们在与用户空间共享的所有结构体中使用__u32和类似的类型。 + 在某些用户空间可见的结构体里，我们不能要求C99类型而且不能用上面提到的 “u32” + 类型。因此，我们在与用户空间共享的所有结构体中使用 __u32 和类似的类型。 -可能还有其他的情况，不过基本的规则是永远不要使用typedef，除非你可以明确的应用上 +可能还有其他的情况，不过基本的规则是永远不要使用 typedef，除非你可以明确的应用上 述某个规则中的一个。 总的来说，如果一个指针或者一个结构体里的元素可以合理的被直接访问到，那么它们就不 -应该是一个typedef。 +应该是一个 typedef。 第六章：函数 函数应该简短而漂亮，并且只完成一件事情。函数应该可以一屏或者两屏显示完（我们都知 -道ISO/ANSI屏幕大小是80x24），只做一件事情，而且把它做好。 +道 ISO/ANSI 屏幕大小是 80x24），只做一件事情，而且把它做好。 一个函数的最大长度是和该函数的复杂度和缩进级数成反比的。所以，如果你有一个理论上 -很简单的只有一个很长（但是简单）的case语句的函数，而且你需要在每个case里做很多很 -小的事情，这样的函数尽管很长，但也是可以的。 +很简单的只有一个很长（但是简单）的 case 语句的函数，而且你需要在每个 case 里做 +很多很小的事情，这样的函数尽管很长，但也是可以的。 不过，如果你有一个复杂的函数，而且你怀疑一个天分不是很高的高中一年级学生可能甚至 搞不清楚这个函数的目的，你应该严格的遵守前面提到的长度限制。使用辅助函数，并为之 取个具描述性的名字（如果你觉得它们的性能很重要的话，可以让编译器内联它们，这样的 效果往往会比你写一个复杂函数的效果要好。） -函数的另外一个衡量标准是本地变量的数量。此数量不应超过5－10个，否则你的函数就有 +函数的另外一个衡量标准是本地变量的数量。此数量不应超过 5－10 个，否则你的函数就有 问题了。重新考虑一下你的函数，把它分拆成更小的函数。人的大脑一般可以轻松的同时跟 -踪7个不同的事物，如果再增多的话，就会糊涂了。即便你聪颖过人，你也可能会记不清你2 -个星期前做过的事情。 +踪 7 个不同的事物，如果再增多的话，就会糊涂了。即便你聪颖过人，你也可能会记不清你 +2 个星期前做过的事情。 -在源文件里，使用空行隔开不同的函数。如果该函数需要被导出，它的EXPORT*宏应该紧贴 +在源文件里，使用空行隔开不同的函数。如果该函数需要被导出，它的 EXPORT* 宏应该紧贴 在它的结束大括号之下。比如： -int system_is_up(void) -{ - return system_state == SYSTEM_RUNNING; -} -EXPORT_SYMBOL(system_is_up); + int system_is_up(void) + { + return system_state == SYSTEM_RUNNING; + } + EXPORT_SYMBOL(system_is_up); -在函数原型中，包含函数名和它们的数据类型。虽然C语言里没有这样的要求，在Linux里这 +在函数原型中，包含函数名和它们的数据类型。虽然C语言里没有这样的要求，在 Linux 里这 是提倡的做法，因为这样可以很简单的给读者提供更多的有价值的信息。 第七章：集中的函数退出途径 -虽然被某些人声称已经过时，但是goto语句的等价物还是经常被编译器所使用，具体形式是 +虽然被某些人声称已经过时，但是 goto 语句的等价物还是经常被编译器所使用，具体形式是 无条件跳转指令。 -当一个函数从多个位置退出并且需要做一些通用的清理工作的时候，goto的好处就显现出来 -了。 +当一个函数从多个位置退出，并且需要做一些类似清理的常见操作时，goto 语句就很方便了。 +如果并不需要清理操作，那么直接 return 即可。 理由是： @@ -354,26 +355,37 @@ EXPORT_SYMBOL(system_is_up); - 可以避免由于修改时忘记更新某个单独的退出点而导致的错误 - 减轻了编译器的工作，无需删除冗余代码;) -int fun(int a) -{ - int result = 0; - char *buffer = kmalloc(SIZE); - - if (buffer == NULL) - return -ENOMEM; - - if (condition1) { - while (loop1) { - ... + int fun(int a) + { + int result = 0; + char *buffer; + + buffer = kmalloc(SIZE, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + if (condition1) { + while (loop1) { + ... + } + result = 1; + goto out_buffer; } - result = 1; - goto out; + ... + out_buffer: + kfree(buffer); + return result; } - ... -out: - kfree(buffer); - return result; -} + +一个需要注意的常见错误是“一个 err 错误”，就像这样： + + err: + kfree(foo->bar); + kfree(foo); + return ret; + +这段代码的错误是，在某些退出路径上 “foo” 是 NULL。通常情况下，通过把它分离成两个 +错误标签 “err_bar:” 和 “err_foo:” 来修复这个错误。 第八章：注释 @@ -386,10 +398,10 @@ out: 加太多。你应该做的，是把注释放在函数的头部，告诉人们它做了什么，也可以加上它做这 些事情的原因。 -当注释内核API函数时，请使用kernel-doc格式。请看 -Documentation/kernel-doc-nano-HOWTO.txt和scripts/kernel-doc以获得详细信息。 +当注释内核API函数时，请使用 kernel-doc 格式。请看 +Documentation/kernel-documentation.rst和scripts/kernel-doc 以获得详细信息。 -Linux的注释风格是C89“/* ... */”风格。不要使用C99风格“// ...”注释。 +Linux的注释风格是 C89 “/* ... */” 风格。不要使用 C99 风格 “// ...” 注释。 长（多行）的首选注释风格是： @@ -402,6 +414,15 @@ Linux的注释风格是C89“/* ... */”风格。不要使用C99风格“// ... * with beginning and ending almost-blank lines. */ +对于在 net/ 和 drivers/net/ 的文件，首选的长（多行）注释风格有些不同。 + + /* The preferred comment style for files in net/ and drivers/net + * looks like this. + * + * It is nearly the same as the generally preferred comment style, + * but there is no initial almost-blank line. + */ + 注释数据也是很重要的，不管是基本类型还是衍生类型。为了方便实现这一点，每一行应只 声明一个数据（不要使用逗号来一次声明多个数据）。这样你就有空间来为每个数据写一段 小注释来解释它们的用途了。 @@ -409,49 +430,63 @@ Linux的注释风格是C89“/* ... */”风格。不要使用C99风格“// ... 第九章：你已经把事情弄糟了 -这没什么，我们都是这样。可能你的使用了很长时间Unix的朋友已经告诉你“GNU emacs”能 -自动帮你格式化C源代码，而且你也注意到了，确实是这样，不过它所使用的默认值和我们 -想要的相去甚远（实际上，甚至比随机打的还要差——无数个猴子在GNU emacs里打字永远不 -会创造出一个好程序）（译注：请参考Infinite Monkey Theorem） - -所以你要么放弃GNU emacs，要么改变它让它使用更合理的设定。要采用后一个方案，你可 -以把下面这段粘贴到你的.emacs文件里。 - -(defun linux-c-mode () - "C mode with adjusted defaults for use with the Linux kernel." - (interactive) - (c-mode) - (c-set-style "K&R") - (setq tab-width 8) - (setq indent-tabs-mode t) - (setq c-basic-offset 8)) - -这样就定义了M-x linux-c-mode命令。当你hack一个模块的时候，如果你把字符串 --*- linux-c -*-放在头两行的某个位置，这个模式将会被自动调用。如果你希望在你修改 -/usr/src/linux里的文件时魔术般自动打开linux-c-mode的话，你也可能需要添加 - -(setq auto-mode-alist (cons '("/usr/src/linux.*/.*\\.[ch]$" . linux-c-mode) - auto-mode-alist)) - -到你的.emacs文件里。 - -不过就算你尝试让emacs正确的格式化代码失败了，也并不意味着你失去了一切：还可以用“ -indent”。 - -不过，GNU indent也有和GNU emacs一样有问题的设定，所以你需要给它一些命令选项。不 -过，这还不算太糟糕，因为就算是GNU indent的作者也认同K&R的权威性（GNU的人并不是坏 -人，他们只是在这个问题上被严重的误导了），所以你只要给indent指定选项“-kr -i8” -（代表“K&R，8个字符缩进”），或者使用“scripts/Lindent”，这样就可以以最时髦的方式 +这没什么，我们都是这样。可能你的使用了很长时间 Unix 的朋友已经告诉你 “GNU emacs” 能 +自动帮你格式化 C 源代码，而且你也注意到了，确实是这样，不过它所使用的默认值和我们 +想要的相去甚远（实际上，甚至比随机打的还要差——无数个猴子在 GNU emacs 里打字永远不 +会创造出一个好程序）（译注：请参考 Infinite Monkey Theorem） + +所以你要么放弃 GNU emacs，要么改变它让它使用更合理的设定。要采用后一个方案，你可 +以把下面这段粘贴到你的 .emacs 文件里。 + +(defun c-lineup-arglist-tabs-only (ignored) + "Line up argument lists by tabs, not spaces" + (let* ((anchor (c-langelem-pos c-syntactic-element)) + (column (c-langelem-2nd-pos c-syntactic-element)) + (offset (- (1+ column) anchor)) + (steps (floor offset c-basic-offset))) + (* (max steps 1) + c-basic-offset))) + +(add-hook 'c-mode-common-hook + (lambda () + ;; Add kernel style + (c-add-style + "linux-tabs-only" + '("linux" (c-offsets-alist + (arglist-cont-nonempty + c-lineup-gcc-asm-reg + c-lineup-arglist-tabs-only)))))) + +(add-hook 'c-mode-hook + (lambda () + (let ((filename (buffer-file-name))) + ;; Enable kernel mode for the appropriate files + (when (and filename + (string-match (expand-file-name "~/src/linux-trees") + filename)) + (setq indent-tabs-mode t) + (setq show-trailing-whitespace t) + (c-set-style "linux-tabs-only"))))) + +这会让 emacs 在 ~/src/linux-trees 目录下的 C 源文件获得更好的内核代码风格。 + +不过就算你尝试让 emacs 正确的格式化代码失败了，也并不意味着你失去了一切：还可以用 +“indent”。 + +不过，GNU indent 也有和 GNU emacs 一样有问题的设定，所以你需要给它一些命令选项。不 +过，这还不算太糟糕，因为就算是 GNU indent 的作者也认同 K&R 的权威性（GNU 的人并不是 +坏人，他们只是在这个问题上被严重的误导了），所以你只要给 indent 指定选项 “-kr -i8” +（代表 “K&R，8 个字符缩进”），或者使用 “scripts/Lindent”，这样就可以以最时髦的方式 缩进源代码。 -“indent”有很多选项，特别是重新格式化注释的时候，你可能需要看一下它的手册页。不过 -记住：“indent”不能修正坏的编程习惯。 +“indent” 有很多选项，特别是重新格式化注释的时候，你可能需要看一下它的手册页。不过 +记住：“indent” 不能修正坏的编程习惯。 - 第十章：Kconfig配置文件 + 第十章：Kconfig 配置文件 -对于遍布源码树的所有Kconfig*配置文件来说，它们缩进方式与C代码相比有所不同。紧挨 -在“config”定义下面的行缩进一个制表符，帮助信息则再多缩进2个空格。比如： +对于遍布源码树的所有 Kconfig* 配置文件来说，它们缩进方式与 C 代码相比有所不同。紧挨 +在 “config” 定义下面的行缩进一个制表符，帮助信息则再多缩进 2 个空格。比如： config AUDIT bool "Auditing support" @@ -470,7 +505,7 @@ config ADFS_FS_RW depends on ADFS_FS ... -要查看配置文件的完整文档，请看Documentation/kbuild/kconfig-language.txt。 +要查看配置文件的完整文档，请看 Documentation/kbuild/kconfig-language.txt。 第十一章：数据结构 @@ -489,11 +524,11 @@ config ADFS_FS_RW 很多数据结构实际上有2级引用计数，它们通常有不同“类”的用户。子类计数器统计子类用 户的数量，每当子类计数器减至零时，全局计数器减一。 -这种“多级引用计数”的例子可以在内存管理（“struct mm_struct”：mm_users和mm_count） +这种“多级引用计数”的例子可以在内存管理（“struct mm_struct”：mm_users 和 mm_count） 和文件系统（“struct super_block”：s_count和s_active）中找到。 记住：如果另一个执行线索可以找到你的数据结构，但是这个数据结构没有引用计数器，这 -里几乎肯定是一个bug。 +里几乎肯定是一个 bug。 第十二章：宏，枚举和RTL @@ -508,102 +543,128 @@ config ADFS_FS_RW 一般的，如果能写成内联函数就不要写成像函数的宏。 -含有多个语句的宏应该被包含在一个do-while代码块里： +含有多个语句的宏应该被包含在一个 do-while 代码块里： -#define macrofun(a, b, c) \ - do { \ - if (a == 5) \ - do_this(b, c); \ - } while (0) + #define macrofun(a, b, c) \ + do { \ + if (a == 5) \ + do_this(b, c); \ + } while (0) 使用宏的时候应避免的事情： 1) 影响控制流程的宏： -#define FOO(x) \ - do { \ - if (blah(x) < 0) \ - return -EBUGGERED; \ - } while(0) + #define FOO(x) \ + do { \ + if (blah(x) < 0) \ + return -EBUGGERED; \ + } while (0) 非常不好。它看起来像一个函数，不过却能导致“调用”它的函数退出；不要打乱读者大脑里 的语法分析器。 2) 依赖于一个固定名字的本地变量的宏： -#define FOO(val) bar(index, val) + #define FOO(val) bar(index, val) 可能看起来像是个不错的东西，不过它非常容易把读代码的人搞糊涂，而且容易导致看起来 不相关的改动带来错误。 -3) 作为左值的带参数的宏： FOO(x) = y；如果有人把FOO变成一个内联函数的话，这种用 +3) 作为左值的带参数的宏： FOO(x) = y；如果有人把 FOO 变成一个内联函数的话，这种用 法就会出错了。 4) 忘记了优先级：使用表达式定义常量的宏必须将表达式置于一对小括号之内。带参数的 宏也要注意此类问题。 -#define CONSTANT 0x4000 -#define CONSTEXP (CONSTANT | 3) + #define CONSTANT 0x4000 + #define CONSTEXP (CONSTANT | 3) + +5) 在宏里定义类似函数的本地变量时命名冲突： -cpp手册对宏的讲解很详细。Gcc internals手册也详细讲解了RTL（译注：register + #define FOO(x) \ + ({ \ + typeof(x) ret; \ + ret = calc_ret(x); \ + (ret); \ + }) + +ret 是本地变量的通用名字 - __foo_ret 更不容易与一个已存在的变量冲突。 + +cpp 手册对宏的讲解很详细。gcc internals 手册也详细讲解了 RTL（译注：register transfer language），内核里的汇编语言经常用到它。 第十三章：打印内核消息 内核开发者应该是受过良好教育的。请一定注意内核信息的拼写，以给人以好的印象。不要 -用不规范的单词比如“dont”，而要用“do not”或者“don't”。保证这些信息简单、明了、无 -歧义。 +用不规范的单词比如 “dont”，而要用 “do not”或者 “don't”。保证这些信息简单、明了、 +无歧义。 内核信息不必以句号（译注：英文句号，即点）结束。 -在小括号里打印数字(%d)没有任何价值，应该避免这样做。 +在小括号里打印数字 (%d) 没有任何价值，应该避免这样做。 -里有一些驱动模型诊断宏，你应该使用它们，以确保信息对应于正确的 -设备和驱动，并且被标记了正确的消息级别。这些宏有：dev_err(), dev_warn(), -dev_info()等等。对于那些不和某个特定设备相关连的信息，定义了 -pr_debug()和pr_info()。 + 里有一些驱动模型诊断宏，你应该使用它们，以确保信息对应于正确的 +设备和驱动，并且被标记了正确的消息级别。这些宏有：dev_err()，dev_warn()， +dev_info() 等等。对于那些不和某个特定设备相关连的信息， 定义了 +pr_notice()，pr_info()，pr_warn()，pr_err() 和其他。 -写出好的调试信息可以是一个很大的挑战；当你写出来之后，这些信息在远程除错的时候 -就会成为极大的帮助。当DEBUG符号没有被定义的时候，这些信息不应该被编译进内核里 -（也就是说，默认地，它们不应该被包含在内）。如果你使用dev_dbg()或者pr_debug()， -就能自动达到这个效果。很多子系统拥有Kconfig选项来启用-DDEBUG。还有一个相关的惯例 -是使用VERBOSE_DEBUG来添加dev_vdbg()消息到那些已经由DEBUG启用的消息之上。 +写出好的调试信息可以是一个很大的挑战；一旦你写出后，这些信息在远程除错时能提供极大 +的帮助。然而打印调试信息的处理方式同打印非调试信息不同。其他 pr_XXX() 函数能无条件地 +打印，pr_debug() 却不；默认情况下它不会被编译，除非定义了 DEBUG 或设定了 +CONFIG_DYNAMIC_DEBUG。实际这同样是为了 dev_dbg()，一个相关约定是在一个已经开启了 +DEBUG 时，使用 VERBOSE_DEBUG 来添加 dev_vdbg()。 + +许多子系统拥有 Kconfig 调试选项来开启 -DDEBUG 在对应的 Makefile 里面；在其他 +情况下，特殊文件使用 #define DEBUG。当一条调试信息需要被无条件打印时，例如，如果 +已经包含一个调试相关的 #ifdef 条件，printk(KERN_DEBUG ...) 就可被使用。 第十四章：分配内存 -内核提供了下面的一般用途的内存分配函数：kmalloc()，kzalloc()，kcalloc()和 -vmalloc()。请参考API文档以获取有关它们的详细信息。 +内核提供了下面的一般用途的内存分配函数： +kmalloc()，kzalloc()，kmalloc_array()，kcalloc()，vmalloc() 和 vzalloc()。 +请参考 API 文档以获取有关它们的详细信息。 传递结构体大小的首选形式是这样的： p = kmalloc(sizeof(*p), ...); -另外一种传递方式中，sizeof的操作数是结构体的名字，这样会降低可读性，并且可能会引 -入bug。有可能指针变量类型被改变时，而对应的传递给内存分配函数的sizeof的结果不变。 +另外一种传递方式中，sizeof 的操作数是结构体的名字，这样会降低可读性，并且可能会引 +入 bug。有可能指针变量类型被改变时，而对应的传递给内存分配函数的 sizeof 的结果不变。 -强制转换一个void指针返回值是多余的。C语言本身保证了从void指针到其他任何指针类型 +强制转换一个 void 指针返回值是多余的。C 语言本身保证了从 void 指针到其他任何指针类型 的转换是没有问题的。 +分配一个数组的首选形式是这样的： + + p = kmalloc_array(n, sizeof(...), ...); + +分配一个零长数组的首选形式是这样的： + + p = kcalloc(n, sizeof(...), ...); + +两种形式检查分配大小 n * sizeof(...) 的溢出，如果溢出返回 NULL。 + 第十五章：内联弊病 -有一个常见的误解是内联函数是gcc提供的可以让代码运行更快的一个选项。虽然使用内联 +有一个常见的误解是内联函数是 gcc 提供的可以让代码运行更快的一个选项。虽然使用内联 函数有时候是恰当的（比如作为一种替代宏的方式，请看第十二章），不过很多情况下不是 -这样。inline关键字的过度使用会使内核变大，从而使整个系统运行速度变慢。因为大内核 +这样。inline 关键字的过度使用会使内核变大，从而使整个系统运行速度变慢。因为大内核 会占用更多的指令高速缓存（译注：一级缓存通常是指令缓存和数据缓存分开的）而且会导 -致pagecache的可用内存减少。想象一下，一次pagecache未命中就会导致一次磁盘寻址，将 -耗时5毫秒。5毫秒的时间内CPU能执行很多很多指令。 +致 pagecache 的可用内存减少。想象一下，一次pagecache未命中就会导致一次磁盘寻址， +将耗时 5 毫秒。5 毫秒的时间内 CPU 能执行很多很多指令。 -一个基本的原则是如果一个函数有3行以上，就不要把它变成内联函数。这个原则的一个例 +一个基本的原则是如果一个函数有 3 行以上，就不要把它变成内联函数。这个原则的一个例 外是，如果你知道某个参数是一个编译时常量，而且因为这个常量你确定编译器在编译时能 -优化掉你的函数的大部分代码，那仍然可以给它加上inline关键字。kmalloc()内联函数就 +优化掉你的函数的大部分代码，那仍然可以给它加上 inline 关键字。kmalloc() 内联函数就 是一个很好的例子。 -人们经常主张给static的而且只用了一次的函数加上inline，如此不会有任何损失，因为没 -有什么好权衡的。虽然从技术上说这是正确的，但是实际上这种情况下即使不加inline gcc -也可以自动使其内联。而且其他用户可能会要求移除inline，由此而来的争论会抵消inline +人们经常主张给 static 的而且只用了一次的函数加上 inline，如此不会有任何损失，因为没 +有什么好权衡的。虽然从技术上说这是正确的，但是实际上这种情况下即使不加 inline gcc +也可以自动使其内联。而且其他用户可能会要求移除 inline，由此而来的争论会抵消 inline 自身的潜在价值，得不偿失。 @@ -613,37 +674,37 @@ vmalloc()。请参考API文档以获取有关它们的详细信息。 的一个值可以表示为一个错误代码整数（-Exxx＝失败，0＝成功）或者一个“成功”布尔值（ 0＝失败，非0＝成功）。 -混合使用这两种表达方式是难于发现的bug的来源。如果C语言本身严格区分整形和布尔型变 -量，那么编译器就能够帮我们发现这些错误……不过C语言不区分。为了避免产生这种bug，请 +混合使用这两种表达方式是难于发现的 bug 的来源。如果 C 语言本身严格区分整形和布尔型变 +量，那么编译器就能够帮我们发现这些错误……不过 C 语言不区分。为了避免产生这种 bug，请 遵循下面的惯例： 如果函数的名字是一个动作或者强制性的命令，那么这个函数应该返回错误代码整 数。如果是一个判断，那么函数应该返回一个“成功”布尔值。 -比如，“add work”是一个命令，所以add_work()函数在成功时返回0，在失败时返回-EBUSY。 -类似的，因为“PCI device present”是一个判断，所以pci_dev_present()函数在成功找到 -一个匹配的设备时应该返回1，如果找不到时应该返回0。 +比如，“add work” 是一个命令，所以 add_work() 函数在成功时返回 0，在失败时返回 -EBUSY。 +类似的，因为 “PCI device present” 是一个判断，所以 pci_dev_present() 函数在成功找到 +一个匹配的设备时应该返回 1，如果找不到时应该返回 0。 所有导出（译注：EXPORT）的函数都必须遵守这个惯例，所有的公共函数也都应该如此。私 有（static）函数不需要如此，但是我们也推荐这样做。 返回值是实际计算结果而不是计算是否成功的标志的函数不受此惯例的限制。一般的，他们 通过返回一些正常值范围之外的结果来表示出错。典型的例子是返回指针的函数，他们使用 -NULL或者ERR_PTR机制来报告错误。 +NULL 或者 ERR_PTR 机制来报告错误。 第十七章：不要重新发明内核宏 -头文件include/linux/kernel.h包含了一些宏，你应该使用它们，而不要自己写一些它们的 +头文件 include/linux/kernel.h 包含了一些宏，你应该使用它们，而不要自己写一些它们的 变种。比如，如果你需要计算一个数组的长度，使用这个宏 - #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 类似的，如果你要计算某结构体成员的大小，使用 - #define FIELD_SIZEOF(t, f) (sizeof(((t*)0)->f)) + #define FIELD_SIZEOF(t, f) (sizeof(((t*)0)->f)) -还有可以做严格的类型检查的min()和max()宏，如果你需要可以使用它们。你可以自己看看 +还有可以做严格的类型检查的 min() 和 max() 宏，如果你需要可以使用它们。你可以自己看看 那个头文件里还定义了什么你可以拿来用的东西，如果有定义的话，你就不应在你的代码里 自己重新定义。 @@ -653,42 +714,100 @@ NULL或者ERR_PTR机制来报告错误。 有一些编辑器可以解释嵌入在源文件里的由一些特殊标记标明的配置信息。比如，emacs 能够解释被标记成这样的行： --*- mode: c -*- + -*- mode: c -*- 或者这样的： -/* -Local Variables: -compile-command: "gcc -DMAGIC_DEBUG_FLAG foo.c" -End: -*/ + /* + Local Variables: + compile-command: "gcc -DMAGIC_DEBUG_FLAG foo.c" + End: + */ -Vim能够解释这样的标记： +Vim 能够解释这样的标记： -/* vim:set sw=8 noet */ + /* vim:set sw=8 noet */ 不要在源代码中包含任何这样的内容。每个人都有他自己的编辑器配置，你的源文件不应 该覆盖别人的配置。这包括有关缩进和模式配置的标记。人们可以使用他们自己定制的模 式，或者使用其他可以产生正确的缩进的巧妙方法。 + 第十九章：内联汇编 + +在特定架构的代码中，你也许需要内联汇编来使用 CPU 接口和平台相关功能。在需要 +这么做时，不要犹豫。然而，当 C 可以完成工作时，不要无端地使用内联汇编。如果 +可能，你可以并且应该用 C 和硬件交互。 + +考虑去写通用一点的内联汇编作为简明的辅助函数，而不是重复写下它们的细节。记住 +内联汇编可以使用 C 参数。 + +大而特殊的汇编函数应该放在 .S 文件中，对应 C 的原型定义在 C 头文件中。汇编 +函数的 C 原型应该使用 “asmlinkage”。 + +你可能需要将你的汇编语句标记为 volatile，来阻止 GCC 在没发现任何副作用后就 +移除了它。你不必总是这样做，虽然，这样可以限制不必要的优化。 + +在写一个包含多条指令的单个内联汇编语句时，把每条指令用引号字符串分离，并写在 +单独一行，在每个字符串结尾，除了 \n\t 结尾之外，在汇编输出中适当地缩进下 +一条指令： + + asm ("magic %reg1, #42\n\t" + "more_magic %reg2, %reg3" + : /* outputs */ : /* inputs */ : /* clobbers */); + + + 第二十章：条件编译 + +只要可能，就不要在 .c 文件里面使用预处理条件；这样做让代码更难阅读并且逻辑难以 +跟踪。替代方案是，在头文件定义函数在这些 .c 文件中使用这类的条件表达式，提供空 +操作的桩版本（译注：桩程序，是指用来替换一部分功能的程序段）在 #else 情况下， +再从 .c 文件中无条件地调用这些函数。编译器会避免生成任何桩调用的代码，产生一致 +的结果，但逻辑将更加清晰。 + +宁可编译整个函数，而不是部分函数或部分表达式。而不是在一个表达式添加 ifdef， +解析部分或全部表达式到一个单独的辅助函数，并应用条件到该函数内。 + +如果你有一个在特定配置中可能是未使用的函数或变量，编译器将警告它定义了但未使用， +标记这个定义为 __maybe_unused 而不是将它包含在一个预处理条件中。（然而，如果 +一个函数或变量总是未使用的，就直接删除它。） + +在代码中，可能的情况下，使用 IS_ENABLED 宏来转化某个 Kconfig 标记为 C 的布尔 +表达式，并在正常的 C 条件中使用它： + + if (IS_ENABLED(CONFIG_SOMETHING)) { + ... + } + +编译器会无条件地做常数合并，就像使用 #ifdef 那样，包含或排除代码块，所以这不会 +带来任何运行时开销。然而，这种方法依旧允许 C 编译器查看块内的代码，并检查它的正确 +性（语法，类型，符号引用，等等）。因此，如果条件不满足，代码块内的引用符号将不存在， +你必须继续使用 #ifdef。 + +在任何有意义的 #if 或 #ifdef 块的末尾（超过几行），在 #endif 同一行的后面写下 +注释，指出该条件表达式被使用。例如： + + #ifdef CONFIG_SOMETHING + ... + #endif /* CONFIG_SOMETHING */ + 附录 I：参考 -The C Programming Language, 第二版, 作者Brian W. Kernighan和Denni -M. Ritchie. Prentice Hall, Inc., 1988. ISBN 0-13-110362-8 (软皮), -0-13-110370-9 (硬皮). URL: http://cm.bell-labs.com/cm/cs/cbook/ +The C Programming Language, 第二版 +作者：Brian W. Kernighan 和 Denni M. Ritchie. +Prentice Hall, Inc., 1988. +ISBN 0-13-110362-8 (软皮), 0-13-110370-9 (硬皮). -The Practice of Programming 作者Brian W. Kernighan和Rob Pike. Addison-Wesley, -Inc., 1999. ISBN 0-201-61586-X. URL: http://cm.bell-labs.com/cm/cs/tpop/ +The Practice of Programming +作者：Brian W. Kernighan 和 Rob Pike. +Addison-Wesley, Inc., 1999. +ISBN 0-201-61586-X. -cpp，gcc，gcc internals和indent的GNU手册——和K&R及本文相符合的部分，全部可以在 -http://www.gnu.org/manual/找到 +GNU 手册 - 遵循 K&R 标准和此文本 - cpp, gcc, gcc internals and indent, +都可以从 http://www.gnu.org/manual/ 找到 WG14是C语言的国际标准化工作组，URL: http://www.open-std.org/JTC1/SC22/WG14/ -Kernel CodingStyle，作者greg@kroah.com发表于OLS 2002： +Kernel CodingStyle，作者 greg@kroah.com 发表于OLS 2002： http://www.kroah.com/linux/talks/ols_2002_kernel_codingstyle_talk/html/ - --- -最后更新于2007å¹´7月13日。 diff --git a/MAINTAINERS b/MAINTAINERS index 419763e3b52e..3a7b3998e845 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -288,6 +288,7 @@ F: include/linux/acpi.h F: include/acpi/ F: Documentation/acpi/ F: Documentation/ABI/testing/sysfs-bus-acpi +F: Documentation/ABI/testing/configfs-acpi F: drivers/pci/*acpi* F: drivers/pci/*/*acpi* F: drivers/pci/*/*/*acpi* @@ -595,6 +596,10 @@ S: Odd Fixes L: linux-alpha@vger.kernel.org F: arch/alpha/ +ALPS PS/2 TOUCHPAD DRIVER +R: Pali Rohár +F: drivers/input/mouse/alps.* + ALTERA MAILBOX DRIVER M: Ley Foon Tan L: nios2-dev@lists.rocketboards.org (moderated for non-subscribers) @@ -835,7 +840,9 @@ M: Iyappan Subramanian M: Keyur Chudgar S: Supported F: drivers/net/ethernet/apm/xgene/ +F: drivers/net/phy/mdio-xgene.c F: Documentation/devicetree/bindings/net/apm-xgene-enet.txt +F: Documentation/devicetree/bindings/net/apm-xgene-mdio.txt APTINA CAMERA SENSOR PLL M: Laurent Pinchart @@ -1643,6 +1650,13 @@ L: linux-media@vger.kernel.org S: Maintained F: drivers/media/platform/s5p-tv/ +ARM/SAMSUNG S5P SERIES HDMI CEC SUBSYSTEM SUPPORT +M: Kyungmin Park +L: linux-arm-kernel@lists.infradead.org +L: linux-media@vger.kernel.org +S: Maintained +F: drivers/staging/media/platform/s5p-cec/ + ARM/SAMSUNG S5P SERIES JPEG CODEC SUPPORT M: Andrzej Pietrasiewicz M: Jacek Anaszewski @@ -1665,7 +1679,6 @@ F: arch/arm/boot/dts/sh* F: arch/arm/configs/shmobile_defconfig F: arch/arm/include/debug/renesas-scif.S F: arch/arm/mach-shmobile/ -F: drivers/sh/ F: drivers/soc/renesas/ F: include/linux/soc/renesas/ @@ -1690,8 +1703,6 @@ S: Maintained F: drivers/edac/altera_edac. ARM/STI ARCHITECTURE -M: Srinivas Kandagatla -M: Maxime Coquelin M: Patrice Chotard L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: kernel@stlinux.com @@ -1724,6 +1735,7 @@ F: drivers/ata/ahci_st.c ARM/STM32 ARCHITECTURE M: Maxime Coquelin +M: Alexandre Torgue L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/mcoquelin/stm32.git @@ -2293,6 +2305,7 @@ S: Maintained F: Documentation/ABI/testing/sysfs-class-net-batman-adv F: Documentation/ABI/testing/sysfs-class-net-mesh F: Documentation/networking/batman-adv.txt +F: include/uapi/linux/batman_adv.h F: net/batman-adv/ BAYCOM/HDLCDRV DRIVERS FOR AX.25 @@ -2456,6 +2469,14 @@ L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/broadcom/b44.* +BROADCOM B53 ETHERNET SWITCH DRIVER +M: Florian Fainelli +L: netdev@vger.kernel.org +L: openwrt-devel@lists.openwrt.org (subscribers-only) +S: Supported +F: drivers/net/dsa/b53/* +F: include/linux/platform_data/b53.h + BROADCOM GENET ETHERNET DRIVER M: Florian Fainelli L: netdev@vger.kernel.org @@ -2572,12 +2593,11 @@ S: Supported F: drivers/net/ethernet/broadcom/tg3.* BROADCOM BRCM80211 IEEE802.11n WIRELESS DRIVER -M: Brett Rudley -M: Arend van Spriel -M: Franky (Zhenhui) Lin -M: Hante Meuleman +M: Arend van Spriel +M: Franky Lin +M: Hante Meuleman L: linux-wireless@vger.kernel.org -L: brcm80211-dev-list@broadcom.com +L: brcm80211-dev-list.pdl@broadcom.com S: Supported F: drivers/net/wireless/broadcom/brcm80211/ @@ -2776,9 +2796,9 @@ F: include/net/caif/ F: net/caif/ CALGARY x86-64 IOMMU -M: Muli Ben-Yehuda -M: "Jon D. Mason" -L: discuss@x86-64.org +M: Muli Ben-Yehuda +M: Jon Mason +L: iommu@lists.linux-foundation.org S: Maintained F: arch/x86/kernel/pci-calgary_64.c F: arch/x86/kernel/tce_64.c @@ -2809,6 +2829,7 @@ W: https://github.com/linux-can T: git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can.git T: git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can-next.git S: Maintained +F: Documentation/devicetree/bindings/net/can/ F: drivers/net/can/ F: include/linux/can/dev.h F: include/linux/can/platform/ @@ -2848,6 +2869,22 @@ F: drivers/net/ieee802154/cc2520.c F: include/linux/spi/cc2520.h F: Documentation/devicetree/bindings/net/ieee802154/cc2520.txt +CEC DRIVER +M: Hans Verkuil +L: linux-media@vger.kernel.org +T: git git://linuxtv.org/media_tree.git +W: http://linuxtv.org +S: Supported +F: Documentation/cec.txt +F: Documentation/DocBook/media/v4l/cec* +F: drivers/staging/media/cec/ +F: drivers/media/cec-edid.c +F: drivers/media/rc/keymaps/rc-cec.c +F: include/media/cec.h +F: include/media/cec-edid.h +F: include/linux/cec.h +F: include/linux/cec-funcs.h + CELL BROADBAND ENGINE ARCHITECTURE M: Arnd Bergmann L: linuxppc-dev@lists.ozlabs.org @@ -3284,6 +3321,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git T: git git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git S: Maintained F: Documentation/crypto/ +F: Documentation/devicetree/bindings/crypto/ F: Documentation/DocBook/crypto-API.tmpl F: arch/*/crypto/ F: crypto/ @@ -4473,7 +4511,7 @@ S: Orphan F: fs/efs/ EHEA (IBM pSeries eHEA 10Gb ethernet adapter) DRIVER -M: Thadeu Lima de Souza Cascardo +M: Douglas Miller L: netdev@vger.kernel.org S: Maintained F: drivers/net/ethernet/ibm/ehea/ @@ -4883,6 +4921,13 @@ F: drivers/net/ethernet/freescale/gianfar* X: drivers/net/ethernet/freescale/gianfar_ptp.c F: Documentation/devicetree/bindings/net/fsl-tsec-phy.txt +FREESCALE QUICC ENGINE UCC HDLC DRIVER +M: Zhao Qiang +L: netdev@vger.kernel.org +L: linuxppc-dev@lists.ozlabs.org +S: Maintained +F: drivers/net/wan/fsl_ucc_hdlc* + FREESCALE QUICC ENGINE UCC UART DRIVER M: Timur Tabi L: linuxppc-dev@lists.ozlabs.org @@ -4938,6 +4983,13 @@ F: Documentation/filesystems/caching/ F: fs/fscache/ F: include/linux/fscache*.h +FS-CRYPTO: FILE SYSTEM LEVEL ENCRYPTION SUPPORT +M: Theodore Y. Ts'o +M: Jaegeuk Kim +S: Supported +F: fs/crypto/ +F: include/linux/fscrypto.h + F2FS FILE SYSTEM M: Jaegeuk Kim M: Changman Lee @@ -5166,10 +5218,10 @@ S: Maintained F: drivers/media/usb/gspca/m5602/ GSPCA PAC207 SONIXB SUBDRIVER -M: Hans de Goede +M: Hans Verkuil L: linux-media@vger.kernel.org T: git git://linuxtv.org/media_tree.git -S: Maintained +S: Odd Fixes F: drivers/media/usb/gspca/pac207.c GSPCA SN9C20X SUBDRIVER @@ -5187,10 +5239,10 @@ S: Maintained F: drivers/media/usb/gspca/t613.c GSPCA USB WEBCAM DRIVER -M: Hans de Goede +M: Hans Verkuil L: linux-media@vger.kernel.org T: git git://linuxtv.org/media_tree.git -S: Maintained +S: Odd Fixes F: drivers/media/usb/gspca/ GUID PARTITION TABLE (GPT) @@ -5271,6 +5323,7 @@ M: Matt Mackall M: Herbert Xu L: linux-crypto@vger.kernel.org S: Odd fixes +F: Documentation/devicetree/bindings/rng/ F: Documentation/hw_random.txt F: drivers/char/hw_random/ F: include/linux/hw_random.h @@ -5418,6 +5471,15 @@ F: include/uapi/linux/if_hippi.h F: net/802/hippi.c F: drivers/net/hippi/ +HISILICON NETWORK SUBSYSTEM DRIVER +M: Yisen Zhuang +M: Salil Mehta +L: netdev@vger.kernel.org +W: http://www.hisilicon.com +S: Maintained +F: drivers/net/ethernet/hisilicon/ +F: Documentation/devicetree/bindings/net/hisilicon*.txt + HISILICON SAS Controller M: John Garry W: http://www.hisilicon.com @@ -5785,7 +5847,9 @@ R: Hartmut Knaack R: Lars-Peter Clausen R: Peter Meerwald-Stadler L: linux-iio@vger.kernel.org +T: git git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git S: Maintained +F: Documentation/devicetree/bindings/iio/ F: drivers/iio/ F: drivers/staging/iio/ F: include/linux/iio/ @@ -5915,6 +5979,12 @@ L: platform-driver-x86@vger.kernel.org S: Maintained F: drivers/platform/x86/intel-hid.c +INTEL VIRTUAL BUTTON DRIVER +M: AceLan Kao +L: platform-driver-x86@vger.kernel.org +S: Maintained +F: drivers/platform/x86/intel-vbtn.c + INTEL IDLE DRIVER M: Len Brown L: linux-pm@vger.kernel.org @@ -6017,6 +6087,12 @@ L: linux-rdma@vger.kernel.org S: Supported F: drivers/infiniband/hw/i40iw/ +INTEL MERRIFIELD GPIO DRIVER +M: Andy Shevchenko +L: linux-gpio@vger.kernel.org +S: Maintained +F: drivers/gpio/gpio-merrifield.c + INTEL-MID GPIO DRIVER M: David Cohen L: linux-gpio@vger.kernel.org @@ -6231,7 +6307,6 @@ F: Documentation/devicetree/bindings/interrupt-controller/ F: drivers/irqchip/ IRQ DOMAINS (IRQ NUMBER MAPPING LIBRARY) -M: Jiang Liu M: Marc Zyngier S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git irq/core @@ -6764,6 +6839,7 @@ S: Maintained F: drivers/ata/ F: include/linux/ata.h F: include/linux/libata.h +F: Documentation/devicetree/bindings/ata/ LIBATA PATA ARASAN COMPACT FLASH CONTROLLER M: Viresh Kumar @@ -6968,7 +7044,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/jikos/livepatching.git LINUX KERNEL DUMP TEST MODULE (LKDTM) M: Kees Cook S: Maintained -F: drivers/misc/lkdtm.c +F: drivers/misc/lkdtm* LLC (802.2) M: Arnaldo Carvalho de Melo @@ -7021,15 +7097,23 @@ Q: http://patchwork.linuxtv.org/project/linux-media/list/ S: Maintained F: drivers/media/usb/dvb-usb-v2/lmedm04* -LOCKDEP AND LOCKSTAT +LOCKING PRIMITIVES M: Peter Zijlstra M: Ingo Molnar L: linux-kernel@vger.kernel.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git core/locking +T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking/core S: Maintained -F: Documentation/locking/lockdep*.txt -F: Documentation/locking/lockstat.txt +F: Documentation/locking/ F: include/linux/lockdep.h +F: include/linux/spinlock*.h +F: arch/*/include/asm/spinlock*.h +F: include/linux/rwlock*.h +F: include/linux/mutex*.h +F: arch/*/include/asm/mutex*.h +F: include/linux/rwsem*.h +F: arch/*/include/asm/rwsem.h +F: include/linux/seqlock.h +F: lib/locking*.[ch] F: kernel/locking/ LOGICAL DISK MANAGER SUPPORT (LDM, Windows 2000/XP/Vista Dynamic Disks) @@ -7170,6 +7254,12 @@ W: http://www.kernel.org/doc/man-pages L: linux-man@vger.kernel.org S: Maintained +MARVELL 88E6XXX ETHERNET SWITCH FABRIC DRIVER +M: Andrew Lunn +M: Vivien Didelot +S: Maintained +F: drivers/net/dsa/mv88e6xxx/ + MARVELL ARMADA DRM SUPPORT M: Russell King S: Maintained @@ -7177,11 +7267,6 @@ F: drivers/gpu/drm/armada/ F: include/uapi/drm/armada_drm.h F: Documentation/devicetree/bindings/display/armada/ -MARVELL 88E6352 DSA support -M: Guenter Roeck -S: Maintained -F: drivers/net/dsa/mv88e6352.c - MARVELL CRYPTO DRIVER M: Boris Brezillon M: Arnaud Ebalard @@ -7317,6 +7402,16 @@ L: linux-iio@vger.kernel.org S: Maintained F: drivers/iio/potentiometer/mcp4531.c +MEDIA DRIVERS FOR RENESAS - FCP +M: Laurent Pinchart +L: linux-media@vger.kernel.org +L: linux-renesas-soc@vger.kernel.org +T: git git://linuxtv.org/media_tree.git +S: Supported +F: Documentation/devicetree/bindings/media/renesas,fcp.txt +F: drivers/media/platform/rcar-fcp.c +F: include/media/rcar-fcp.h + MEDIA DRIVERS FOR RENESAS - VSP1 M: Laurent Pinchart L: linux-media@vger.kernel.org @@ -7326,8 +7421,18 @@ S: Supported F: Documentation/devicetree/bindings/media/renesas,vsp1.txt F: drivers/media/platform/vsp1/ +MEDIA DRIVERS FOR HELENE +M: Abylay Ospan +L: linux-media@vger.kernel.org +W: https://linuxtv.org +W: http://netup.tv/ +T: git git://linuxtv.org/media_tree.git +S: Supported +F: drivers/media/dvb-frontends/helene* + MEDIA DRIVERS FOR ASCOT2E M: Sergey Kozlov +M: Abylay Ospan L: linux-media@vger.kernel.org W: https://linuxtv.org W: http://netup.tv/ @@ -7337,6 +7442,7 @@ F: drivers/media/dvb-frontends/ascot2e* MEDIA DRIVERS FOR CXD2841ER M: Sergey Kozlov +M: Abylay Ospan L: linux-media@vger.kernel.org W: https://linuxtv.org W: http://netup.tv/ @@ -7346,6 +7452,7 @@ F: drivers/media/dvb-frontends/cxd2841er* MEDIA DRIVERS FOR HORUS3A M: Sergey Kozlov +M: Abylay Ospan L: linux-media@vger.kernel.org W: https://linuxtv.org W: http://netup.tv/ @@ -7355,6 +7462,7 @@ F: drivers/media/dvb-frontends/horus3a* MEDIA DRIVERS FOR LNBH25 M: Sergey Kozlov +M: Abylay Ospan L: linux-media@vger.kernel.org W: https://linuxtv.org W: http://netup.tv/ @@ -7364,6 +7472,7 @@ F: drivers/media/dvb-frontends/lnbh25* MEDIA DRIVERS FOR NETUP PCI UNIVERSAL DVB devices M: Sergey Kozlov +M: Abylay Ospan L: linux-media@vger.kernel.org W: https://linuxtv.org W: http://netup.tv/ @@ -7421,7 +7530,7 @@ F: drivers/scsi/megaraid.* F: drivers/scsi/megaraid/ MELLANOX ETHERNET DRIVER (mlx4_en) -M: Eugenia Emantayev +M: Tariq Toukan L: netdev@vger.kernel.org S: Supported W: http://www.mellanox.com @@ -7473,6 +7582,7 @@ Q: http://patchwork.ozlabs.org/project/linux-mtd/list/ T: git git://git.infradead.org/linux-mtd.git T: git git://git.infradead.org/l2-mtd.git S: Maintained +F: Documentation/devicetree/bindings/mtd/ F: drivers/mtd/ F: include/linux/mtd/ F: include/uapi/mtd/ @@ -7612,10 +7722,8 @@ L: linux-media@vger.kernel.org W: https://linuxtv.org W: http://palosaari.fi/linux/ Q: http://patchwork.linuxtv.org/project/linux-media/list/ -T: git git://linuxtv.org/anttip/media_tree.git S: Maintained -F: drivers/staging/media/mn88472/ -F: drivers/media/dvb-frontends/mn88472.h +F: drivers/media/dvb-frontends/mn88472* MN88473 MEDIA DRIVER M: Antti Palosaari @@ -8170,6 +8278,13 @@ S: Supported F: drivers/nvme/host/ F: include/linux/nvme.h +NVM EXPRESS TARGET DRIVER +M: Christoph Hellwig +M: Sagi Grimberg +L: linux-nvme@lists.infradead.org +S: Supported +F: drivers/nvme/target/ + NVMEM FRAMEWORK M: Srinivas Kandagatla M: Maxime Ripard @@ -8960,6 +9075,7 @@ L: linux-gpio@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git S: Maintained F: Documentation/devicetree/bindings/pinctrl/ +F: Documentation/pinctrl.txt F: drivers/pinctrl/ F: include/linux/pinctrl/ @@ -9082,6 +9198,8 @@ M: David Woodhouse L: linux-pm@vger.kernel.org T: git git://git.infradead.org/battery-2.6.git S: Maintained +F: Documentation/devicetree/bindings/power/ +F: Documentation/devicetree/bindings/power_supply/ F: include/linux/power_supply.h F: drivers/power/ X: drivers/power/avs/ @@ -9219,6 +9337,13 @@ F: include/linux/tracehook.h F: include/uapi/linux/ptrace.h F: kernel/ptrace.c +PULSE8-CEC DRIVER +M: Hans Verkuil +L: linux-media@vger.kernel.org +T: git git://linuxtv.org/media_tree.git +S: Maintained +F: drivers/staging/media/pulse8-cec + PVRUSB2 VIDEO4LINUX DRIVER M: Mike Isely L: pvrusb2@isely.net (subscribers-only) @@ -9230,10 +9355,10 @@ F: Documentation/video4linux/README.pvrusb2 F: drivers/media/usb/pvrusb2/ PWC WEBCAM DRIVER -M: Hans de Goede +M: Hans Verkuil L: linux-media@vger.kernel.org T: git git://linuxtv.org/media_tree.git -S: Maintained +S: Odd Fixes F: drivers/media/usb/pwc/* PWM FAN DRIVER @@ -9306,7 +9431,8 @@ L: rtc-linux@googlegroups.com S: Maintained QAT DRIVER -M: Tadeusz Struk +M: Giovanni Cabiddu +M: Salvatore Benedetto L: qat-linux@intel.com S: Supported F: drivers/crypto/qat/ @@ -9448,14 +9574,14 @@ F: drivers/video/fbdev/aty/radeon* F: include/uapi/linux/radeonfb.h RADIOSHARK RADIO DRIVER -M: Hans de Goede +M: Hans Verkuil L: linux-media@vger.kernel.org T: git git://linuxtv.org/media_tree.git S: Maintained F: drivers/media/radio/radio-shark.c RADIOSHARK2 RADIO DRIVER -M: Hans de Goede +M: Hans Verkuil L: linux-media@vger.kernel.org T: git git://linuxtv.org/media_tree.git S: Maintained @@ -9529,7 +9655,7 @@ M: Florian Fainelli S: Maintained RDC R6040 FAST ETHERNET DRIVER -M: Florian Fainelli +M: Florian Fainelli L: netdev@vger.kernel.org S: Maintained F: drivers/net/ethernet/rdc/r6040.c @@ -9596,6 +9722,7 @@ M: Mark Brown L: linux-kernel@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git S: Supported +F: Documentation/devicetree/bindings/regmap/ F: drivers/base/regmap/ F: include/linux/regmap.h @@ -9680,7 +9807,6 @@ F: Documentation/ABI/*/sysfs-driver-hid-roccat* ROCKER DRIVER M: Jiri Pirko -M: Scott Feldman L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/rocker/ @@ -9997,6 +10123,7 @@ SERIAL DRIVERS M: Greg Kroah-Hartman L: linux-serial@vger.kernel.org S: Maintained +F: Documentation/devicetree/bindings/serial/ F: drivers/tty/serial/ SYNOPSYS DESIGNWARE DMAC DRIVER @@ -10277,10 +10404,9 @@ W: http://www.avagotech.com S: Supported F: drivers/scsi/be2iscsi/ -Emulex 10Gbps NIC BE2, BE3-R, Lancer, Skyhawk-R DRIVER +Emulex 10Gbps NIC BE2, BE3-R, Lancer, Skyhawk-R DRIVER (be2net) M: Sathya Perla M: Ajit Khaparde -M: Padmanabh Ratnakar M: Sriharsha Basavapatna M: Somnath Kotur L: netdev@vger.kernel.org @@ -10857,6 +10983,7 @@ STAGING - INDUSTRIAL IO M: Jonathan Cameron L: linux-iio@vger.kernel.org S: Odd Fixes +F: Documentation/devicetree/bindings/staging/iio/ F: drivers/staging/iio/ STAGING - LIRC (LINUX INFRARED REMOTE CONTROL) DRIVERS diff --git a/Makefile b/Makefile index 4fb6beac5f09..d384848478b9 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ VERSION = 4 PATCHLEVEL = 7 SUBLEVEL = 0 -EXTRAVERSION = -rc4 +EXTRAVERSION = NAME = Psychotic Stoned Sheep # *DOCUMENTATION* @@ -363,12 +363,14 @@ CHECK = sparse CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \ -Wbitwise -Wno-return-void $(CF) +NOSTDINC_FLAGS = CFLAGS_MODULE = AFLAGS_MODULE = LDFLAGS_MODULE = CFLAGS_KERNEL = AFLAGS_KERNEL = -CFLAGS_GCOV = -fprofile-arcs -ftest-coverage -fno-tree-loop-im -Wno-maybe-uninitialized +LDFLAGS_vmlinux = +CFLAGS_GCOV = -fprofile-arcs -ftest-coverage -fno-tree-loop-im CFLAGS_KCOV = -fsanitize-coverage=trace-pc @@ -617,12 +619,13 @@ ARCH_CFLAGS := include arch/$(SRCARCH)/Makefile KBUILD_CFLAGS += $(call cc-option,-fno-delete-null-pointer-checks,) +KBUILD_CFLAGS += $(call cc-disable-warning,maybe-uninitialized,) ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE -KBUILD_CFLAGS += -Os $(call cc-disable-warning,maybe-uninitialized,) +KBUILD_CFLAGS += -Os else ifdef CONFIG_PROFILE_ALL_BRANCHES -KBUILD_CFLAGS += -O2 $(call cc-disable-warning,maybe-uninitialized,) +KBUILD_CFLAGS += -O2 else KBUILD_CFLAGS += -O2 endif @@ -645,41 +648,28 @@ ifneq ($(CONFIG_FRAME_WARN),0) KBUILD_CFLAGS += $(call cc-option,-Wframe-larger-than=${CONFIG_FRAME_WARN}) endif -# Handle stack protector mode. -# -# Since kbuild can potentially perform two passes (first with the old -# .config values and then with updated .config values), we cannot error out -# if a desired compiler option is unsupported. If we were to error, kbuild -# could never get to the second pass and actually notice that we changed -# the option to something that was supported. -# -# Additionally, we don't want to fallback and/or silently change which compiler -# flags will be used, since that leads to producing kernels with different -# security feature characteristics depending on the compiler used. ("But I -# selected CC_STACKPROTECTOR_STRONG! Why did it build with _REGULAR?!") -# -# The middle ground is to warn here so that the failed option is obvious, but -# to let the build fail with bad compiler flags so that we can't produce a -# kernel when there is a CONFIG and compiler mismatch. -# +# This selects the stack protector compiler flag. Testing it is delayed +# until after .config has been reprocessed, in the prepare-compiler-check +# target. ifdef CONFIG_CC_STACKPROTECTOR_REGULAR stackp-flag := -fstack-protector - ifeq ($(call cc-option, $(stackp-flag)),) - $(warning Cannot use CONFIG_CC_STACKPROTECTOR_REGULAR: \ - -fstack-protector not supported by compiler) - endif + stackp-name := REGULAR else ifdef CONFIG_CC_STACKPROTECTOR_STRONG stackp-flag := -fstack-protector-strong - ifeq ($(call cc-option, $(stackp-flag)),) - $(warning Cannot use CONFIG_CC_STACKPROTECTOR_STRONG: \ - -fstack-protector-strong not supported by compiler) - endif + stackp-name := STRONG else # Force off for distro compilers that enable stack protector by default. stackp-flag := $(call cc-option, -fno-stack-protector) endif endif +# Find arch-specific stack protector compiler sanity-checking script. +ifdef CONFIG_CC_STACKPROTECTOR + stackp-path := $(srctree)/scripts/gcc-$(ARCH)_$(BITS)-has-stack-protector.sh + ifneq ($(wildcard $(stackp-path)),) + stackp-check := $(stackp-path) + endif +endif KBUILD_CFLAGS += $(stackp-flag) ifdef CONFIG_KCOV @@ -1015,8 +1005,10 @@ ifneq ($(KBUILD_SRC),) fi; endif -# prepare2 creates a makefile if using a separate output directory -prepare2: prepare3 outputmakefile asm-generic +# prepare2 creates a makefile if using a separate output directory. +# From this point forward, .config has been reprocessed, so any rules +# that need to depend on updated CONFIG_* values can be checked here. +prepare2: prepare3 prepare-compiler-check outputmakefile asm-generic prepare1: prepare2 $(version_h) include/generated/utsrelease.h \ include/config/auto.conf @@ -1038,7 +1030,7 @@ ifdef CONFIG_STACK_VALIDATION ifeq ($(has_libelf),1) objtool_target := tools/objtool FORCE else - $(warning "Cannot use CONFIG_STACK_VALIDATION, please install libelf-dev or elfutils-libelf-devel") + $(warning "Cannot use CONFIG_STACK_VALIDATION, please install libelf-dev, libelf-devel or elfutils-libelf-devel") SKIP_STACK_VALIDATION := 1 export SKIP_STACK_VALIDATION endif @@ -1047,6 +1039,32 @@ endif PHONY += prepare-objtool prepare-objtool: $(objtool_target) +# Check for CONFIG flags that require compiler support. Abort the build +# after .config has been processed, but before the kernel build starts. +# +# For security-sensitive CONFIG options, we don't want to fallback and/or +# silently change which compiler flags will be used, since that leads to +# producing kernels with different security feature characteristics +# depending on the compiler used. (For example, "But I selected +# CC_STACKPROTECTOR_STRONG! Why did it build with _REGULAR?!") +PHONY += prepare-compiler-check +prepare-compiler-check: FORCE +# Make sure compiler supports requested stack protector flag. +ifdef stackp-name + ifeq ($(call cc-option, $(stackp-flag)),) + @echo Cannot use CONFIG_CC_STACKPROTECTOR_$(stackp-name): \ + $(stackp-flag) not supported by compiler >&2 && exit 1 + endif +endif +# Make sure compiler does not have buggy stack-protector support. +ifdef stackp-check + ifneq ($(shell $(CONFIG_SHELL) $(stackp-check) $(CC) $(KBUILD_CPPFLAGS) $(biarch)),y) + @echo Cannot use CONFIG_CC_STACKPROTECTOR_$(stackp-name): \ + $(stackp-flag) available but compiler is broken >&2 && exit 1 + endif +endif + @: + # Generate some files # --------------------------------------------------------------------------- @@ -1364,6 +1382,8 @@ help: @$(MAKE) $(build)=$(package-dir) help @echo '' @echo 'Documentation targets:' + @$(MAKE) -f $(srctree)/Documentation/Makefile.sphinx dochelp + @echo '' @$(MAKE) -f $(srctree)/Documentation/DocBook/Makefile dochelp @echo '' @echo 'Architecture specific targets ($(SRCARCH)):' @@ -1412,8 +1432,11 @@ $(help-board-dirs): help-%: # Documentation targets # --------------------------------------------------------------------------- -%docs: scripts_basic FORCE +DOC_TARGETS := xmldocs sgmldocs psdocs pdfdocs htmldocs mandocs installmandocs epubdocs cleandocs cleanmediadocs +PHONY += $(DOC_TARGETS) +$(DOC_TARGETS): scripts_basic FORCE $(Q)$(MAKE) $(build)=scripts build_docproc build_check-lc_ctype + $(Q)$(MAKE) $(build)=Documentation -f $(srctree)/Documentation/Makefile.sphinx $@ $(Q)$(MAKE) $(build)=Documentation/DocBook $@ else # KBUILD_EXTMOD diff --git a/arch/Kconfig b/arch/Kconfig index e9734796531f..15996290fed4 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -226,8 +226,8 @@ config ARCH_INIT_TASK config ARCH_TASK_STRUCT_ALLOCATOR bool -# Select if arch has its private alloc_thread_info() function -config ARCH_THREAD_INFO_ALLOCATOR +# Select if arch has its private alloc_thread_stack() function +config ARCH_THREAD_STACK_ALLOCATOR bool # Select if arch wants to size task_struct dynamically via arch_task_struct_size: diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index 572b228c44c7..498933a7df97 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -46,10 +46,9 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \ } \ #define ATOMIC_OP_RETURN(op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ { \ long temp, result; \ - smp_mb(); \ __asm__ __volatile__( \ "1: ldl_l %0,%1\n" \ " " #asm_op " %0,%3,%2\n" \ @@ -61,7 +60,23 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ".previous" \ :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ :"Ir" (i), "m" (v->counter) : "memory"); \ - smp_mb(); \ + return result; \ +} + +#define ATOMIC_FETCH_OP(op, asm_op) \ +static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +{ \ + long temp, result; \ + __asm__ __volatile__( \ + "1: ldl_l %2,%1\n" \ + " " #asm_op " %2,%3,%0\n" \ + " stl_c %0,%1\n" \ + " beq %0,2f\n" \ + ".subsection 2\n" \ + "2: br 1b\n" \ + ".previous" \ + :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ + :"Ir" (i), "m" (v->counter) : "memory"); \ return result; \ } @@ -82,10 +97,9 @@ static __inline__ void atomic64_##op(long i, atomic64_t * v) \ } \ #define ATOMIC64_OP_RETURN(op, asm_op) \ -static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ +static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v) \ { \ long temp, result; \ - smp_mb(); \ __asm__ __volatile__( \ "1: ldq_l %0,%1\n" \ " " #asm_op " %0,%3,%2\n" \ @@ -97,34 +111,77 @@ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ ".previous" \ :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ :"Ir" (i), "m" (v->counter) : "memory"); \ - smp_mb(); \ + return result; \ +} + +#define ATOMIC64_FETCH_OP(op, asm_op) \ +static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \ +{ \ + long temp, result; \ + __asm__ __volatile__( \ + "1: ldq_l %2,%1\n" \ + " " #asm_op " %2,%3,%0\n" \ + " stq_c %0,%1\n" \ + " beq %0,2f\n" \ + ".subsection 2\n" \ + "2: br 1b\n" \ + ".previous" \ + :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ + :"Ir" (i), "m" (v->counter) : "memory"); \ return result; \ } #define ATOMIC_OPS(op) \ ATOMIC_OP(op, op##l) \ ATOMIC_OP_RETURN(op, op##l) \ + ATOMIC_FETCH_OP(op, op##l) \ ATOMIC64_OP(op, op##q) \ - ATOMIC64_OP_RETURN(op, op##q) + ATOMIC64_OP_RETURN(op, op##q) \ + ATOMIC64_FETCH_OP(op, op##q) ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define atomic_add_return_relaxed atomic_add_return_relaxed +#define atomic_sub_return_relaxed atomic_sub_return_relaxed +#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed +#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed + +#define atomic64_add_return_relaxed atomic64_add_return_relaxed +#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed +#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed +#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed + #define atomic_andnot atomic_andnot #define atomic64_andnot atomic64_andnot -ATOMIC_OP(and, and) -ATOMIC_OP(andnot, bic) -ATOMIC_OP(or, bis) -ATOMIC_OP(xor, xor) -ATOMIC64_OP(and, and) -ATOMIC64_OP(andnot, bic) -ATOMIC64_OP(or, bis) -ATOMIC64_OP(xor, xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, asm) \ + ATOMIC_OP(op, asm) \ + ATOMIC_FETCH_OP(op, asm) \ + ATOMIC64_OP(op, asm) \ + ATOMIC64_FETCH_OP(op, asm) + +ATOMIC_OPS(and, and) +ATOMIC_OPS(andnot, bic) +ATOMIC_OPS(or, bis) +ATOMIC_OPS(xor, xor) + +#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed +#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed +#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed +#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed + +#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed +#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed +#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed +#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed #undef ATOMIC_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/alpha/include/asm/pgalloc.h b/arch/alpha/include/asm/pgalloc.h index aab14a019c20..c2ebb6f36c9d 100644 --- a/arch/alpha/include/asm/pgalloc.h +++ b/arch/alpha/include/asm/pgalloc.h @@ -40,7 +40,7 @@ pgd_free(struct mm_struct *mm, pgd_t *pgd) static inline pmd_t * pmd_alloc_one(struct mm_struct *mm, unsigned long address) { - pmd_t *ret = (pmd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pmd_t *ret = (pmd_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); return ret; } @@ -53,7 +53,7 @@ pmd_free(struct mm_struct *mm, pmd_t *pmd) static inline pte_t * pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); return pte; } diff --git a/arch/alpha/include/asm/rwsem.h b/arch/alpha/include/asm/rwsem.h index 0131a7058778..77873d0ad293 100644 --- a/arch/alpha/include/asm/rwsem.h +++ b/arch/alpha/include/asm/rwsem.h @@ -25,8 +25,8 @@ static inline void __down_read(struct rw_semaphore *sem) { long oldcount; #ifndef CONFIG_SMP - oldcount = sem->count; - sem->count += RWSEM_ACTIVE_READ_BIAS; + oldcount = sem->count.counter; + sem->count.counter += RWSEM_ACTIVE_READ_BIAS; #else long temp; __asm__ __volatile__( @@ -52,13 +52,13 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) { long old, new, res; - res = sem->count; + res = atomic_long_read(&sem->count); do { new = res + RWSEM_ACTIVE_READ_BIAS; if (new <= 0) break; old = res; - res = cmpxchg(&sem->count, old, new); + res = atomic_long_cmpxchg(&sem->count, old, new); } while (res != old); return res >= 0 ? 1 : 0; } @@ -67,8 +67,8 @@ static inline long ___down_write(struct rw_semaphore *sem) { long oldcount; #ifndef CONFIG_SMP - oldcount = sem->count; - sem->count += RWSEM_ACTIVE_WRITE_BIAS; + oldcount = sem->count.counter; + sem->count.counter += RWSEM_ACTIVE_WRITE_BIAS; #else long temp; __asm__ __volatile__( @@ -106,7 +106,7 @@ static inline int __down_write_killable(struct rw_semaphore *sem) */ static inline int __down_write_trylock(struct rw_semaphore *sem) { - long ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE, + long ret = atomic_long_cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE, RWSEM_ACTIVE_WRITE_BIAS); if (ret == RWSEM_UNLOCKED_VALUE) return 1; @@ -117,8 +117,8 @@ static inline void __up_read(struct rw_semaphore *sem) { long oldcount; #ifndef CONFIG_SMP - oldcount = sem->count; - sem->count -= RWSEM_ACTIVE_READ_BIAS; + oldcount = sem->count.counter; + sem->count.counter -= RWSEM_ACTIVE_READ_BIAS; #else long temp; __asm__ __volatile__( @@ -142,8 +142,8 @@ static inline void __up_write(struct rw_semaphore *sem) { long count; #ifndef CONFIG_SMP - sem->count -= RWSEM_ACTIVE_WRITE_BIAS; - count = sem->count; + sem->count.counter -= RWSEM_ACTIVE_WRITE_BIAS; + count = sem->count.counter; #else long temp; __asm__ __volatile__( @@ -171,8 +171,8 @@ static inline void __downgrade_write(struct rw_semaphore *sem) { long oldcount; #ifndef CONFIG_SMP - oldcount = sem->count; - sem->count -= RWSEM_WAITING_BIAS; + oldcount = sem->count.counter; + sem->count.counter -= RWSEM_WAITING_BIAS; #else long temp; __asm__ __volatile__( @@ -191,47 +191,5 @@ static inline void __downgrade_write(struct rw_semaphore *sem) rwsem_downgrade_wake(sem); } -static inline void rwsem_atomic_add(long val, struct rw_semaphore *sem) -{ -#ifndef CONFIG_SMP - sem->count += val; -#else - long temp; - __asm__ __volatile__( - "1: ldq_l %0,%1\n" - " addq %0,%2,%0\n" - " stq_c %0,%1\n" - " beq %0,2f\n" - ".subsection 2\n" - "2: br 1b\n" - ".previous" - :"=&r" (temp), "=m" (sem->count) - :"Ir" (val), "m" (sem->count)); -#endif -} - -static inline long rwsem_atomic_update(long val, struct rw_semaphore *sem) -{ -#ifndef CONFIG_SMP - sem->count += val; - return sem->count; -#else - long ret, temp; - __asm__ __volatile__( - "1: ldq_l %0,%1\n" - " addq %0,%3,%2\n" - " addq %0,%3,%0\n" - " stq_c %2,%1\n" - " beq %2,2f\n" - ".subsection 2\n" - "2: br 1b\n" - ".previous" - :"=&r" (ret), "=m" (sem->count), "=&r" (temp) - :"Ir" (val), "m" (sem->count)); - - return ret; -#endif -} - #endif /* __KERNEL__ */ #endif /* _ALPHA_RWSEM_H */ diff --git a/arch/alpha/include/asm/spinlock.h b/arch/alpha/include/asm/spinlock.h index fed9c6f44c19..a40b9fc0c6c3 100644 --- a/arch/alpha/include/asm/spinlock.h +++ b/arch/alpha/include/asm/spinlock.h @@ -3,6 +3,8 @@ #include #include +#include +#include /* * Simple spin lock operations. There are two variants, one clears IRQ's @@ -13,8 +15,11 @@ #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) #define arch_spin_is_locked(x) ((x)->lock != 0) -#define arch_spin_unlock_wait(x) \ - do { cpu_relax(); } while ((x)->lock) + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->lock, !VAL); +} static inline int arch_spin_value_unlocked(arch_spinlock_t lock) { diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index 4a905bd667e2..83e9eee57a55 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c @@ -147,7 +147,7 @@ retry: /* If for any reason at all we couldn't handle the fault, make sure we exit gracefully rather than endlessly redo the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/arc/Makefile b/arch/arc/Makefile index d4df6be66d58..601ed173080b 100644 --- a/arch/arc/Makefile +++ b/arch/arc/Makefile @@ -66,8 +66,6 @@ endif endif -cflags-$(CONFIG_ARC_DW2_UNWIND) += -fasynchronous-unwind-tables - # By default gcc 4.8 generates dwarf4 which kernel unwinder can't grok ifeq ($(atleast_gcc48),y) cflags-$(CONFIG_ARC_DW2_UNWIND) += -gdwarf-2 @@ -76,9 +74,7 @@ endif ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE # Generic build system uses -O2, we want -O3 # Note: No need to add to cflags-y as that happens anyways -# -# Disable the false maybe-uninitialized warings gcc spits out at -O3 -ARCH_CFLAGS += -O3 $(call cc-disable-warning,maybe-uninitialized,) +ARCH_CFLAGS += -O3 endif # small data is default for elf32 tool-chain. If not usable, disable it diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h index dd683995bc9d..4e3c1b6b0806 100644 --- a/arch/arc/include/asm/atomic.h +++ b/arch/arc/include/asm/atomic.h @@ -67,6 +67,33 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return val; \ } +#define ATOMIC_FETCH_OP(op, c_op, asm_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned int val, orig; \ + \ + /* \ + * Explicit full memory barrier needed before/after as \ + * LLOCK/SCOND thmeselves don't provide any such semantics \ + */ \ + smp_mb(); \ + \ + __asm__ __volatile__( \ + "1: llock %[orig], [%[ctr]] \n" \ + " " #asm_op " %[val], %[orig], %[i] \n" \ + " scond %[val], [%[ctr]] \n" \ + " \n" \ + : [val] "=&r" (val), \ + [orig] "=&r" (orig) \ + : [ctr] "r" (&v->counter), \ + [i] "ir" (i) \ + : "cc"); \ + \ + smp_mb(); \ + \ + return orig; \ +} + #else /* !CONFIG_ARC_HAS_LLSC */ #ifndef CONFIG_SMP @@ -129,25 +156,44 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return temp; \ } +#define ATOMIC_FETCH_OP(op, c_op, asm_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned long flags; \ + unsigned long orig; \ + \ + /* \ + * spin lock/unlock provides the needed smp_mb() before/after \ + */ \ + atomic_ops_lock(flags); \ + orig = v->counter; \ + v->counter c_op i; \ + atomic_ops_unlock(flags); \ + \ + return orig; \ +} + #endif /* !CONFIG_ARC_HAS_LLSC */ #define ATOMIC_OPS(op, c_op, asm_op) \ ATOMIC_OP(op, c_op, asm_op) \ - ATOMIC_OP_RETURN(op, c_op, asm_op) + ATOMIC_OP_RETURN(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) #define atomic_andnot atomic_andnot -ATOMIC_OP(and, &=, and) -ATOMIC_OP(andnot, &= ~, bic) -ATOMIC_OP(or, |=, or) -ATOMIC_OP(xor, ^=, xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op, asm_op) \ + ATOMIC_OP(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) -#undef SCOND_FAIL_RETRY_VAR_DEF -#undef SCOND_FAIL_RETRY_ASM -#undef SCOND_FAIL_RETRY_VARS +ATOMIC_OPS(and, &=, and) +ATOMIC_OPS(andnot, &= ~, bic) +ATOMIC_OPS(or, |=, or) +ATOMIC_OPS(xor, ^=, xor) #else /* CONFIG_ARC_PLAT_EZNPS */ @@ -208,22 +254,51 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return temp; \ } +#define ATOMIC_FETCH_OP(op, c_op, asm_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned int temp = i; \ + \ + /* Explicit full memory barrier needed before/after */ \ + smp_mb(); \ + \ + __asm__ __volatile__( \ + " mov r2, %0\n" \ + " mov r3, %1\n" \ + " .word %2\n" \ + " mov %0, r2" \ + : "+r"(temp) \ + : "r"(&v->counter), "i"(asm_op) \ + : "r2", "r3", "memory"); \ + \ + smp_mb(); \ + \ + return temp; \ +} + #define ATOMIC_OPS(op, c_op, asm_op) \ ATOMIC_OP(op, c_op, asm_op) \ - ATOMIC_OP_RETURN(op, c_op, asm_op) + ATOMIC_OP_RETURN(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3) #define atomic_sub(i, v) atomic_add(-(i), (v)) #define atomic_sub_return(i, v) atomic_add_return(-(i), (v)) -ATOMIC_OP(and, &=, CTOP_INST_AAND_DI_R2_R2_R3) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op, asm_op) \ + ATOMIC_OP(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) + +ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3) #define atomic_andnot(mask, v) atomic_and(~(mask), (v)) -ATOMIC_OP(or, |=, CTOP_INST_AOR_DI_R2_R2_R3) -ATOMIC_OP(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) +ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3) +ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) #endif /* CONFIG_ARC_PLAT_EZNPS */ #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h index 86ed671286df..3749234b7419 100644 --- a/arch/arc/include/asm/pgalloc.h +++ b/arch/arc/include/asm/pgalloc.h @@ -95,7 +95,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, { pte_t *pte; - pte = (pte_t *) __get_free_pages(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO, + pte = (pte_t *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, __get_order_pte()); return pte; @@ -107,7 +107,7 @@ pte_alloc_one(struct mm_struct *mm, unsigned long address) pgtable_t pte_pg; struct page *page; - pte_pg = (pgtable_t)__get_free_pages(GFP_KERNEL | __GFP_REPEAT, __get_order_pte()); + pte_pg = (pgtable_t)__get_free_pages(GFP_KERNEL, __get_order_pte()); if (!pte_pg) return 0; memzero((void *)pte_pg, PTRS_PER_PTE * sizeof(pte_t)); diff --git a/arch/arc/include/asm/spinlock.h b/arch/arc/include/asm/spinlock.h index cded4a9b5438..233d5ffe6ec7 100644 --- a/arch/arc/include/asm/spinlock.h +++ b/arch/arc/include/asm/spinlock.h @@ -15,8 +15,11 @@ #define arch_spin_is_locked(x) ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__) #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) -#define arch_spin_unlock_wait(x) \ - do { while (arch_spin_is_locked(x)) cpu_relax(); } while (0) + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->slock, !VAL); +} #ifdef CONFIG_ARC_HAS_LLSC diff --git a/arch/arc/kernel/stacktrace.c b/arch/arc/kernel/stacktrace.c index e0efff15a5ae..b9192a653b7e 100644 --- a/arch/arc/kernel/stacktrace.c +++ b/arch/arc/kernel/stacktrace.c @@ -142,7 +142,7 @@ arc_unwind_core(struct task_struct *tsk, struct pt_regs *regs, * prelogue is setup (callee regs saved and then fp set and not other * way around */ - pr_warn("CONFIG_ARC_DW2_UNWIND needs to be enabled\n"); + pr_warn_once("CONFIG_ARC_DW2_UNWIND needs to be enabled\n"); return 0; #endif diff --git a/arch/arc/kernel/time.c b/arch/arc/kernel/time.c index 4549ab255dd1..98f22d2eb563 100644 --- a/arch/arc/kernel/time.c +++ b/arch/arc/kernel/time.c @@ -116,19 +116,19 @@ static struct clocksource arc_counter_gfrc = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -static void __init arc_cs_setup_gfrc(struct device_node *node) +static int __init arc_cs_setup_gfrc(struct device_node *node) { int exists = cpuinfo_arc700[0].extn.gfrc; int ret; if (WARN(!exists, "Global-64-bit-Ctr clocksource not detected")) - return; + return -ENXIO; ret = arc_get_timer_clk(node); if (ret) - return; + return ret; - clocksource_register_hz(&arc_counter_gfrc, arc_timer_freq); + return clocksource_register_hz(&arc_counter_gfrc, arc_timer_freq); } CLOCKSOURCE_OF_DECLARE(arc_gfrc, "snps,archs-timer-gfrc", arc_cs_setup_gfrc); @@ -172,25 +172,25 @@ static struct clocksource arc_counter_rtc = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -static void __init arc_cs_setup_rtc(struct device_node *node) +static int __init arc_cs_setup_rtc(struct device_node *node) { int exists = cpuinfo_arc700[smp_processor_id()].extn.rtc; int ret; if (WARN(!exists, "Local-64-bit-Ctr clocksource not detected")) - return; + return -ENXIO; /* Local to CPU hence not usable in SMP */ if (WARN(IS_ENABLED(CONFIG_SMP), "Local-64-bit-Ctr not usable in SMP")) - return; + return -EINVAL; ret = arc_get_timer_clk(node); if (ret) - return; + return ret; write_aux_reg(AUX_RTC_CTRL, 1); - clocksource_register_hz(&arc_counter_rtc, arc_timer_freq); + return clocksource_register_hz(&arc_counter_rtc, arc_timer_freq); } CLOCKSOURCE_OF_DECLARE(arc_rtc, "snps,archs-timer-rtc", arc_cs_setup_rtc); @@ -213,23 +213,23 @@ static struct clocksource arc_counter_timer1 = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -static void __init arc_cs_setup_timer1(struct device_node *node) +static int __init arc_cs_setup_timer1(struct device_node *node) { int ret; /* Local to CPU hence not usable in SMP */ if (IS_ENABLED(CONFIG_SMP)) - return; + return -EINVAL; ret = arc_get_timer_clk(node); if (ret) - return; + return ret; write_aux_reg(ARC_REG_TIMER1_LIMIT, ARC_TIMER_MAX); write_aux_reg(ARC_REG_TIMER1_CNT, 0); write_aux_reg(ARC_REG_TIMER1_CTRL, TIMER_CTRL_NH); - clocksource_register_hz(&arc_counter_timer1, arc_timer_freq); + return clocksource_register_hz(&arc_counter_timer1, arc_timer_freq); } /********** Clock Event Device *********/ @@ -324,20 +324,28 @@ static struct notifier_block arc_timer_cpu_nb = { /* * clockevent setup for boot CPU */ -static void __init arc_clockevent_setup(struct device_node *node) +static int __init arc_clockevent_setup(struct device_node *node) { struct clock_event_device *evt = this_cpu_ptr(&arc_clockevent_device); int ret; - register_cpu_notifier(&arc_timer_cpu_nb); + ret = register_cpu_notifier(&arc_timer_cpu_nb); + if (ret) { + pr_err("Failed to register cpu notifier"); + return ret; + } arc_timer_irq = irq_of_parse_and_map(node, 0); - if (arc_timer_irq <= 0) - panic("clockevent: missing irq"); + if (arc_timer_irq <= 0) { + pr_err("clockevent: missing irq"); + return -EINVAL; + } ret = arc_get_timer_clk(node); - if (ret) - panic("clockevent: missing clk"); + if (ret) { + pr_err("clockevent: missing clk"); + return ret; + } evt->irq = arc_timer_irq; evt->cpumask = cpumask_of(smp_processor_id()); @@ -347,22 +355,29 @@ static void __init arc_clockevent_setup(struct device_node *node) /* Needs apriori irq_set_percpu_devid() done in intc map function */ ret = request_percpu_irq(arc_timer_irq, timer_irq_handler, "Timer0 (per-cpu-tick)", evt); - if (ret) - panic("clockevent: unable to request irq\n"); + if (ret) { + pr_err("clockevent: unable to request irq\n"); + return ret; + } enable_percpu_irq(arc_timer_irq, 0); + + return 0; } -static void __init arc_of_timer_init(struct device_node *np) +static int __init arc_of_timer_init(struct device_node *np) { static int init_count = 0; + int ret; if (!init_count) { init_count = 1; - arc_clockevent_setup(np); + ret = arc_clockevent_setup(np); } else { - arc_cs_setup_timer1(np); + ret = arc_cs_setup_timer1(np); } + + return ret; } CLOCKSOURCE_OF_DECLARE(arc_clkevt, "snps,arc-timer", arc_of_timer_init); diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index af63f4a13e60..e94e5aa33985 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -137,7 +137,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); /* If Pagefault was interrupted by SIGKILL, exit page fault "early" */ if (unlikely(fatal_signal_pending(current))) { diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 90542db1220d..f0636ec94903 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -358,10 +358,10 @@ config ARCH_CLPS711X bool "Cirrus Logic CLPS711x/EP721x/EP731x-based" select ARCH_REQUIRE_GPIOLIB select AUTO_ZRELADDR - select CLKSRC_MMIO select COMMON_CLK select CPU_ARM720T select GENERIC_CLOCKEVENTS + select CLPS711X_TIMER select MFD_SYSCON select SOC_BUS help diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi index 52be48bbd2dd..7fa295155543 100644 --- a/arch/arm/boot/dts/am33xx.dtsi +++ b/arch/arm/boot/dts/am33xx.dtsi @@ -766,7 +766,6 @@ ale_entries = <1024>; bd_ram_size = <0x2000>; no_bd_ram = <0>; - rx_descs = <64>; mac_control = <0x20>; slaves = <2>; active_slave = <0>; @@ -789,7 +788,7 @@ status = "disabled"; davinci_mdio: mdio@4a101000 { - compatible = "ti,davinci_mdio"; + compatible = "ti,cpsw-mdio","ti,davinci_mdio"; #address-cells = <1>; #size-cells = <0>; ti,hwmods = "davinci_mdio"; diff --git a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi index 12fcde4d4d2e..cd81ecf12731 100644 --- a/arch/arm/boot/dts/am4372.dtsi +++ b/arch/arm/boot/dts/am4372.dtsi @@ -626,7 +626,6 @@ ale_entries = <1024>; bd_ram_size = <0x2000>; no_bd_ram = <0>; - rx_descs = <64>; mac_control = <0x20>; slaves = <2>; active_slave = <0>; @@ -636,7 +635,7 @@ syscon = <&scm_conf>; davinci_mdio: mdio@4a101000 { - compatible = "ti,am4372-mdio","ti,davinci_mdio"; + compatible = "ti,am4372-mdio","ti,cpsw-mdio","ti,davinci_mdio"; reg = <0x4a101000 0x100>; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm/boot/dts/armada-385-linksys.dtsi b/arch/arm/boot/dts/armada-385-linksys.dtsi index 8450944b28e6..22f7a13e20b4 100644 --- a/arch/arm/boot/dts/armada-385-linksys.dtsi +++ b/arch/arm/boot/dts/armada-385-linksys.dtsi @@ -58,8 +58,8 @@ soc { ranges = ; + MBUS_ID(0x09, 0x19) 0 0xf1100000 0x10000 + MBUS_ID(0x09, 0x15) 0 0xf1110000 0x10000>; internal-regs { diff --git a/arch/arm/boot/dts/bcm-nsp.dtsi b/arch/arm/boot/dts/bcm-nsp.dtsi index def9e783b5c6..6a40ed7d0502 100644 --- a/arch/arm/boot/dts/bcm-nsp.dtsi +++ b/arch/arm/boot/dts/bcm-nsp.dtsi @@ -206,6 +206,11 @@ brcm,nand-has-wp; }; + rng: rng@33000 { + compatible = "brcm,bcm-nsp-rng"; + reg = <0x33000 0x14>; + }; + ccbtimer0: timer@34000 { compatible = "arm,sp804"; reg = <0x34000 0x1000>; @@ -266,6 +271,48 @@ <0x30028 0x04>, <0x3f408 0x04>; }; + + sata_phy: sata_phy@40100 { + compatible = "brcm,iproc-nsp-sata-phy"; + reg = <0x40100 0x340>; + reg-names = "phy"; + #address-cells = <1>; + #size-cells = <0>; + + sata_phy0: sata-phy@0 { + reg = <0>; + #phy-cells = <0>; + status = "disabled"; + }; + + sata_phy1: sata-phy@1 { + reg = <1>; + #phy-cells = <0>; + status = "disabled"; + }; + }; + + sata: ahci@41000 { + compatible = "brcm,bcm-nsp-ahci"; + reg-names = "ahci", "top-ctrl"; + reg = <0x41000 0x1000>, <0x40020 0x1c>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + sata0: sata-port@0 { + reg = <0>; + phys = <&sata_phy0>; + phy-names = "sata-phy"; + }; + + sata1: sata-port@1 { + reg = <1>; + phys = <&sata_phy1>; + phy-names = "sata-phy"; + }; + }; }; pcie0: pcie@18012000 { diff --git a/arch/arm/boot/dts/bcm958625k.dts b/arch/arm/boot/dts/bcm958625k.dts index e298450b49b2..2d8422632b2b 100644 --- a/arch/arm/boot/dts/bcm958625k.dts +++ b/arch/arm/boot/dts/bcm958625k.dts @@ -68,6 +68,18 @@ status = "okay"; }; +&sata_phy0 { + status = "okay"; +}; + +&sata_phy1 { + status = "okay"; +}; + +&sata { + status = "okay"; +}; + &nand { nandcs@0 { compatible = "brcm,nandcs"; diff --git a/arch/arm/boot/dts/dm814x.dtsi b/arch/arm/boot/dts/dm814x.dtsi index d4537dc61497..f23cae0c2179 100644 --- a/arch/arm/boot/dts/dm814x.dtsi +++ b/arch/arm/boot/dts/dm814x.dtsi @@ -509,7 +509,6 @@ ale_entries = <1024>; bd_ram_size = <0x2000>; no_bd_ram = <0>; - rx_descs = <64>; mac_control = <0x20>; slaves = <2>; active_slave = <0>; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index 3a8f3976f6f9..de559f6e4fee 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -1628,7 +1628,6 @@ ale_entries = <1024>; bd_ram_size = <0x2000>; no_bd_ram = <0>; - rx_descs = <64>; mac_control = <0x20>; slaves = <2>; active_slave = <0>; @@ -1663,7 +1662,7 @@ status = "disabled"; davinci_mdio: mdio@48485000 { - compatible = "ti,davinci_mdio"; + compatible = "ti,cpsw-mdio","ti,davinci_mdio"; #address-cells = <1>; #size-cells = <0>; ti,hwmods = "davinci_mdio"; diff --git a/arch/arm/boot/dts/meson8-minix-neo-x8.dts b/arch/arm/boot/dts/meson8-minix-neo-x8.dts index 4f536bb1f002..8bceb8d343f6 100644 --- a/arch/arm/boot/dts/meson8-minix-neo-x8.dts +++ b/arch/arm/boot/dts/meson8-minix-neo-x8.dts @@ -80,6 +80,7 @@ pmic@32 { compatible = "ricoh,rn5t618"; reg = <0x32>; + system-power-controller; regulators { }; diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi index 3b44ef3cff12..3ebee530f2b0 100644 --- a/arch/arm/boot/dts/rk3288.dtsi +++ b/arch/arm/boot/dts/rk3288.dtsi @@ -539,8 +539,9 @@ gmac: ethernet@ff290000 { compatible = "rockchip,rk3288-gmac"; reg = <0xff290000 0x10000>; - interrupts = ; - interrupt-names = "macirq"; + interrupts = , + ; + interrupt-names = "macirq", "eth_wake_irq"; rockchip,grf = <&grf>; clocks = <&cru SCLK_MAC>, <&cru SCLK_MAC_RX>, <&cru SCLK_MAC_TX>, diff --git a/arch/arm/boot/dts/sama5d2.dtsi b/arch/arm/boot/dts/sama5d2.dtsi index 2827e7ab5ebc..5dd2734e67ba 100644 --- a/arch/arm/boot/dts/sama5d2.dtsi +++ b/arch/arm/boot/dts/sama5d2.dtsi @@ -232,7 +232,7 @@ }; usb1: ohci@00400000 { - compatible = "atmel,at91rm9200-ohci", "usb-ohci"; + compatible = "atmel,sama5d2-ohci", "usb-ohci"; reg = <0x00400000 0x100000>; interrupts = <41 IRQ_TYPE_LEVEL_HIGH 2>; clocks = <&uhphs_clk>, <&uhphs_clk>, <&uhpck>; diff --git a/arch/arm/boot/dts/socfpga_arria10.dtsi b/arch/arm/boot/dts/socfpga_arria10.dtsi index 17e81dc9213e..5820b70c95b3 100644 --- a/arch/arm/boot/dts/socfpga_arria10.dtsi +++ b/arch/arm/boot/dts/socfpga_arria10.dtsi @@ -621,6 +621,22 @@ compatible = "altr,socfpga-a10-ocram-ecc"; reg = <0xff8c3000 0x400>; }; + + emac0-rx-ecc@ff8c0800 { + compatible = "altr,socfpga-eth-mac-ecc"; + reg = <0xff8c0800 0x400>; + altr,ecc-parent = <&gmac0>; + interrupts = <4 IRQ_TYPE_LEVEL_HIGH>, + <36 IRQ_TYPE_LEVEL_HIGH>; + }; + + emac0-tx-ecc@ff8c0c00 { + compatible = "altr,socfpga-eth-mac-ecc"; + reg = <0xff8c0c00 0x400>; + altr,ecc-parent = <&gmac0>; + interrupts = <5 IRQ_TYPE_LEVEL_HIGH>, + <37 IRQ_TYPE_LEVEL_HIGH>; + }; }; rst: rstmgr@ffd05000 { diff --git a/arch/arm/boot/dts/sun4i-a10.dtsi b/arch/arm/boot/dts/sun4i-a10.dtsi index a03e56fb5dbc..ca58eb279d55 100644 --- a/arch/arm/boot/dts/sun4i-a10.dtsi +++ b/arch/arm/boot/dts/sun4i-a10.dtsi @@ -65,8 +65,9 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_be0-lcd0-hdmi"; - clocks = <&pll5 1>, <&ahb_gates 36>, <&ahb_gates 43>, - <&ahb_gates 44>, <&dram_gates 26>; + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 36>, + <&ahb_gates 43>, <&ahb_gates 44>, + <&dram_gates 26>; status = "disabled"; }; @@ -74,8 +75,9 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_fe0-de_be0-lcd0-hdmi"; - clocks = <&pll5 1>, <&ahb_gates 36>, <&ahb_gates 43>, - <&ahb_gates 44>, <&ahb_gates 46>, + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 36>, + <&ahb_gates 43>, <&ahb_gates 44>, + <&ahb_gates 46>, <&dram_gates 25>, <&dram_gates 26>; status = "disabled"; }; @@ -84,9 +86,9 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_fe0-de_be0-lcd0"; - clocks = <&pll5 1>, <&ahb_gates 36>, <&ahb_gates 44>, - <&ahb_gates 46>, <&dram_gates 25>, - <&dram_gates 26>; + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 36>, + <&ahb_gates 44>, <&ahb_gates 46>, + <&dram_gates 25>, <&dram_gates 26>; status = "disabled"; }; @@ -94,8 +96,9 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_fe0-de_be0-lcd0-tve0"; - clocks = <&pll5 1>, <&ahb_gates 34>, <&ahb_gates 36>, - <&ahb_gates 44>, <&ahb_gates 46>, + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 34>, + <&ahb_gates 36>, <&ahb_gates 44>, + <&ahb_gates 46>, <&dram_gates 5>, <&dram_gates 25>, <&dram_gates 26>; status = "disabled"; }; diff --git a/arch/arm/boot/dts/sun5i-a10s.dtsi b/arch/arm/boot/dts/sun5i-a10s.dtsi index bddd0de88af6..367f33012493 100644 --- a/arch/arm/boot/dts/sun5i-a10s.dtsi +++ b/arch/arm/boot/dts/sun5i-a10s.dtsi @@ -65,8 +65,8 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_be0-lcd0-hdmi"; - clocks = <&pll5 1>, <&ahb_gates 36>, <&ahb_gates 43>, - <&ahb_gates 44>; + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 36>, + <&ahb_gates 43>, <&ahb_gates 44>; status = "disabled"; }; @@ -74,7 +74,8 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_be0-lcd0"; - clocks = <&pll5 1>, <&ahb_gates 36>, <&ahb_gates 44>; + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 36>, + <&ahb_gates 44>; status = "disabled"; }; @@ -82,8 +83,8 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_be0-lcd0-tve0"; - clocks = <&pll5 1>, <&ahb_gates 34>, <&ahb_gates 36>, - <&ahb_gates 44>; + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 34>, + <&ahb_gates 36>, <&ahb_gates 44>; status = "disabled"; }; }; diff --git a/arch/arm/boot/dts/sun5i-r8-chip.dts b/arch/arm/boot/dts/sun5i-r8-chip.dts index a8d8b4582397..f694482bdeb6 100644 --- a/arch/arm/boot/dts/sun5i-r8-chip.dts +++ b/arch/arm/boot/dts/sun5i-r8-chip.dts @@ -52,7 +52,7 @@ / { model = "NextThing C.H.I.P."; - compatible = "nextthing,chip", "allwinner,sun5i-r8"; + compatible = "nextthing,chip", "allwinner,sun5i-r8", "allwinner,sun5i-a13"; aliases { i2c0 = &i2c0; diff --git a/arch/arm/boot/dts/sun7i-a20.dtsi b/arch/arm/boot/dts/sun7i-a20.dtsi index febdf4c72fb0..2c34bbbb9570 100644 --- a/arch/arm/boot/dts/sun7i-a20.dtsi +++ b/arch/arm/boot/dts/sun7i-a20.dtsi @@ -67,8 +67,9 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_be0-lcd0-hdmi"; - clocks = <&pll5 1>, <&ahb_gates 36>, <&ahb_gates 43>, - <&ahb_gates 44>, <&dram_gates 26>; + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 36>, + <&ahb_gates 43>, <&ahb_gates 44>, + <&dram_gates 26>; status = "disabled"; }; @@ -76,8 +77,8 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_be0-lcd0"; - clocks = <&pll5 1>, <&ahb_gates 36>, <&ahb_gates 44>, - <&dram_gates 26>; + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 36>, + <&ahb_gates 44>, <&dram_gates 26>; status = "disabled"; }; @@ -85,7 +86,7 @@ compatible = "allwinner,simple-framebuffer", "simple-framebuffer"; allwinner,pipeline = "de_be0-lcd0-tve0"; - clocks = <&pll5 1>, + clocks = <&pll3>, <&pll5 1>, <&ahb_gates 34>, <&ahb_gates 36>, <&ahb_gates 44>, <&dram_gates 5>, <&dram_gates 26>; status = "disabled"; @@ -231,6 +232,7 @@ pll3x2: pll3x2_clk { #clock-cells = <0>; compatible = "fixed-factor-clock"; + clocks = <&pll3>; clock-div = <1>; clock-mult = <2>; clock-output-names = "pll3-2x"; @@ -272,6 +274,7 @@ pll7x2: pll7x2_clk { #clock-cells = <0>; compatible = "fixed-factor-clock"; + clocks = <&pll7>; clock-div = <1>; clock-mult = <2>; clock-output-names = "pll7-2x"; diff --git a/arch/arm/boot/dts/tegra30-beaver.dts b/arch/arm/boot/dts/tegra30-beaver.dts index 1eca3b28ac64..b6da15d823a6 100644 --- a/arch/arm/boot/dts/tegra30-beaver.dts +++ b/arch/arm/boot/dts/tegra30-beaver.dts @@ -1843,7 +1843,7 @@ ldo5_reg: ldo5 { regulator-name = "vddio_sdmmc,avdd_vdac"; - regulator-min-microvolt = <3300000>; + regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; regulator-always-on; }; @@ -1914,6 +1914,7 @@ sdhci@78000000 { status = "okay"; + vqmmc-supply = <&ldo5_reg>; cd-gpios = <&gpio TEGRA_GPIO(I, 5) GPIO_ACTIVE_LOW>; wp-gpios = <&gpio TEGRA_GPIO(T, 3) GPIO_ACTIVE_HIGH>; power-gpios = <&gpio TEGRA_GPIO(D, 7) GPIO_ACTIVE_HIGH>; diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts index 6c60b7f91104..5c1fcab4a6f7 100644 --- a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts +++ b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts @@ -85,187 +85,199 @@ reg = <1>; #address-cells = <1>; #size-cells = <0>; + + switch0: switch0@0 { + compatible = "marvell,mv88e6085"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + dsa,member = <0 0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + label = "lan0"; + }; + + port@1 { + reg = <1>; + label = "lan1"; + }; + + port@2 { + reg = <2>; + label = "lan2"; + }; + + switch0port5: port@5 { + reg = <5>; + label = "dsa"; + phy-mode = "rgmii-txid"; + link = <&switch1port6 + &switch2port9>; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; + + port@6 { + reg = <6>; + label = "cpu"; + ethernet = <&fec1>; + fixed-link { + speed = <100>; + full-duplex; + }; + }; + }; + }; }; mdio_mux_2: mdio@2 { reg = <2>; #address-cells = <1>; #size-cells = <0>; - }; - - mdio_mux_4: mdio@4 { - reg = <4>; - #address-cells = <1>; - #size-cells = <0>; - }; - - mdio_mux_8: mdio@8 { - reg = <8>; - #address-cells = <1>; - #size-cells = <0>; - }; - }; - - dsa { - compatible = "marvell,dsa"; - #address-cells = <2>; - #size-cells = <0>; - dsa,ethernet = <&fec1>; - dsa,mii-bus = <&mdio_mux_1>; - - /* 6352 - Primary - 7 ports */ - switch0: switch@0-0 { - #address-cells = <1>; - #size-cells = <0>; - reg = <0x00 0>; - eeprom-length = <512>; - port@0 { + switch1: switch1@0 { + compatible = "marvell,mv88e6085"; + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - label = "lan0"; - }; - - port@1 { - reg = <1>; - label = "lan1"; - }; - - port@2 { - reg = <2>; - label = "lan2"; - }; - - switch0port5: port@5 { - reg = <5>; - label = "dsa"; - phy-mode = "rgmii-txid"; - link = <&switch1port6 - &switch2port9>; - - fixed-link { - speed = <1000>; - full-duplex; + dsa,member = <0 1>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + label = "lan3"; + phy-handle = <&switch1phy0>; + }; + + port@1 { + reg = <1>; + label = "lan4"; + phy-handle = <&switch1phy1>; + }; + + port@2 { + reg = <2>; + label = "lan5"; + phy-handle = <&switch1phy2>; + }; + + switch1port5: port@5 { + reg = <5>; + label = "dsa"; + link = <&switch2port9>; + phy-mode = "rgmii-txid"; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; + + switch1port6: port@6 { + reg = <6>; + label = "dsa"; + phy-mode = "rgmii-txid"; + link = <&switch0port5>; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; }; - }; - - port@6 { - reg = <6>; - label = "cpu"; - - fixed-link { - speed = <100>; - full-duplex; + mdio { + #address-cells = <1>; + #size-cells = <0>; + switch1phy0: switch1phy0@0 { + reg = <0>; + }; + switch1phy1: switch1phy0@1 { + reg = <1>; + }; + switch1phy2: switch1phy0@2 { + reg = <2>; + }; }; }; - }; - /* 6352 - Secondary - 7 ports */ - switch1: switch@0-1 { + mdio_mux_4: mdio@4 { #address-cells = <1>; #size-cells = <0>; - reg = <0x00 1>; - eeprom-length = <512>; - mii-bus = <&mdio_mux_2>; + reg = <4>; - port@0 { + switch2: switch2@0 { + compatible = "marvell,mv88e6085"; + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - label = "lan3"; - }; - - port@1 { - reg = <1>; - label = "lan4"; - }; - - port@2 { - reg = <2>; - label = "lan5"; - }; - - switch1port5: port@5 { - reg = <5>; - label = "dsa"; - link = <&switch2port9>; - phy-mode = "rgmii-txid"; - - fixed-link { - speed = <1000>; - full-duplex; - }; - }; - - switch1port6: port@6 { - reg = <6>; - label = "dsa"; - phy-mode = "rgmii-txid"; - link = <&switch0port5>; - - fixed-link { - speed = <1000>; - full-duplex; + dsa,member = <0 2>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + label = "lan6"; + }; + + port@1 { + reg = <1>; + label = "lan7"; + }; + + port@2 { + reg = <2>; + label = "lan8"; + }; + + port@3 { + reg = <3>; + label = "optical3"; + fixed-link { + speed = <1000>; + full-duplex; + link-gpios = <&gpio6 2 + GPIO_ACTIVE_HIGH>; + }; + }; + + port@4 { + reg = <4>; + label = "optical4"; + fixed-link { + speed = <1000>; + full-duplex; + link-gpios = <&gpio6 3 + GPIO_ACTIVE_HIGH>; + }; + }; + + switch2port9: port@9 { + reg = <9>; + label = "dsa"; + phy-mode = "rgmii-txid"; + link = <&switch1port5 + &switch0port5>; + fixed-link { + speed = <1000>; + full-duplex; + }; + }; }; }; }; - /* 6185 - 10 ports */ - switch2: switch@0-2 { + mdio_mux_8: mdio@8 { + reg = <8>; #address-cells = <1>; #size-cells = <0>; - reg = <0x00 2>; - mii-bus = <&mdio_mux_4>; - - port@0 { - reg = <0>; - label = "lan6"; - }; - - port@1 { - reg = <1>; - label = "lan7"; - }; - - port@2 { - reg = <2>; - label = "lan8"; - }; - - port@3 { - reg = <3>; - label = "optical3"; - - fixed-link { - speed = <1000>; - full-duplex; - link-gpios = <&gpio6 2 - GPIO_ACTIVE_HIGH>; - }; - }; - - port@4 { - reg = <4>; - label = "optical4"; - - fixed-link { - speed = <1000>; - full-duplex; - link-gpios = <&gpio6 3 - GPIO_ACTIVE_HIGH>; - }; - }; - - switch2port9: port@9 { - reg = <9>; - label = "dsa"; - phy-mode = "rgmii-txid"; - link = <&switch1port5 - &switch0port5>; - - fixed-link { - speed = <1000>; - full-duplex; - }; - }; }; }; diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index 03a39fe29246..1568cb5cd870 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -154,30 +154,23 @@ static int ghash_async_init(struct ahash_request *req) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); struct ahash_request *cryptd_req = ahash_request_ctx(req); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; + struct shash_desc *desc = cryptd_shash_desc(cryptd_req); + struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); - if (!may_use_simd()) { - memcpy(cryptd_req, req, sizeof(*req)); - ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); - return crypto_ahash_init(cryptd_req); - } else { - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); - - desc->tfm = child; - desc->flags = req->base.flags; - return crypto_shash_init(desc); - } + desc->tfm = child; + desc->flags = req->base.flags; + return crypto_shash_init(desc); } static int ghash_async_update(struct ahash_request *req) { struct ahash_request *cryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!may_use_simd()) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - + if (!may_use_simd() || + (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); return crypto_ahash_update(cryptd_req); @@ -190,12 +183,12 @@ static int ghash_async_update(struct ahash_request *req) static int ghash_async_final(struct ahash_request *req) { struct ahash_request *cryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!may_use_simd()) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - + if (!may_use_simd() || + (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); return crypto_ahash_final(cryptd_req); @@ -212,7 +205,8 @@ static int ghash_async_digest(struct ahash_request *req) struct ahash_request *cryptd_req = ahash_request_ctx(req); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!may_use_simd()) { + if (!may_use_simd() || + (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); return crypto_ahash_digest(cryptd_req); diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 9e10c4567eb4..66d0e215a773 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -77,8 +77,36 @@ static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ return result; \ } +#define ATOMIC_FETCH_OP(op, c_op, asm_op) \ +static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +{ \ + unsigned long tmp; \ + int result, val; \ + \ + prefetchw(&v->counter); \ + \ + __asm__ __volatile__("@ atomic_fetch_" #op "\n" \ +"1: ldrex %0, [%4]\n" \ +" " #asm_op " %1, %0, %5\n" \ +" strex %2, %1, [%4]\n" \ +" teq %2, #0\n" \ +" bne 1b" \ + : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter) \ + : "r" (&v->counter), "Ir" (i) \ + : "cc"); \ + \ + return result; \ +} + #define atomic_add_return_relaxed atomic_add_return_relaxed #define atomic_sub_return_relaxed atomic_sub_return_relaxed +#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed +#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed + +#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed +#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed +#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed +#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new) { @@ -159,6 +187,20 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return val; \ } +#define ATOMIC_FETCH_OP(op, c_op, asm_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned long flags; \ + int val; \ + \ + raw_local_irq_save(flags); \ + val = v->counter; \ + v->counter c_op i; \ + raw_local_irq_restore(flags); \ + \ + return val; \ +} + static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; @@ -187,19 +229,26 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define ATOMIC_OPS(op, c_op, asm_op) \ ATOMIC_OP(op, c_op, asm_op) \ - ATOMIC_OP_RETURN(op, c_op, asm_op) + ATOMIC_OP_RETURN(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) #define atomic_andnot atomic_andnot -ATOMIC_OP(and, &=, and) -ATOMIC_OP(andnot, &= ~, bic) -ATOMIC_OP(or, |=, orr) -ATOMIC_OP(xor, ^=, eor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op, asm_op) \ + ATOMIC_OP(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) + +ATOMIC_OPS(and, &=, and) +ATOMIC_OPS(andnot, &= ~, bic) +ATOMIC_OPS(or, |=, orr) +ATOMIC_OPS(xor, ^=, eor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -317,24 +366,61 @@ atomic64_##op##_return_relaxed(long long i, atomic64_t *v) \ return result; \ } +#define ATOMIC64_FETCH_OP(op, op1, op2) \ +static inline long long \ +atomic64_fetch_##op##_relaxed(long long i, atomic64_t *v) \ +{ \ + long long result, val; \ + unsigned long tmp; \ + \ + prefetchw(&v->counter); \ + \ + __asm__ __volatile__("@ atomic64_fetch_" #op "\n" \ +"1: ldrexd %0, %H0, [%4]\n" \ +" " #op1 " %Q1, %Q0, %Q5\n" \ +" " #op2 " %R1, %R0, %R5\n" \ +" strexd %2, %1, %H1, [%4]\n" \ +" teq %2, #0\n" \ +" bne 1b" \ + : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter) \ + : "r" (&v->counter), "r" (i) \ + : "cc"); \ + \ + return result; \ +} + #define ATOMIC64_OPS(op, op1, op2) \ ATOMIC64_OP(op, op1, op2) \ - ATOMIC64_OP_RETURN(op, op1, op2) + ATOMIC64_OP_RETURN(op, op1, op2) \ + ATOMIC64_FETCH_OP(op, op1, op2) ATOMIC64_OPS(add, adds, adc) ATOMIC64_OPS(sub, subs, sbc) #define atomic64_add_return_relaxed atomic64_add_return_relaxed #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed +#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed +#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed + +#undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, op1, op2) \ + ATOMIC64_OP(op, op1, op2) \ + ATOMIC64_FETCH_OP(op, op1, op2) #define atomic64_andnot atomic64_andnot -ATOMIC64_OP(and, and, and) -ATOMIC64_OP(andnot, bic, bic) -ATOMIC64_OP(or, orr, orr) -ATOMIC64_OP(xor, eor, eor) +ATOMIC64_OPS(and, and, and) +ATOMIC64_OPS(andnot, bic, bic) +ATOMIC64_OPS(or, orr, orr) +ATOMIC64_OPS(xor, eor, eor) + +#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed +#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed +#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed +#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed #undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h index a708fa1f0905..766bf9b78160 100644 --- a/arch/arm/include/asm/efi.h +++ b/arch/arm/include/asm/efi.h @@ -28,10 +28,10 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); #define arch_efi_call_virt_setup() efi_virtmap_load() #define arch_efi_call_virt_teardown() efi_virtmap_unload() -#define arch_efi_call_virt(f, args...) \ +#define arch_efi_call_virt(p, f, args...) \ ({ \ efi_##f##_t *__f; \ - __f = efi.systab->runtime->f; \ + __f = p->f; \ __f(args); \ }) diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h index 19cfab526d13..b2902a5cd780 100644 --- a/arch/arm/include/asm/pgalloc.h +++ b/arch/arm/include/asm/pgalloc.h @@ -29,7 +29,7 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); + return (pmd_t *)get_zeroed_page(GFP_KERNEL); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) @@ -57,7 +57,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) extern pgd_t *pgd_alloc(struct mm_struct *mm); extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); -#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) +#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO) static inline void clean_pte_table(pte_t *pte) { diff --git a/arch/arm/include/asm/spinlock.h b/arch/arm/include/asm/spinlock.h index 0fa418463f49..4bec45442072 100644 --- a/arch/arm/include/asm/spinlock.h +++ b/arch/arm/include/asm/spinlock.h @@ -6,6 +6,8 @@ #endif #include +#include +#include /* * sev and wfe are ARMv6K extensions. Uniprocessor ARMv6 may not have the K @@ -50,8 +52,21 @@ static inline void dsb_sev(void) * memory. */ -#define arch_spin_unlock_wait(lock) \ - do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + u16 owner = READ_ONCE(lock->tickets.owner); + + for (;;) { + arch_spinlock_t tmp = READ_ONCE(*lock); + + if (tmp.tickets.owner == tmp.tickets.next || + tmp.tickets.owner != owner) + break; + + wfe(); + } + smp_acquire__after_ctrl_dep(); +} #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index 3cadb726ec88..1e25cd80589e 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -209,17 +209,38 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) tlb_flush(tlb); } -static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) +static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page) { + if (tlb->nr == tlb->max) + return true; tlb->pages[tlb->nr++] = page; - VM_BUG_ON(tlb->nr > tlb->max); - return tlb->max - tlb->nr; + return false; } static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) { - if (!__tlb_remove_page(tlb, page)) + if (__tlb_remove_page(tlb, page)) { tlb_flush_mmu(tlb); + __tlb_remove_page(tlb, page); + } +} + +static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return __tlb_remove_page(tlb, page); +} + +static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, + struct page *page) +{ + return __tlb_remove_page(tlb, page); +} + +static inline void tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return tlb_remove_page(tlb, page); } static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, diff --git a/arch/arm/include/asm/xen/hypercall.h b/arch/arm/include/asm/xen/hypercall.h index b6b962d70db9..9d874db13c0e 100644 --- a/arch/arm/include/asm/xen/hypercall.h +++ b/arch/arm/include/asm/xen/hypercall.h @@ -52,6 +52,7 @@ int HYPERVISOR_memory_op(unsigned int cmd, void *arg); int HYPERVISOR_physdev_op(int cmd, void *arg); int HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args); int HYPERVISOR_tmem_op(void *arg); +int HYPERVISOR_vm_assist(unsigned int cmd, unsigned int type); int HYPERVISOR_platform_op_raw(void *arg); static inline int HYPERVISOR_platform_op(struct xen_platform_op *op) { diff --git a/arch/arm/include/asm/xen/xen-ops.h b/arch/arm/include/asm/xen/xen-ops.h new file mode 100644 index 000000000000..ec154e719b11 --- /dev/null +++ b/arch/arm/include/asm/xen/xen-ops.h @@ -0,0 +1,6 @@ +#ifndef _ASM_XEN_OPS_H +#define _ASM_XEN_OPS_H + +void xen_efi_runtime_setup(void); + +#endif /* _ASM_XEN_OPS_H */ diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h index df3f60cb1168..a2b3eb313a25 100644 --- a/arch/arm/include/uapi/asm/kvm.h +++ b/arch/arm/include/uapi/asm/kvm.h @@ -139,8 +139,8 @@ struct kvm_arch_memory_slot { #define ARM_CP15_REG64(...) __ARM_CP15_REG64(__VA_ARGS__) #define KVM_REG_ARM_TIMER_CTL ARM_CP15_REG32(0, 14, 3, 1) -#define KVM_REG_ARM_TIMER_CNT ARM_CP15_REG64(1, 14) -#define KVM_REG_ARM_TIMER_CVAL ARM_CP15_REG64(3, 14) +#define KVM_REG_ARM_TIMER_CNT ARM_CP15_REG64(1, 14) +#define KVM_REG_ARM_TIMER_CVAL ARM_CP15_REG64(3, 14) /* Normal registers are mapped as coprocessor 16. */ #define KVM_REG_ARM_CORE (0x0010 << KVM_REG_ARM_COPROC_SHIFT) diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 7b5350060612..261dae6f3fec 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -1064,6 +1064,7 @@ void __init setup_arch(char **cmdline_p) early_paging_init(mdesc); #endif setup_dma_zone(mdesc); + xen_early_init(); efi_init(); sanity_check_meminfo(); arm_memblock_init(mdesc); @@ -1080,7 +1081,6 @@ void __init setup_arch(char **cmdline_p) arm_dt_init_cpu_maps(); psci_dt_init(); - xen_early_init(); #ifdef CONFIG_SMP if (is_smp()) { if (!mdesc->smp_init || !mdesc->smp_init()) { diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c index 1bfa7a7f5533..b6ec65e68009 100644 --- a/arch/arm/kernel/smp_twd.c +++ b/arch/arm/kernel/smp_twd.c @@ -390,7 +390,7 @@ int __init twd_local_timer_register(struct twd_local_timer *tlt) } #ifdef CONFIG_OF -static void __init twd_local_timer_of_register(struct device_node *np) +static int __init twd_local_timer_of_register(struct device_node *np) { int err; @@ -410,6 +410,7 @@ static void __init twd_local_timer_of_register(struct device_node *np) out: WARN(err, "twd_local_timer_of_register failed (%d)\n", err); + return err; } CLOCKSOURCE_OF_DECLARE(arm_twd_a9, "arm,cortex-a9-twd-timer", twd_local_timer_of_register); CLOCKSOURCE_OF_DECLARE(arm_twd_a5, "arm,cortex-a5-twd-timer", twd_local_timer_of_register); diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 893941ec98dc..f1bde7c4e736 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -263,6 +263,7 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) kvm_timer_vcpu_terminate(vcpu); kvm_vgic_vcpu_destroy(vcpu); kvm_pmu_vcpu_destroy(vcpu); + kvm_vcpu_uninit(vcpu); kmem_cache_free(kvm_vcpu_cache, vcpu); } diff --git a/arch/arm/mach-bcm/Kconfig b/arch/arm/mach-bcm/Kconfig index 68ab6412392a..4f1709b31822 100644 --- a/arch/arm/mach-bcm/Kconfig +++ b/arch/arm/mach-bcm/Kconfig @@ -89,6 +89,7 @@ config ARCH_BCM_MOBILE select HAVE_ARM_ARCH_TIMER select PINCTRL select ARCH_BCM_MOBILE_SMP if SMP + select BCM_KONA_TIMER help This enables support for systems based on Broadcom mobile SoCs. @@ -143,6 +144,7 @@ config ARCH_BCM2835 select ARM_TIMER_SP804 select HAVE_ARM_ARCH_TIMER if ARCH_MULTI_V7 select CLKSRC_OF + select BCM2835_TIMER select PINCTRL select PINCTRL_BCM2835 help diff --git a/arch/arm/mach-integrator/Kconfig b/arch/arm/mach-integrator/Kconfig index b2a85ba13f08..291262e5aeaf 100644 --- a/arch/arm/mach-integrator/Kconfig +++ b/arch/arm/mach-integrator/Kconfig @@ -20,7 +20,7 @@ if ARCH_INTEGRATOR config ARCH_INTEGRATOR_AP bool "Support Integrator/AP and Integrator/PP2 platforms" - select CLKSRC_MMIO + select INTEGRATOR_AP_TIMER select MIGHT_HAVE_PCI select SERIAL_AMBA_PL010 if TTY select SERIAL_AMBA_PL010_CONSOLE if TTY diff --git a/arch/arm/mach-keystone/Kconfig b/arch/arm/mach-keystone/Kconfig index ea955f6db8b7..bac577badc7e 100644 --- a/arch/arm/mach-keystone/Kconfig +++ b/arch/arm/mach-keystone/Kconfig @@ -4,7 +4,7 @@ config ARCH_KEYSTONE depends on ARM_PATCH_PHYS_VIRT select ARM_GIC select HAVE_ARM_ARCH_TIMER - select CLKSRC_MMIO + select KEYSTONE_TIMER select ARM_ERRATA_798181 if SMP select COMMON_CLK_KEYSTONE select ARCH_SUPPORTS_BIG_ENDIAN diff --git a/arch/arm/mach-moxart/Kconfig b/arch/arm/mach-moxart/Kconfig index 180d9d216719..ddc79cea32d3 100644 --- a/arch/arm/mach-moxart/Kconfig +++ b/arch/arm/mach-moxart/Kconfig @@ -3,7 +3,7 @@ menuconfig ARCH_MOXART depends on ARCH_MULTI_V4 select CPU_FA526 select ARM_DMA_MEM_BUFFERABLE - select CLKSRC_MMIO + select MOXART_TIMER select GENERIC_IRQ_CHIP select ARCH_REQUIRE_GPIOLIB select PHYLIB if NETDEVICES diff --git a/arch/arm/mach-mvebu/Makefile b/arch/arm/mach-mvebu/Makefile index ecf9e0c3b107..e53c6cfcab51 100644 --- a/arch/arm/mach-mvebu/Makefile +++ b/arch/arm/mach-mvebu/Makefile @@ -7,9 +7,15 @@ CFLAGS_pmsu.o := -march=armv7-a obj-$(CONFIG_MACH_MVEBU_ANY) += system-controller.o mvebu-soc-id.o ifeq ($(CONFIG_MACH_MVEBU_V7),y) -obj-y += cpu-reset.o board-v7.o coherency.o coherency_ll.o pmsu.o pmsu_ll.o pm.o pm-board.o +obj-y += cpu-reset.o board-v7.o coherency.o coherency_ll.o pmsu.o pmsu_ll.o + +obj-$(CONFIG_PM) += pm.o pm-board.o obj-$(CONFIG_SMP) += platsmp.o headsmp.o platsmp-a9.o headsmp-a9.o endif obj-$(CONFIG_MACH_DOVE) += dove.o -obj-$(CONFIG_MACH_KIRKWOOD) += kirkwood.o kirkwood-pm.o + +ifeq ($(CONFIG_MACH_KIRKWOOD),y) +obj-y += kirkwood.o +obj-$(CONFIG_PM) += kirkwood-pm.o +endif diff --git a/arch/arm/mach-mvebu/coherency.c b/arch/arm/mach-mvebu/coherency.c index 7e989d61159c..e80f0dde2189 100644 --- a/arch/arm/mach-mvebu/coherency.c +++ b/arch/arm/mach-mvebu/coherency.c @@ -162,22 +162,16 @@ exit: } /* - * This ioremap hook is used on Armada 375/38x to ensure that PCIe - * memory areas are mapped as MT_UNCACHED instead of MT_DEVICE. This - * is needed as a workaround for a deadlock issue between the PCIe - * interface and the cache controller. + * This ioremap hook is used on Armada 375/38x to ensure that all MMIO + * areas are mapped as MT_UNCACHED instead of MT_DEVICE. This is + * needed for the HW I/O coherency mechanism to work properly without + * deadlock. */ static void __iomem * -armada_pcie_wa_ioremap_caller(phys_addr_t phys_addr, size_t size, - unsigned int mtype, void *caller) +armada_wa_ioremap_caller(phys_addr_t phys_addr, size_t size, + unsigned int mtype, void *caller) { - struct resource pcie_mem; - - mvebu_mbus_get_pcie_mem_aperture(&pcie_mem); - - if (pcie_mem.start <= phys_addr && (phys_addr + size) <= pcie_mem.end) - mtype = MT_UNCACHED; - + mtype = MT_UNCACHED; return __arm_ioremap_caller(phys_addr, size, mtype, caller); } @@ -186,7 +180,8 @@ static void __init armada_375_380_coherency_init(struct device_node *np) struct device_node *cache_dn; coherency_cpu_base = of_iomap(np, 0); - arch_ioremap_caller = armada_pcie_wa_ioremap_caller; + arch_ioremap_caller = armada_wa_ioremap_caller; + pci_ioremap_set_mem_type(MT_UNCACHED); /* * We should switch the PL310 to I/O coherency mode only if diff --git a/arch/arm/mach-mxs/Kconfig b/arch/arm/mach-mxs/Kconfig index 84794137b175..68a3a9ec605d 100644 --- a/arch/arm/mach-mxs/Kconfig +++ b/arch/arm/mach-mxs/Kconfig @@ -16,7 +16,7 @@ config ARCH_MXS bool "Freescale MXS (i.MX23, i.MX28) support" depends on ARCH_MULTI_V5 select ARCH_REQUIRE_GPIOLIB - select CLKSRC_MMIO + select MXS_TIMER select PINCTRL select SOC_BUS select SOC_IMX23 diff --git a/arch/arm/mach-nspire/Kconfig b/arch/arm/mach-nspire/Kconfig index bc41f26c1a12..d4985305cab2 100644 --- a/arch/arm/mach-nspire/Kconfig +++ b/arch/arm/mach-nspire/Kconfig @@ -7,5 +7,6 @@ config ARCH_NSPIRE select ARM_AMBA select ARM_VIC select ARM_TIMER_SP804 + select NSPIRE_TIMER help This enables support for systems using the TI-NSPIRE CPU diff --git a/arch/arm/mach-prima2/Kconfig b/arch/arm/mach-prima2/Kconfig index 0cf4426183cf..9e938f2961cf 100644 --- a/arch/arm/mach-prima2/Kconfig +++ b/arch/arm/mach-prima2/Kconfig @@ -28,6 +28,7 @@ config ARCH_ATLAS7 default y select ARM_GIC select CPU_V7 + select ATLAS7_TIMER select HAVE_ARM_SCU if SMP select HAVE_SMP help @@ -38,6 +39,7 @@ config ARCH_PRIMA2 default y select SIRF_IRQ select ZONE_DMA + select PRIMA2_TIMER help Support for CSR SiRFSoC ARM Cortex A9 Platform diff --git a/arch/arm/mach-u300/Kconfig b/arch/arm/mach-u300/Kconfig index 301a98498453..4fdc3425ffbd 100644 --- a/arch/arm/mach-u300/Kconfig +++ b/arch/arm/mach-u300/Kconfig @@ -4,7 +4,7 @@ menuconfig ARCH_U300 select ARCH_REQUIRE_GPIOLIB select ARM_AMBA select ARM_VIC - select CLKSRC_MMIO + select U300_TIMER select CPU_ARM926T select HAVE_TCM select PINCTRL diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index ad5841856007..3a2e678b8d30 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -243,7 +243,7 @@ good_area: goto out; } - return handle_mm_fault(mm, vma, addr & PAGE_MASK, flags); + return handle_mm_fault(vma, addr & PAGE_MASK, flags); check_stack: /* Don't allow expansion below FIRST_USER_ADDRESS */ diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index b8d477321730..c1c1a5c67da1 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -23,7 +23,7 @@ #define __pgd_alloc() kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL) #define __pgd_free(pgd) kfree(pgd) #else -#define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_REPEAT, 2) +#define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2) #define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) #endif diff --git a/arch/arm/xen/Makefile b/arch/arm/xen/Makefile index 12969523414c..227952103b0b 100644 --- a/arch/arm/xen/Makefile +++ b/arch/arm/xen/Makefile @@ -1 +1,2 @@ obj-y := enlighten.o hypercall.o grant-table.o p2m.o mm.o +obj-$(CONFIG_XEN_EFI) += efi.o diff --git a/arch/arm/xen/efi.c b/arch/arm/xen/efi.c new file mode 100644 index 000000000000..16db419f9e90 --- /dev/null +++ b/arch/arm/xen/efi.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2015, Linaro Limited, Shannon Zhao + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ + +#include +#include +#include + +/* Set XEN EFI runtime services function pointers. Other fields of struct efi, + * e.g. efi.systab, will be set like normal EFI. + */ +void __init xen_efi_runtime_setup(void) +{ + efi.get_time = xen_efi_get_time; + efi.set_time = xen_efi_set_time; + efi.get_wakeup_time = xen_efi_get_wakeup_time; + efi.set_wakeup_time = xen_efi_set_wakeup_time; + efi.get_variable = xen_efi_get_variable; + efi.get_next_variable = xen_efi_get_next_variable; + efi.set_variable = xen_efi_set_variable; + efi.query_variable_info = xen_efi_query_variable_info; + efi.update_capsule = xen_efi_update_capsule; + efi.query_capsule_caps = xen_efi_query_capsule_caps; + efi.get_next_high_mono_count = xen_efi_get_next_high_mono_count; + efi.reset_system = NULL; /* Functionality provided by Xen. */ +} +EXPORT_SYMBOL_GPL(xen_efi_runtime_setup); diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index 75cd7345c654..0bea3d271f6e 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -12,14 +12,16 @@ #include #include #include -#include #include #include +#include #include +#include #include #include #include #include +#include #include #include #include @@ -30,6 +32,7 @@ #include #include #include +#include #include @@ -46,14 +49,16 @@ struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info; DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu); static struct vcpu_info __percpu *xen_vcpu_info; +/* Linux <-> Xen vCPU id mapping */ +DEFINE_PER_CPU(int, xen_vcpu_id) = -1; +EXPORT_PER_CPU_SYMBOL(xen_vcpu_id); + /* These are unused until we support booting "pre-ballooned" */ unsigned long xen_released_pages; struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata; static __read_mostly unsigned int xen_events_irq; -static __initdata struct device_node *xen_node; - int xen_remap_domain_gfn_array(struct vm_area_struct *vma, unsigned long addr, xen_pfn_t *gfn, int nr, @@ -84,19 +89,6 @@ int xen_unmap_domain_gfn_range(struct vm_area_struct *vma, } EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range); -static unsigned long long xen_stolen_accounting(int cpu) -{ - struct vcpu_runstate_info state; - - BUG_ON(cpu != smp_processor_id()); - - xen_get_runstate_snapshot(&state); - - WARN_ON(state.state != RUNSTATE_running); - - return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline]; -} - static void xen_read_wallclock(struct timespec64 *ts) { u32 version; @@ -179,10 +171,14 @@ static void xen_percpu_init(void) pr_info("Xen: initializing cpu%d\n", cpu); vcpup = per_cpu_ptr(xen_vcpu_info, cpu); + /* Direct vCPU id mapping for ARM guests. */ + per_cpu(xen_vcpu_id, cpu) = cpu; + info.mfn = virt_to_gfn(vcpup); info.offset = xen_offset_in_page(vcpup); - err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info); + err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, xen_vcpu_nr(cpu), + &info); BUG_ON(err); per_cpu(xen_vcpu, cpu) = vcpup; @@ -237,6 +233,46 @@ static irqreturn_t xen_arm_callback(int irq, void *arg) return IRQ_HANDLED; } +static __initdata struct { + const char *compat; + const char *prefix; + const char *version; + bool found; +} hyper_node = {"xen,xen", "xen,xen-", NULL, false}; + +static int __init fdt_find_hyper_node(unsigned long node, const char *uname, + int depth, void *data) +{ + const void *s = NULL; + int len; + + if (depth != 1 || strcmp(uname, "hypervisor") != 0) + return 0; + + if (of_flat_dt_is_compatible(node, hyper_node.compat)) + hyper_node.found = true; + + s = of_get_flat_dt_prop(node, "compatible", &len); + if (strlen(hyper_node.prefix) + 3 < len && + !strncmp(hyper_node.prefix, s, strlen(hyper_node.prefix))) + hyper_node.version = s + strlen(hyper_node.prefix); + + /* + * Check if Xen supports EFI by checking whether there is the + * "/hypervisor/uefi" node in DT. If so, runtime services are available + * through proxy functions (e.g. in case of Xen dom0 EFI implementation + * they call special hypercall which executes relevant EFI functions) + * and that is why they are always enabled. + */ + if (IS_ENABLED(CONFIG_XEN_EFI)) { + if ((of_get_flat_dt_subnode_by_name(node, "uefi") > 0) && + !efi_runtime_disabled()) + set_bit(EFI_RUNTIME_SERVICES, &efi.flags); + } + + return 0; +} + /* * see Documentation/devicetree/bindings/arm/xen.txt for the * documentation of the Xen Device Tree format. @@ -244,26 +280,18 @@ static irqreturn_t xen_arm_callback(int irq, void *arg) #define GRANT_TABLE_PHYSADDR 0 void __init xen_early_init(void) { - int len; - const char *s = NULL; - const char *version = NULL; - const char *xen_prefix = "xen,xen-"; - - xen_node = of_find_compatible_node(NULL, NULL, "xen,xen"); - if (!xen_node) { + of_scan_flat_dt(fdt_find_hyper_node, NULL); + if (!hyper_node.found) { pr_debug("No Xen support\n"); return; } - s = of_get_property(xen_node, "compatible", &len); - if (strlen(xen_prefix) + 3 < len && - !strncmp(xen_prefix, s, strlen(xen_prefix))) - version = s + strlen(xen_prefix); - if (version == NULL) { + + if (hyper_node.version == NULL) { pr_debug("Xen version not found\n"); return; } - pr_info("Xen %s support found\n", version); + pr_info("Xen %s support found\n", hyper_node.version); xen_domain_type = XEN_HVM_DOMAIN; @@ -278,28 +306,68 @@ void __init xen_early_init(void) add_preferred_console("hvc", 0, NULL); } +static void __init xen_acpi_guest_init(void) +{ +#ifdef CONFIG_ACPI + struct xen_hvm_param a; + int interrupt, trigger, polarity; + + a.domid = DOMID_SELF; + a.index = HVM_PARAM_CALLBACK_IRQ; + + if (HYPERVISOR_hvm_op(HVMOP_get_param, &a) + || (a.value >> 56) != HVM_PARAM_CALLBACK_TYPE_PPI) { + xen_events_irq = 0; + return; + } + + interrupt = a.value & 0xff; + trigger = ((a.value >> 8) & 0x1) ? ACPI_EDGE_SENSITIVE + : ACPI_LEVEL_SENSITIVE; + polarity = ((a.value >> 8) & 0x2) ? ACPI_ACTIVE_LOW + : ACPI_ACTIVE_HIGH; + xen_events_irq = acpi_register_gsi(NULL, interrupt, trigger, polarity); +#endif +} + +static void __init xen_dt_guest_init(void) +{ + struct device_node *xen_node; + + xen_node = of_find_compatible_node(NULL, NULL, "xen,xen"); + if (!xen_node) { + pr_err("Xen support was detected before, but it has disappeared\n"); + return; + } + + xen_events_irq = irq_of_parse_and_map(xen_node, 0); +} + static int __init xen_guest_init(void) { struct xen_add_to_physmap xatp; struct shared_info *shared_info_page = NULL; - struct resource res; - phys_addr_t grant_frames; if (!xen_domain()) return 0; - if (of_address_to_resource(xen_node, GRANT_TABLE_PHYSADDR, &res)) { - pr_err("Xen grant table base address not found\n"); - return -ENODEV; - } - grant_frames = res.start; + if (!acpi_disabled) + xen_acpi_guest_init(); + else + xen_dt_guest_init(); - xen_events_irq = irq_of_parse_and_map(xen_node, 0); if (!xen_events_irq) { pr_err("Xen event channel interrupt not found\n"); return -ENODEV; } + /* + * The fdt parsing codes have set EFI_RUNTIME_SERVICES if Xen EFI + * parameters are found. Force enable runtime services. + */ + if (efi_enabled(EFI_RUNTIME_SERVICES)) + xen_efi_runtime_setup(); + shared_info_page = (struct shared_info *)get_zeroed_page(GFP_KERNEL); if (!shared_info_page) { @@ -328,7 +396,13 @@ static int __init xen_guest_init(void) if (xen_vcpu_info == NULL) return -ENOMEM; - if (gnttab_setup_auto_xlat_frames(grant_frames)) { + /* Direct vCPU id mapping for ARM guests. */ + per_cpu(xen_vcpu_id, 0) = 0; + + xen_auto_xlat_grant_frames.count = gnttab_max_grant_frames(); + if (xen_xlate_map_ballooned_pages(&xen_auto_xlat_grant_frames.pfn, + &xen_auto_xlat_grant_frames.vaddr, + xen_auto_xlat_grant_frames.count)) { free_percpu(xen_vcpu_info); return -ENOMEM; } @@ -355,8 +429,8 @@ static int __init xen_guest_init(void) register_cpu_notifier(&xen_cpu_notifier); - pv_time_ops.steal_clock = xen_stolen_accounting; - static_key_slow_inc(¶virt_steal_enabled); + xen_time_setup_guest(); + if (xen_initial_domain()) pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier); @@ -403,4 +477,5 @@ EXPORT_SYMBOL_GPL(HYPERVISOR_vcpu_op); EXPORT_SYMBOL_GPL(HYPERVISOR_tmem_op); EXPORT_SYMBOL_GPL(HYPERVISOR_platform_op); EXPORT_SYMBOL_GPL(HYPERVISOR_multicall); +EXPORT_SYMBOL_GPL(HYPERVISOR_vm_assist); EXPORT_SYMBOL_GPL(privcmd_call); diff --git a/arch/arm/xen/hypercall.S b/arch/arm/xen/hypercall.S index 9a36f4f49c10..a648dfc3be30 100644 --- a/arch/arm/xen/hypercall.S +++ b/arch/arm/xen/hypercall.S @@ -91,6 +91,7 @@ HYPERCALL3(vcpu_op); HYPERCALL1(tmem_op); HYPERCALL1(platform_op_raw); HYPERCALL2(multicall); +HYPERCALL2(vm_assist); ENTRY(privcmd_call) stmdb sp!, {r4} diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 5a0a691d4220..9f8b99e20557 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -4,9 +4,11 @@ config ARM64 select ACPI_GENERIC_GSI if ACPI select ACPI_REDUCED_HARDWARE_ONLY if ACPI select ARCH_HAS_DEVMEM_IS_ALLOWED + select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_HAS_KCOV select ARCH_HAS_SG_CHAIN select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_USE_CMPXCHG_LOCKREF @@ -85,8 +87,11 @@ config ARM64 select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RCU_TABLE_FREE select HAVE_SYSCALL_TRACEPOINTS + select HAVE_KPROBES + select HAVE_KRETPROBES if HAVE_KPROBES select IOMMU_DMA if IOMMU_SUPPORT select IRQ_DOMAIN select IRQ_FORCED_THREADING @@ -664,6 +669,16 @@ config PARAVIRT_TIME_ACCOUNTING If in doubt, say N here. +config KEXEC + depends on PM_SLEEP_SMP + select KEXEC_CORE + bool "kexec system call" + ---help--- + kexec is a system call that implements the ability to shutdown your + current kernel, and to start another kernel. It is like a reboot + but it is independent of the system firmware. And like a reboot + you can start any kernel with it, not just Linux. + config XEN_DOM0 def_bool y depends on XEN @@ -872,7 +887,7 @@ config RELOCATABLE config RANDOMIZE_BASE bool "Randomize the address of the kernel image" - select ARM64_MODULE_PLTS + select ARM64_MODULE_PLTS if MODULES select RELOCATABLE help Randomizes the virtual address at which the kernel image is diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 7085e322dc42..d59b6908a21a 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -12,7 +12,6 @@ LDFLAGS_vmlinux :=-p --no-undefined -X CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET) -OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S GZFLAGS :=-9 ifneq ($(CONFIG_RELOCATABLE),) @@ -95,7 +94,7 @@ boot := arch/arm64/boot Image: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ -Image.%: vmlinux +Image.%: Image $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ zinstall install: @@ -121,6 +120,16 @@ archclean: $(Q)$(MAKE) $(clean)=$(boot) $(Q)$(MAKE) $(clean)=$(boot)/dts +# We need to generate vdso-offsets.h before compiling certain files in kernel/. +# In order to do that, we should use the archprepare target, but we can't since +# asm-offsets.h is included in some files used to generate vdso-offsets.h, and +# asm-offsets.h is built in prepare0, for which archprepare is a dependency. +# Therefore we need to generate the header after prepare0 has been made, hence +# this hack. +prepare: vdso_prepare +vdso_prepare: prepare0 + $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso include/generated/vdso-offsets.h + define archhelp echo '* Image.gz - Compressed kernel image (arch/$(ARCH)/boot/Image.gz)' echo ' Image - Uncompressed kernel image (arch/$(ARCH)/boot/Image)' diff --git a/arch/arm64/boot/Makefile b/arch/arm64/boot/Makefile index 305c552b5ec1..1f012c506434 100644 --- a/arch/arm64/boot/Makefile +++ b/arch/arm64/boot/Makefile @@ -14,6 +14,8 @@ # Based on the ia64 boot/Makefile. # +OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S + targets := Image Image.gz $(obj)/Image: vmlinux FORCE diff --git a/arch/arm64/boot/dts/apm/apm-merlin.dts b/arch/arm64/boot/dts/apm/apm-merlin.dts index 387c6a8d0da9..b0f64414c1b0 100644 --- a/arch/arm64/boot/dts/apm/apm-merlin.dts +++ b/arch/arm64/boot/dts/apm/apm-merlin.dts @@ -83,3 +83,9 @@ status = "ok"; }; }; + +&mdio { + sgenet0phy: phy@0 { + reg = <0x0>; + }; +}; diff --git a/arch/arm64/boot/dts/apm/apm-mustang.dts b/arch/arm64/boot/dts/apm/apm-mustang.dts index 44db32ec5e9c..b7fb5d9295c2 100644 --- a/arch/arm64/boot/dts/apm/apm-mustang.dts +++ b/arch/arm64/boot/dts/apm/apm-mustang.dts @@ -79,3 +79,15 @@ &mmc0 { status = "ok"; }; + +&mdio { + menet0phy: phy@3 { + reg = <0x3>; + }; + sgenet0phy: phy@4 { + reg = <0x4>; + }; + sgenet1phy: phy@5 { + reg = <0x5>; + }; +}; diff --git a/arch/arm64/boot/dts/apm/apm-shadowcat.dtsi b/arch/arm64/boot/dts/apm/apm-shadowcat.dtsi index c569f761d090..2e1e5daa1dc7 100644 --- a/arch/arm64/boot/dts/apm/apm-shadowcat.dtsi +++ b/arch/arm64/boot/dts/apm/apm-shadowcat.dtsi @@ -625,10 +625,18 @@ apm,irq-start = <8>; }; + mdio: mdio@1f610000 { + compatible = "apm,xgene-mdio-xfi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x1f610000 0x0 0xd100>; + clocks = <&xge0clk 0>; + }; + sgenet0: ethernet@1f610000 { compatible = "apm,xgene2-sgenet"; status = "disabled"; - reg = <0x0 0x1f610000 0x0 0x10000>, + reg = <0x0 0x1f610000 0x0 0xd100>, <0x0 0x1f600000 0x0 0Xd100>, <0x0 0x20000000 0x0 0X20000>; interrupts = <0 96 4>, @@ -637,6 +645,7 @@ clocks = <&xge0clk 0>; local-mac-address = [00 01 73 00 00 01]; phy-connection-type = "sgmii"; + phy-handle = <&sgenet0phy>; }; xgenet1: ethernet@1f620000 { diff --git a/arch/arm64/boot/dts/apm/apm-storm.dtsi b/arch/arm64/boot/dts/apm/apm-storm.dtsi index 5147d7698924..6bf7cbe2e72d 100644 --- a/arch/arm64/boot/dts/apm/apm-storm.dtsi +++ b/arch/arm64/boot/dts/apm/apm-storm.dtsi @@ -237,20 +237,11 @@ clocks = <&socplldiv2 0>; reg = <0x0 0x1f21c000 0x0 0x1000>; reg-names = "csr-reg"; - csr-mask = <0x3>; + csr-mask = <0xa>; + enable-mask = <0xf>; clock-output-names = "sge0clk"; }; - sge1clk: sge1clk@1f21c000 { - compatible = "apm,xgene-device-clock"; - #clock-cells = <1>; - clocks = <&socplldiv2 0>; - reg = <0x0 0x1f21c000 0x0 0x1000>; - reg-names = "csr-reg"; - csr-mask = <0xc>; - clock-output-names = "sge1clk"; - }; - xge0clk: xge0clk@1f61c000 { compatible = "apm,xgene-device-clock"; #clock-cells = <1>; @@ -921,6 +912,14 @@ clocks = <&rtcclk 0>; }; + mdio: mdio@17020000 { + compatible = "apm,xgene-mdio-rgmii"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x17020000 0x0 0xd100>; + clocks = <&menetclk 0>; + }; + menet: ethernet@17020000 { compatible = "apm,xgene-enet"; status = "disabled"; @@ -934,7 +933,7 @@ /* mac address will be overwritten by the bootloader */ local-mac-address = [00 00 00 00 00 00]; phy-connection-type = "rgmii"; - phy-handle = <&menetphy>; + phy-handle = <&menet0phy>,<&menetphy>; mdio { compatible = "apm,xgene-mdio"; #address-cells = <1>; @@ -960,6 +959,7 @@ clocks = <&sge0clk 0>; local-mac-address = [00 00 00 00 00 00]; phy-connection-type = "sgmii"; + phy-handle = <&sgenet0phy>; }; sgenet1: ethernet@1f210030 { @@ -973,9 +973,9 @@ <0x0 0xAD 0x4>; port-id = <1>; dma-coherent; - clocks = <&sge1clk 0>; local-mac-address = [00 00 00 00 00 00]; phy-connection-type = "sgmii"; + phy-handle = <&sgenet1phy>; }; xgenet: ethernet@1f610000 { diff --git a/arch/arm64/boot/dts/broadcom/ns2-svk.dts b/arch/arm64/boot/dts/broadcom/ns2-svk.dts index 54ca40c9f711..ea5603fd106a 100644 --- a/arch/arm64/boot/dts/broadcom/ns2-svk.dts +++ b/arch/arm64/boot/dts/broadcom/ns2-svk.dts @@ -52,6 +52,14 @@ }; }; +&pci_phy0 { + status = "ok"; +}; + +&pci_phy1 { + status = "ok"; +}; + &pcie0 { status = "ok"; }; @@ -132,3 +140,11 @@ #size-cells = <1>; }; }; + +&mdio_mux_iproc { + mdio@10 { + gphy0: eth-phy@10 { + reg = <0x10>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/broadcom/ns2.dtsi b/arch/arm64/boot/dts/broadcom/ns2.dtsi index ec68ec1a80c8..46b78fa89f4c 100644 --- a/arch/arm64/boot/dts/broadcom/ns2.dtsi +++ b/arch/arm64/boot/dts/broadcom/ns2.dtsi @@ -263,6 +263,45 @@ IRQ_TYPE_LEVEL_HIGH)>; }; + mdio_mux_iproc: mdio-mux@6602023c { + compatible = "brcm,mdio-mux-iproc"; + reg = <0x6602023c 0x14>; + #address-cells = <1>; + #size-cells = <0>; + + mdio@0 { + reg = <0x0>; + #address-cells = <1>; + #size-cells = <0>; + + pci_phy0: pci-phy@0 { + compatible = "brcm,ns2-pcie-phy"; + reg = <0x0>; + #phy-cells = <0>; + status = "disabled"; + }; + }; + + mdio@7 { + reg = <0x7>; + #address-cells = <1>; + #size-cells = <0>; + + pci_phy1: pci-phy@0 { + compatible = "brcm,ns2-pcie-phy"; + reg = <0x0>; + #phy-cells = <0>; + status = "disabled"; + }; + }; + + mdio@10 { + reg = <0x10>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + timer0: timer@66030000 { compatible = "arm,sp804", "arm,primecell"; reg = <0x66030000 0x1000>; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts index f895fc02ab06..40846319be69 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts @@ -49,6 +49,10 @@ / { model = "LS1043A RDB Board"; + + aliases { + crypto = &crypto; + }; }; &i2c0 { diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi index de0323b48b1e..6bd46c133010 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi @@ -159,6 +159,49 @@ big-endian; }; + crypto: crypto@1700000 { + compatible = "fsl,sec-v5.4", "fsl,sec-v5.0", + "fsl,sec-v4.0"; + fsl,sec-era = <3>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x00 0x1700000 0x100000>; + reg = <0x00 0x1700000 0x0 0x100000>; + interrupts = <0 75 0x4>; + + sec_jr0: jr@10000 { + compatible = "fsl,sec-v5.4-job-ring", + "fsl,sec-v5.0-job-ring", + "fsl,sec-v4.0-job-ring"; + reg = <0x10000 0x10000>; + interrupts = <0 71 0x4>; + }; + + sec_jr1: jr@20000 { + compatible = "fsl,sec-v5.4-job-ring", + "fsl,sec-v5.0-job-ring", + "fsl,sec-v4.0-job-ring"; + reg = <0x20000 0x10000>; + interrupts = <0 72 0x4>; + }; + + sec_jr2: jr@30000 { + compatible = "fsl,sec-v5.4-job-ring", + "fsl,sec-v5.0-job-ring", + "fsl,sec-v4.0-job-ring"; + reg = <0x30000 0x10000>; + interrupts = <0 73 0x4>; + }; + + sec_jr3: jr@40000 { + compatible = "fsl,sec-v5.4-job-ring", + "fsl,sec-v5.0-job-ring", + "fsl,sec-v4.0-job-ring"; + reg = <0x40000 0x10000>; + interrupts = <0 74 0x4>; + }; + }; + dcfg: dcfg@1ee0000 { compatible = "fsl,ls1043a-dcfg", "syscon"; reg = <0x0 0x1ee0000 0x0 0x10000>; diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi index 05f89c4a5413..77b8c4e388ca 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -168,6 +168,18 @@ }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + vpu_dma_reserved: vpu_dma_mem_region { + compatible = "shared-dma-pool"; + reg = <0 0xb7000000 0 0x500000>; + alignment = <0x1000>; + no-map; + }; + }; + timer { compatible = "arm,armv8-timer"; interrupt-parent = <&gic>; @@ -312,6 +324,17 @@ clock-names = "spi", "wrap"; }; + vpu: vpu@10020000 { + compatible = "mediatek,mt8173-vpu"; + reg = <0 0x10020000 0 0x30000>, + <0 0x10050000 0 0x100>; + reg-names = "tcm", "cfg_reg"; + interrupts = ; + clocks = <&topckgen CLK_TOP_SCP_SEL>; + clock-names = "main"; + memory-region = <&vpu_dma_reserved>; + }; + sysirq: intpol-controller@10200620 { compatible = "mediatek,mt8173-sysirq", "mediatek,mt6577-sysirq"; @@ -754,6 +777,45 @@ clock-names = "apb", "smi"; }; + vcodec_enc: vcodec@18002000 { + compatible = "mediatek,mt8173-vcodec-enc"; + reg = <0 0x18002000 0 0x1000>, /* VENC_SYS */ + <0 0x19002000 0 0x1000>; /* VENC_LT_SYS */ + interrupts = , + ; + mediatek,larb = <&larb3>, + <&larb5>; + iommus = <&iommu M4U_PORT_VENC_RCPU>, + <&iommu M4U_PORT_VENC_REC>, + <&iommu M4U_PORT_VENC_BSDMA>, + <&iommu M4U_PORT_VENC_SV_COMV>, + <&iommu M4U_PORT_VENC_RD_COMV>, + <&iommu M4U_PORT_VENC_CUR_LUMA>, + <&iommu M4U_PORT_VENC_CUR_CHROMA>, + <&iommu M4U_PORT_VENC_REF_LUMA>, + <&iommu M4U_PORT_VENC_REF_CHROMA>, + <&iommu M4U_PORT_VENC_NBM_RDMA>, + <&iommu M4U_PORT_VENC_NBM_WDMA>, + <&iommu M4U_PORT_VENC_RCPU_SET2>, + <&iommu M4U_PORT_VENC_REC_FRM_SET2>, + <&iommu M4U_PORT_VENC_BSDMA_SET2>, + <&iommu M4U_PORT_VENC_SV_COMA_SET2>, + <&iommu M4U_PORT_VENC_RD_COMA_SET2>, + <&iommu M4U_PORT_VENC_CUR_LUMA_SET2>, + <&iommu M4U_PORT_VENC_CUR_CHROMA_SET2>, + <&iommu M4U_PORT_VENC_REF_LUMA_SET2>, + <&iommu M4U_PORT_VENC_REC_CHROMA_SET2>; + mediatek,vpu = <&vpu>; + clocks = <&topckgen CLK_TOP_VENCPLL_D2>, + <&topckgen CLK_TOP_VENC_SEL>, + <&topckgen CLK_TOP_UNIVPLL1_D2>, + <&topckgen CLK_TOP_VENC_LT_SEL>; + clock-names = "venc_sel_src", + "venc_sel", + "venc_lt_sel_src", + "venc_lt_sel"; + }; + vencltsys: clock-controller@19000000 { compatible = "mediatek,mt8173-vencltsys", "syscon"; reg = <0 0x19000000 0 0x1000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi index d7f8e06910bc..188bbeab92b9 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi @@ -492,6 +492,14 @@ interrupts = ; }; + rktimer: rktimer@ff850000 { + compatible = "rockchip,rk3399-timer"; + reg = <0x0 0xff850000 0x0 0x1000>; + interrupts = ; + clocks = <&cru PCLK_TIMER0>, <&cru SCLK_TIMER00>; + clock-names = "pclk", "timer"; + }; + spdif: spdif@ff870000 { compatible = "rockchip,rk3399-spdif"; reg = <0x0 0xff870000 0x0 0x1000>; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index fd2d74d0491e..4ed4756dfa97 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -70,6 +70,7 @@ CONFIG_KSM=y CONFIG_TRANSPARENT_HUGEPAGE=y CONFIG_CMA=y CONFIG_XEN=y +CONFIG_KEXEC=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_COMPAT=y CONFIG_CPU_IDLE=y diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index cff532a6744e..f43d2c44c765 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -1,6 +1,5 @@ generic-y += bug.h generic-y += bugs.h -generic-y += checksum.h generic-y += clkdev.h generic-y += cputime.h generic-y += current.h diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h index aee323b13802..5420cb0fcb3e 100644 --- a/arch/arm64/include/asm/acpi.h +++ b/arch/arm64/include/asm/acpi.h @@ -113,4 +113,14 @@ static inline const char *acpi_get_enable_method(int cpu) pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr); #endif +#ifdef CONFIG_ACPI_NUMA +int arm64_acpi_numa_init(void); +int acpi_numa_get_nid(unsigned int cpu, u64 hwid); +#else +static inline int arm64_acpi_numa_init(void) { return -ENOSYS; } +static inline int acpi_numa_get_nid(unsigned int cpu, u64 hwid) { return NUMA_NO_NODE; } +#endif /* CONFIG_ACPI_NUMA */ + +#define ACPI_TABLE_UPGRADE_MAX_PHYS MEMBLOCK_ALLOC_ACCESSIBLE + #endif /*_ASM_ACPI_H*/ diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index beccbdefa106..8746ff6abd77 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -95,13 +95,11 @@ void apply_alternatives(void *start, size_t length); * The code that follows this macro will be assembled and linked as * normal. There are no restrictions on this code. */ -.macro alternative_if_not cap, enable = 1 - .if \enable +.macro alternative_if_not cap .pushsection .altinstructions, "a" altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f .popsection 661: - .endif .endm /* @@ -118,27 +116,27 @@ void apply_alternatives(void *start, size_t length); * alternative sequence it is defined in (branches into an * alternative sequence are not fixed up). */ -.macro alternative_else, enable = 1 - .if \enable +.macro alternative_else 662: .pushsection .altinstr_replacement, "ax" 663: - .endif .endm /* * Complete an alternative code sequence. */ -.macro alternative_endif, enable = 1 - .if \enable +.macro alternative_endif 664: .popsection .org . - (664b-663b) + (662b-661b) .org . - (662b-661b) + (664b-663b) - .endif .endm #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) +.macro user_alt, label, oldinstr, newinstr, cond +9999: alternative_insn "\oldinstr", "\newinstr", \cond + _ASM_EXTABLE 9999b, \label +.endm /* * Generate the assembly for UAO alternatives with exception table entries. diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 10b017c4bdd8..d5025c69ca81 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -24,6 +24,7 @@ #define __ASM_ASSEMBLER_H #include +#include #include #include #include @@ -261,7 +262,16 @@ lr .req x30 // link register add \size, \kaddr, \size sub \tmp2, \tmp1, #1 bic \kaddr, \kaddr, \tmp2 -9998: dc \op, \kaddr +9998: + .if (\op == cvau || \op == cvac) +alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE + dc \op, \kaddr +alternative_else + dc civac, \kaddr +alternative_endif + .else + dc \op, \kaddr + .endif add \kaddr, \kaddr, \tmp1 cmp \kaddr, \size b.lo 9998b diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h index f3a3586a421c..c0235e0ff849 100644 --- a/arch/arm64/include/asm/atomic.h +++ b/arch/arm64/include/asm/atomic.h @@ -76,6 +76,36 @@ #define atomic_dec_return_release(v) atomic_sub_return_release(1, (v)) #define atomic_dec_return(v) atomic_sub_return(1, (v)) +#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed +#define atomic_fetch_add_acquire atomic_fetch_add_acquire +#define atomic_fetch_add_release atomic_fetch_add_release +#define atomic_fetch_add atomic_fetch_add + +#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed +#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire +#define atomic_fetch_sub_release atomic_fetch_sub_release +#define atomic_fetch_sub atomic_fetch_sub + +#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed +#define atomic_fetch_and_acquire atomic_fetch_and_acquire +#define atomic_fetch_and_release atomic_fetch_and_release +#define atomic_fetch_and atomic_fetch_and + +#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed +#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire +#define atomic_fetch_andnot_release atomic_fetch_andnot_release +#define atomic_fetch_andnot atomic_fetch_andnot + +#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed +#define atomic_fetch_or_acquire atomic_fetch_or_acquire +#define atomic_fetch_or_release atomic_fetch_or_release +#define atomic_fetch_or atomic_fetch_or + +#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed +#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire +#define atomic_fetch_xor_release atomic_fetch_xor_release +#define atomic_fetch_xor atomic_fetch_xor + #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) #define atomic_xchg_acquire(v, new) xchg_acquire(&((v)->counter), (new)) #define atomic_xchg_release(v, new) xchg_release(&((v)->counter), (new)) @@ -125,6 +155,36 @@ #define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v)) #define atomic64_dec_return(v) atomic64_sub_return(1, (v)) +#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed +#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire +#define atomic64_fetch_add_release atomic64_fetch_add_release +#define atomic64_fetch_add atomic64_fetch_add + +#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed +#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire +#define atomic64_fetch_sub_release atomic64_fetch_sub_release +#define atomic64_fetch_sub atomic64_fetch_sub + +#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed +#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire +#define atomic64_fetch_and_release atomic64_fetch_and_release +#define atomic64_fetch_and atomic64_fetch_and + +#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed +#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire +#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release +#define atomic64_fetch_andnot atomic64_fetch_andnot + +#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed +#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire +#define atomic64_fetch_or_release atomic64_fetch_or_release +#define atomic64_fetch_or atomic64_fetch_or + +#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed +#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire +#define atomic64_fetch_xor_release atomic64_fetch_xor_release +#define atomic64_fetch_xor atomic64_fetch_xor + #define atomic64_xchg_relaxed atomic_xchg_relaxed #define atomic64_xchg_acquire atomic_xchg_acquire #define atomic64_xchg_release atomic_xchg_release diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h index f61c84f6ba02..f819fdcff1ac 100644 --- a/arch/arm64/include/asm/atomic_ll_sc.h +++ b/arch/arm64/include/asm/atomic_ll_sc.h @@ -77,26 +77,57 @@ __LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v)) \ } \ __LL_SC_EXPORT(atomic_##op##_return##name); +#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \ +__LL_SC_INLINE int \ +__LL_SC_PREFIX(atomic_fetch_##op##name(int i, atomic_t *v)) \ +{ \ + unsigned long tmp; \ + int val, result; \ + \ + asm volatile("// atomic_fetch_" #op #name "\n" \ +" prfm pstl1strm, %3\n" \ +"1: ld" #acq "xr %w0, %3\n" \ +" " #asm_op " %w1, %w0, %w4\n" \ +" st" #rel "xr %w2, %w1, %3\n" \ +" cbnz %w2, 1b\n" \ +" " #mb \ + : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ + : "Ir" (i) \ + : cl); \ + \ + return result; \ +} \ +__LL_SC_EXPORT(atomic_fetch_##op##name); + #define ATOMIC_OPS(...) \ ATOMIC_OP(__VA_ARGS__) \ - ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__) - -#define ATOMIC_OPS_RLX(...) \ - ATOMIC_OPS(__VA_ARGS__) \ + ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)\ ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\ ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\ - ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__) + ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__)\ + ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ + ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ + ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ + ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) -ATOMIC_OPS_RLX(add, add) -ATOMIC_OPS_RLX(sub, sub) +ATOMIC_OPS(add, add) +ATOMIC_OPS(sub, sub) + +#undef ATOMIC_OPS +#define ATOMIC_OPS(...) \ + ATOMIC_OP(__VA_ARGS__) \ + ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ + ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ + ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ + ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) -ATOMIC_OP(and, and) -ATOMIC_OP(andnot, bic) -ATOMIC_OP(or, orr) -ATOMIC_OP(xor, eor) +ATOMIC_OPS(and, and) +ATOMIC_OPS(andnot, bic) +ATOMIC_OPS(or, orr) +ATOMIC_OPS(xor, eor) -#undef ATOMIC_OPS_RLX #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -140,26 +171,57 @@ __LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v)) \ } \ __LL_SC_EXPORT(atomic64_##op##_return##name); +#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \ +__LL_SC_INLINE long \ +__LL_SC_PREFIX(atomic64_fetch_##op##name(long i, atomic64_t *v)) \ +{ \ + long result, val; \ + unsigned long tmp; \ + \ + asm volatile("// atomic64_fetch_" #op #name "\n" \ +" prfm pstl1strm, %3\n" \ +"1: ld" #acq "xr %0, %3\n" \ +" " #asm_op " %1, %0, %4\n" \ +" st" #rel "xr %w2, %1, %3\n" \ +" cbnz %w2, 1b\n" \ +" " #mb \ + : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ + : "Ir" (i) \ + : cl); \ + \ + return result; \ +} \ +__LL_SC_EXPORT(atomic64_fetch_##op##name); + #define ATOMIC64_OPS(...) \ ATOMIC64_OP(__VA_ARGS__) \ - ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__) - -#define ATOMIC64_OPS_RLX(...) \ - ATOMIC64_OPS(__VA_ARGS__) \ + ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__) \ ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \ ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \ - ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__) + ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__) \ + ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ + ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ + ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ + ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) -ATOMIC64_OPS_RLX(add, add) -ATOMIC64_OPS_RLX(sub, sub) +ATOMIC64_OPS(add, add) +ATOMIC64_OPS(sub, sub) + +#undef ATOMIC64_OPS +#define ATOMIC64_OPS(...) \ + ATOMIC64_OP(__VA_ARGS__) \ + ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ + ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ + ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ + ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) -ATOMIC64_OP(and, and) -ATOMIC64_OP(andnot, bic) -ATOMIC64_OP(or, orr) -ATOMIC64_OP(xor, eor) +ATOMIC64_OPS(and, and) +ATOMIC64_OPS(andnot, bic) +ATOMIC64_OPS(or, orr) +ATOMIC64_OPS(xor, eor) -#undef ATOMIC64_OPS_RLX #undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h index 39c1d340fec5..b5890be8f257 100644 --- a/arch/arm64/include/asm/atomic_lse.h +++ b/arch/arm64/include/asm/atomic_lse.h @@ -26,54 +26,57 @@ #endif #define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op) - -static inline void atomic_andnot(int i, atomic_t *v) -{ - register int w0 asm ("w0") = i; - register atomic_t *x1 asm ("x1") = v; - - asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot), - " stclr %w[i], %[v]\n") - : [i] "+r" (w0), [v] "+Q" (v->counter) - : "r" (x1) - : __LL_SC_CLOBBERS); +#define ATOMIC_OP(op, asm_op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + register int w0 asm ("w0") = i; \ + register atomic_t *x1 asm ("x1") = v; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op), \ +" " #asm_op " %w[i], %[v]\n") \ + : [i] "+r" (w0), [v] "+Q" (v->counter) \ + : "r" (x1) \ + : __LL_SC_CLOBBERS); \ } -static inline void atomic_or(int i, atomic_t *v) -{ - register int w0 asm ("w0") = i; - register atomic_t *x1 asm ("x1") = v; +ATOMIC_OP(andnot, stclr) +ATOMIC_OP(or, stset) +ATOMIC_OP(xor, steor) +ATOMIC_OP(add, stadd) - asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or), - " stset %w[i], %[v]\n") - : [i] "+r" (w0), [v] "+Q" (v->counter) - : "r" (x1) - : __LL_SC_CLOBBERS); -} +#undef ATOMIC_OP -static inline void atomic_xor(int i, atomic_t *v) -{ - register int w0 asm ("w0") = i; - register atomic_t *x1 asm ("x1") = v; - - asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor), - " steor %w[i], %[v]\n") - : [i] "+r" (w0), [v] "+Q" (v->counter) - : "r" (x1) - : __LL_SC_CLOBBERS); +#define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \ +static inline int atomic_fetch_##op##name(int i, atomic_t *v) \ +{ \ + register int w0 asm ("w0") = i; \ + register atomic_t *x1 asm ("x1") = v; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN( \ + /* LL/SC */ \ + __LL_SC_ATOMIC(fetch_##op##name), \ + /* LSE atomics */ \ +" " #asm_op #mb " %w[i], %w[i], %[v]") \ + : [i] "+r" (w0), [v] "+Q" (v->counter) \ + : "r" (x1) \ + : __LL_SC_CLOBBERS, ##cl); \ + \ + return w0; \ } -static inline void atomic_add(int i, atomic_t *v) -{ - register int w0 asm ("w0") = i; - register atomic_t *x1 asm ("x1") = v; +#define ATOMIC_FETCH_OPS(op, asm_op) \ + ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \ + ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \ + ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \ + ATOMIC_FETCH_OP( , al, op, asm_op, "memory") - asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add), - " stadd %w[i], %[v]\n") - : [i] "+r" (w0), [v] "+Q" (v->counter) - : "r" (x1) - : __LL_SC_CLOBBERS); -} +ATOMIC_FETCH_OPS(andnot, ldclr) +ATOMIC_FETCH_OPS(or, ldset) +ATOMIC_FETCH_OPS(xor, ldeor) +ATOMIC_FETCH_OPS(add, ldadd) + +#undef ATOMIC_FETCH_OP +#undef ATOMIC_FETCH_OPS #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \ static inline int atomic_add_return##name(int i, atomic_t *v) \ @@ -119,6 +122,33 @@ static inline void atomic_and(int i, atomic_t *v) : __LL_SC_CLOBBERS); } +#define ATOMIC_FETCH_OP_AND(name, mb, cl...) \ +static inline int atomic_fetch_and##name(int i, atomic_t *v) \ +{ \ + register int w0 asm ("w0") = i; \ + register atomic_t *x1 asm ("x1") = v; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN( \ + /* LL/SC */ \ + " nop\n" \ + __LL_SC_ATOMIC(fetch_and##name), \ + /* LSE atomics */ \ + " mvn %w[i], %w[i]\n" \ + " ldclr" #mb " %w[i], %w[i], %[v]") \ + : [i] "+r" (w0), [v] "+Q" (v->counter) \ + : "r" (x1) \ + : __LL_SC_CLOBBERS, ##cl); \ + \ + return w0; \ +} + +ATOMIC_FETCH_OP_AND(_relaxed, ) +ATOMIC_FETCH_OP_AND(_acquire, a, "memory") +ATOMIC_FETCH_OP_AND(_release, l, "memory") +ATOMIC_FETCH_OP_AND( , al, "memory") + +#undef ATOMIC_FETCH_OP_AND + static inline void atomic_sub(int i, atomic_t *v) { register int w0 asm ("w0") = i; @@ -164,57 +194,87 @@ ATOMIC_OP_SUB_RETURN(_release, l, "memory") ATOMIC_OP_SUB_RETURN( , al, "memory") #undef ATOMIC_OP_SUB_RETURN -#undef __LL_SC_ATOMIC - -#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op) - -static inline void atomic64_andnot(long i, atomic64_t *v) -{ - register long x0 asm ("x0") = i; - register atomic64_t *x1 asm ("x1") = v; - asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot), - " stclr %[i], %[v]\n") - : [i] "+r" (x0), [v] "+Q" (v->counter) - : "r" (x1) - : __LL_SC_CLOBBERS); +#define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \ +static inline int atomic_fetch_sub##name(int i, atomic_t *v) \ +{ \ + register int w0 asm ("w0") = i; \ + register atomic_t *x1 asm ("x1") = v; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN( \ + /* LL/SC */ \ + " nop\n" \ + __LL_SC_ATOMIC(fetch_sub##name), \ + /* LSE atomics */ \ + " neg %w[i], %w[i]\n" \ + " ldadd" #mb " %w[i], %w[i], %[v]") \ + : [i] "+r" (w0), [v] "+Q" (v->counter) \ + : "r" (x1) \ + : __LL_SC_CLOBBERS, ##cl); \ + \ + return w0; \ } -static inline void atomic64_or(long i, atomic64_t *v) -{ - register long x0 asm ("x0") = i; - register atomic64_t *x1 asm ("x1") = v; +ATOMIC_FETCH_OP_SUB(_relaxed, ) +ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") +ATOMIC_FETCH_OP_SUB(_release, l, "memory") +ATOMIC_FETCH_OP_SUB( , al, "memory") - asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or), - " stset %[i], %[v]\n") - : [i] "+r" (x0), [v] "+Q" (v->counter) - : "r" (x1) - : __LL_SC_CLOBBERS); +#undef ATOMIC_FETCH_OP_SUB +#undef __LL_SC_ATOMIC + +#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op) +#define ATOMIC64_OP(op, asm_op) \ +static inline void atomic64_##op(long i, atomic64_t *v) \ +{ \ + register long x0 asm ("x0") = i; \ + register atomic64_t *x1 asm ("x1") = v; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op), \ +" " #asm_op " %[i], %[v]\n") \ + : [i] "+r" (x0), [v] "+Q" (v->counter) \ + : "r" (x1) \ + : __LL_SC_CLOBBERS); \ } -static inline void atomic64_xor(long i, atomic64_t *v) -{ - register long x0 asm ("x0") = i; - register atomic64_t *x1 asm ("x1") = v; +ATOMIC64_OP(andnot, stclr) +ATOMIC64_OP(or, stset) +ATOMIC64_OP(xor, steor) +ATOMIC64_OP(add, stadd) - asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor), - " steor %[i], %[v]\n") - : [i] "+r" (x0), [v] "+Q" (v->counter) - : "r" (x1) - : __LL_SC_CLOBBERS); +#undef ATOMIC64_OP + +#define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \ +static inline long atomic64_fetch_##op##name(long i, atomic64_t *v) \ +{ \ + register long x0 asm ("x0") = i; \ + register atomic64_t *x1 asm ("x1") = v; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN( \ + /* LL/SC */ \ + __LL_SC_ATOMIC64(fetch_##op##name), \ + /* LSE atomics */ \ +" " #asm_op #mb " %[i], %[i], %[v]") \ + : [i] "+r" (x0), [v] "+Q" (v->counter) \ + : "r" (x1) \ + : __LL_SC_CLOBBERS, ##cl); \ + \ + return x0; \ } -static inline void atomic64_add(long i, atomic64_t *v) -{ - register long x0 asm ("x0") = i; - register atomic64_t *x1 asm ("x1") = v; +#define ATOMIC64_FETCH_OPS(op, asm_op) \ + ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \ + ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ + ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ + ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") - asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add), - " stadd %[i], %[v]\n") - : [i] "+r" (x0), [v] "+Q" (v->counter) - : "r" (x1) - : __LL_SC_CLOBBERS); -} +ATOMIC64_FETCH_OPS(andnot, ldclr) +ATOMIC64_FETCH_OPS(or, ldset) +ATOMIC64_FETCH_OPS(xor, ldeor) +ATOMIC64_FETCH_OPS(add, ldadd) + +#undef ATOMIC64_FETCH_OP +#undef ATOMIC64_FETCH_OPS #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \ static inline long atomic64_add_return##name(long i, atomic64_t *v) \ @@ -260,6 +320,33 @@ static inline void atomic64_and(long i, atomic64_t *v) : __LL_SC_CLOBBERS); } +#define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \ +static inline long atomic64_fetch_and##name(long i, atomic64_t *v) \ +{ \ + register long x0 asm ("w0") = i; \ + register atomic64_t *x1 asm ("x1") = v; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN( \ + /* LL/SC */ \ + " nop\n" \ + __LL_SC_ATOMIC64(fetch_and##name), \ + /* LSE atomics */ \ + " mvn %[i], %[i]\n" \ + " ldclr" #mb " %[i], %[i], %[v]") \ + : [i] "+r" (x0), [v] "+Q" (v->counter) \ + : "r" (x1) \ + : __LL_SC_CLOBBERS, ##cl); \ + \ + return x0; \ +} + +ATOMIC64_FETCH_OP_AND(_relaxed, ) +ATOMIC64_FETCH_OP_AND(_acquire, a, "memory") +ATOMIC64_FETCH_OP_AND(_release, l, "memory") +ATOMIC64_FETCH_OP_AND( , al, "memory") + +#undef ATOMIC64_FETCH_OP_AND + static inline void atomic64_sub(long i, atomic64_t *v) { register long x0 asm ("x0") = i; @@ -306,6 +393,33 @@ ATOMIC64_OP_SUB_RETURN( , al, "memory") #undef ATOMIC64_OP_SUB_RETURN +#define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \ +static inline long atomic64_fetch_sub##name(long i, atomic64_t *v) \ +{ \ + register long x0 asm ("w0") = i; \ + register atomic64_t *x1 asm ("x1") = v; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN( \ + /* LL/SC */ \ + " nop\n" \ + __LL_SC_ATOMIC64(fetch_sub##name), \ + /* LSE atomics */ \ + " neg %[i], %[i]\n" \ + " ldadd" #mb " %[i], %[i], %[v]") \ + : [i] "+r" (x0), [v] "+Q" (v->counter) \ + : "r" (x1) \ + : __LL_SC_CLOBBERS, ##cl); \ + \ + return x0; \ +} + +ATOMIC64_FETCH_OP_SUB(_relaxed, ) +ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory") +ATOMIC64_FETCH_OP_SUB(_release, l, "memory") +ATOMIC64_FETCH_OP_SUB( , al, "memory") + +#undef ATOMIC64_FETCH_OP_SUB + static inline long atomic64_dec_if_positive(atomic64_t *v) { register long x0 asm ("x0") = (long)v; diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index dae5c49618db..4eea7f618dce 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -91,6 +91,19 @@ do { \ __u.__val; \ }) +#define smp_cond_load_acquire(ptr, cond_expr) \ +({ \ + typeof(ptr) __PTR = (ptr); \ + typeof(*ptr) VAL; \ + for (;;) { \ + VAL = smp_load_acquire(__PTR); \ + if (cond_expr) \ + break; \ + __cmpwait_relaxed(__PTR, VAL); \ + } \ + VAL; \ +}) + #include #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h new file mode 100644 index 000000000000..09f65339d66d --- /dev/null +++ b/arch/arm64/include/asm/checksum.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2016 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef __ASM_CHECKSUM_H +#define __ASM_CHECKSUM_H + +#include + +static inline __sum16 csum_fold(__wsum csum) +{ + u32 sum = (__force u32)csum; + sum += (sum >> 16) | (sum << 16); + return ~(__force __sum16)(sum >> 16); +} +#define csum_fold csum_fold + +static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) +{ + __uint128_t tmp; + u64 sum; + + tmp = *(const __uint128_t *)iph; + iph += 16; + ihl -= 4; + tmp += ((tmp >> 64) | (tmp << 64)); + sum = tmp >> 64; + do { + sum += *(const u32 *)iph; + iph += 4; + } while (--ihl); + + sum += ((sum >> 32) | (sum << 32)); + return csum_fold(sum >> 32); +} +#define ip_fast_csum ip_fast_csum + +#include + +#endif /* __ASM_CHECKSUM_H */ diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index 510c7b404454..bd86a79491bc 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -224,4 +224,55 @@ __CMPXCHG_GEN(_mb) __ret; \ }) +#define __CMPWAIT_CASE(w, sz, name) \ +static inline void __cmpwait_case_##name(volatile void *ptr, \ + unsigned long val) \ +{ \ + unsigned long tmp; \ + \ + asm volatile( \ + " ldxr" #sz "\t%" #w "[tmp], %[v]\n" \ + " eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \ + " cbnz %" #w "[tmp], 1f\n" \ + " wfe\n" \ + "1:" \ + : [tmp] "=&r" (tmp), [v] "+Q" (*(unsigned long *)ptr) \ + : [val] "r" (val)); \ +} + +__CMPWAIT_CASE(w, b, 1); +__CMPWAIT_CASE(w, h, 2); +__CMPWAIT_CASE(w, , 4); +__CMPWAIT_CASE( , , 8); + +#undef __CMPWAIT_CASE + +#define __CMPWAIT_GEN(sfx) \ +static inline void __cmpwait##sfx(volatile void *ptr, \ + unsigned long val, \ + int size) \ +{ \ + switch (size) { \ + case 1: \ + return __cmpwait_case##sfx##_1(ptr, (u8)val); \ + case 2: \ + return __cmpwait_case##sfx##_2(ptr, (u16)val); \ + case 4: \ + return __cmpwait_case##sfx##_4(ptr, val); \ + case 8: \ + return __cmpwait_case##sfx##_8(ptr, val); \ + default: \ + BUILD_BUG(); \ + } \ + \ + unreachable(); \ +} + +__CMPWAIT_GEN() + +#undef __CMPWAIT_GEN + +#define __cmpwait_relaxed(ptr, val) \ + __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr))) + #endif /* __ASM_CMPXCHG_H */ diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h index 13a6103130cd..889226b4c6e1 100644 --- a/arch/arm64/include/asm/cpu.h +++ b/arch/arm64/include/asm/cpu.h @@ -25,10 +25,12 @@ */ struct cpuinfo_arm64 { struct cpu cpu; + struct kobject kobj; u32 reg_ctr; u32 reg_cntfrq; u32 reg_dczid; u32 reg_midr; + u32 reg_revidr; u64 reg_id_aa64dfr0; u64 reg_id_aa64dfr1; diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 224efe730e46..49dd1bd3ea50 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -191,7 +191,9 @@ void __init setup_cpu_features(void); void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, const char *info); +void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps); void check_local_cpu_errata(void); +void __init enable_errata_workarounds(void); void verify_local_cpu_errata(void); void verify_local_cpu_capabilities(void); diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 87e1985f3be8..9d9fd4b9a72e 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -80,12 +80,14 @@ #define APM_CPU_PART_POTENZA 0x000 #define CAVIUM_CPU_PART_THUNDERX 0x0A1 +#define CAVIUM_CPU_PART_THUNDERX_81XX 0x0A2 #define BRCM_CPU_PART_VULCAN 0x516 #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) +#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/debug-monitors.h b/arch/arm64/include/asm/debug-monitors.h index 2fcb9b7c876c..4b6b3f72a215 100644 --- a/arch/arm64/include/asm/debug-monitors.h +++ b/arch/arm64/include/asm/debug-monitors.h @@ -66,6 +66,11 @@ #define CACHE_FLUSH_IS_SAFE 1 +/* kprobes BRK opcodes with ESR encoding */ +#define BRK64_ESR_MASK 0xFFFF +#define BRK64_ESR_KPROBES 0x0004 +#define BRK64_OPCODE_KPROBES (AARCH64_BREAK_MON | (BRK64_ESR_KPROBES << 5)) + /* AArch32 */ #define DBG_ESR_EVT_BKPT 0x4 #define DBG_ESR_EVT_VECC 0x5 diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 622db3c6474e..a9e54aad15ef 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -14,8 +14,7 @@ extern void efi_init(void); #endif int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); - -#define efi_set_mapping_permissions efi_create_mapping +int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); #define arch_efi_call_virt_setup() \ ({ \ @@ -23,10 +22,10 @@ int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); efi_virtmap_load(); \ }) -#define arch_efi_call_virt(f, args...) \ +#define arch_efi_call_virt(p, f, args...) \ ({ \ efi_##f##_t *__f; \ - __f = efi.systab->runtime->f; \ + __f = p->f; \ __f(args); \ }) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 77eeb2cc648f..f772e15c4766 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -74,6 +74,7 @@ #define ESR_ELx_EC_SHIFT (26) #define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT) +#define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT) #define ESR_ELx_IL (UL(1) << 25) #define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1) diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 30e50eb54a67..1dbaa901d7e5 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -120,6 +120,29 @@ enum aarch64_insn_register { AARCH64_INSN_REG_SP = 31 /* Stack pointer: as load/store base reg */ }; +enum aarch64_insn_special_register { + AARCH64_INSN_SPCLREG_SPSR_EL1 = 0xC200, + AARCH64_INSN_SPCLREG_ELR_EL1 = 0xC201, + AARCH64_INSN_SPCLREG_SP_EL0 = 0xC208, + AARCH64_INSN_SPCLREG_SPSEL = 0xC210, + AARCH64_INSN_SPCLREG_CURRENTEL = 0xC212, + AARCH64_INSN_SPCLREG_DAIF = 0xDA11, + AARCH64_INSN_SPCLREG_NZCV = 0xDA10, + AARCH64_INSN_SPCLREG_FPCR = 0xDA20, + AARCH64_INSN_SPCLREG_DSPSR_EL0 = 0xDA28, + AARCH64_INSN_SPCLREG_DLR_EL0 = 0xDA29, + AARCH64_INSN_SPCLREG_SPSR_EL2 = 0xE200, + AARCH64_INSN_SPCLREG_ELR_EL2 = 0xE201, + AARCH64_INSN_SPCLREG_SP_EL1 = 0xE208, + AARCH64_INSN_SPCLREG_SPSR_INQ = 0xE218, + AARCH64_INSN_SPCLREG_SPSR_ABT = 0xE219, + AARCH64_INSN_SPCLREG_SPSR_UND = 0xE21A, + AARCH64_INSN_SPCLREG_SPSR_FIQ = 0xE21B, + AARCH64_INSN_SPCLREG_SPSR_EL3 = 0xF200, + AARCH64_INSN_SPCLREG_ELR_EL3 = 0xF201, + AARCH64_INSN_SPCLREG_SP_EL2 = 0xF210 +}; + enum aarch64_insn_variant { AARCH64_INSN_VARIANT_32BIT, AARCH64_INSN_VARIANT_64BIT @@ -223,8 +246,15 @@ static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ { return (val); } +__AARCH64_INSN_FUNCS(adr_adrp, 0x1F000000, 0x10000000) +__AARCH64_INSN_FUNCS(prfm_lit, 0xFF000000, 0xD8000000) __AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800) __AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800) +__AARCH64_INSN_FUNCS(ldr_lit, 0xBF000000, 0x18000000) +__AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000) +__AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000) +__AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000) +__AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000) __AARCH64_INSN_FUNCS(stp_post, 0x7FC00000, 0x28800000) __AARCH64_INSN_FUNCS(ldp_post, 0x7FC00000, 0x28C00000) __AARCH64_INSN_FUNCS(stp_pre, 0x7FC00000, 0x29800000) @@ -273,10 +303,15 @@ __AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001) __AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) __AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003) __AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000) +__AARCH64_INSN_FUNCS(exception, 0xFF000000, 0xD4000000) __AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F) __AARCH64_INSN_FUNCS(br, 0xFFFFFC1F, 0xD61F0000) __AARCH64_INSN_FUNCS(blr, 0xFFFFFC1F, 0xD63F0000) __AARCH64_INSN_FUNCS(ret, 0xFFFFFC1F, 0xD65F0000) +__AARCH64_INSN_FUNCS(eret, 0xFFFFFFFF, 0xD69F03E0) +__AARCH64_INSN_FUNCS(mrs, 0xFFF00000, 0xD5300000) +__AARCH64_INSN_FUNCS(msr_imm, 0xFFF8F01F, 0xD500401F) +__AARCH64_INSN_FUNCS(msr_reg, 0xFFF00000, 0xD5100000) #undef __AARCH64_INSN_FUNCS @@ -286,6 +321,8 @@ bool aarch64_insn_is_branch_imm(u32 insn); int aarch64_insn_read(void *addr, u32 *insnp); int aarch64_insn_write(void *addr, u32 insn); enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn); +bool aarch64_insn_uses_literal(u32 insn); +bool aarch64_insn_is_branch(u32 insn); u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn); u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, u32 insn, u64 imm); @@ -367,9 +404,13 @@ bool aarch32_insn_is_wide(u32 insn); #define A32_RT_OFFSET 12 #define A32_RT2_OFFSET 0 +u32 aarch64_insn_extract_system_reg(u32 insn); u32 aarch32_insn_extract_reg_num(u32 insn, int offset); u32 aarch32_insn_mcr_extract_opc2(u32 insn); u32 aarch32_insn_mcr_extract_crm(u32 insn); + +typedef bool (pstate_check_t)(unsigned long); +extern pstate_check_t * const aarch32_opcode_cond_checks[16]; #endif /* __ASSEMBLY__ */ #endif /* __ASM_INSN_H */ diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 44be1e03ed65..9b6e408cfa51 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -174,13 +174,15 @@ extern void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size); #define iounmap __iounmap /* - * io{read,write}{16,32}be() macros + * io{read,write}{16,32,64}be() macros */ #define ioread16be(p) ({ __u16 __v = be16_to_cpu((__force __be16)__raw_readw(p)); __iormb(); __v; }) #define ioread32be(p) ({ __u32 __v = be32_to_cpu((__force __be32)__raw_readl(p)); __iormb(); __v; }) +#define ioread64be(p) ({ __u64 __v = be64_to_cpu((__force __be64)__raw_readq(p)); __iormb(); __v; }) #define iowrite16be(v,p) ({ __iowmb(); __raw_writew((__force __u16)cpu_to_be16(v), p); }) #define iowrite32be(v,p) ({ __iowmb(); __raw_writel((__force __u32)cpu_to_be32(v), p); }) +#define iowrite64be(v,p) ({ __iowmb(); __raw_writeq((__force __u64)cpu_to_be64(v), p); }) /* * Convert a physical pointer to a virtual kernel pointer for /dev/mem diff --git a/arch/arm64/include/asm/irqflags.h b/arch/arm64/include/asm/irqflags.h index 11cc941bd107..8c581281fa12 100644 --- a/arch/arm64/include/asm/irqflags.h +++ b/arch/arm64/include/asm/irqflags.h @@ -110,8 +110,5 @@ static inline int arch_irqs_disabled_flags(unsigned long flags) : : "r" (flags) : "memory"); \ } while (0) -#define local_dbg_enable() asm("msr daifclr, #8" : : : "memory") -#define local_dbg_disable() asm("msr daifset, #8" : : : "memory") - #endif #endif diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h new file mode 100644 index 000000000000..04744dc5fb61 --- /dev/null +++ b/arch/arm64/include/asm/kexec.h @@ -0,0 +1,48 @@ +/* + * kexec for arm64 + * + * Copyright (C) Linaro. + * Copyright (C) Huawei Futurewei Technologies. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ARM64_KEXEC_H +#define _ARM64_KEXEC_H + +/* Maximum physical address we can use pages from */ + +#define KEXEC_SOURCE_MEMORY_LIMIT (-1UL) + +/* Maximum address we can reach in physical address mode */ + +#define KEXEC_DESTINATION_MEMORY_LIMIT (-1UL) + +/* Maximum address we can use for the control code buffer */ + +#define KEXEC_CONTROL_MEMORY_LIMIT (-1UL) + +#define KEXEC_CONTROL_PAGE_SIZE 4096 + +#define KEXEC_ARCH KEXEC_ARCH_AARCH64 + +#ifndef __ASSEMBLY__ + +/** + * crash_setup_regs() - save registers for the panic kernel + * + * @newregs: registers are saved here + * @oldregs: registers to be saved (may be %NULL) + */ + +static inline void crash_setup_regs(struct pt_regs *newregs, + struct pt_regs *oldregs) +{ + /* Empty routine needed to avoid build errors. */ +} + +#endif /* __ASSEMBLY__ */ + +#endif diff --git a/arch/arm64/include/asm/kprobes.h b/arch/arm64/include/asm/kprobes.h new file mode 100644 index 000000000000..61b49150dfa3 --- /dev/null +++ b/arch/arm64/include/asm/kprobes.h @@ -0,0 +1,62 @@ +/* + * arch/arm64/include/asm/kprobes.h + * + * Copyright (C) 2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#ifndef _ARM_KPROBES_H +#define _ARM_KPROBES_H + +#include +#include +#include + +#define __ARCH_WANT_KPROBES_INSN_SLOT +#define MAX_INSN_SIZE 1 +#define MAX_STACK_SIZE 128 + +#define flush_insn_slot(p) do { } while (0) +#define kretprobe_blacklist_size 0 + +#include + +struct prev_kprobe { + struct kprobe *kp; + unsigned int status; +}; + +/* Single step context for kprobe */ +struct kprobe_step_ctx { + unsigned long ss_pending; + unsigned long match_addr; +}; + +/* per-cpu kprobe control block */ +struct kprobe_ctlblk { + unsigned int kprobe_status; + unsigned long saved_irqflag; + struct prev_kprobe prev_kprobe; + struct kprobe_step_ctx ss_ctx; + struct pt_regs jprobe_saved_regs; + char jprobes_stack[MAX_STACK_SIZE]; +}; + +void arch_remove_kprobe(struct kprobe *); +int kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr); +int kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data); +int kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr); +int kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr); +void kretprobe_trampoline(void); +void __kprobes *trampoline_probe_handler(struct pt_regs *regs); + +#endif /* _ARM_KPROBES_H */ diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 40bc1681b6d5..4cdeae3b17c6 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -210,7 +210,7 @@ static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu) static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_get_hsr(vcpu) >> ESR_ELx_EC_SHIFT; + return ESR_ELx_EC(kvm_vcpu_get_hsr(vcpu)); } static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 97b1d8f26b9c..8d9fce037b2f 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -34,7 +34,7 @@ extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); extern void init_mem_pgprot(void); extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, - pgprot_t prot); + pgprot_t prot, bool allow_block_mappings); extern void *fixmap_remap_fdt(phys_addr_t dt_phys); #endif diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h index e9b4f2942335..600887e491fd 100644 --- a/arch/arm64/include/asm/numa.h +++ b/arch/arm64/include/asm/numa.h @@ -5,6 +5,8 @@ #ifdef CONFIG_NUMA +#define NR_NODE_MEMBLKS (MAX_NUMNODES * 2) + /* currently, arm64 implements flat NUMA topology */ #define parent_node(node) (node) diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index ff98585d085a..d25f4f137c2a 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -26,7 +26,7 @@ #define check_pgt_cache() do { } while (0) -#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) +#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO) #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) #if CONFIG_PGTABLE_LEVELS > 2 diff --git a/arch/arm64/include/asm/probes.h b/arch/arm64/include/asm/probes.h new file mode 100644 index 000000000000..5af574d632fa --- /dev/null +++ b/arch/arm64/include/asm/probes.h @@ -0,0 +1,35 @@ +/* + * arch/arm64/include/asm/probes.h + * + * Copyright (C) 2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#ifndef _ARM_PROBES_H +#define _ARM_PROBES_H + +#include + +struct kprobe; +struct arch_specific_insn; + +typedef u32 kprobe_opcode_t; +typedef void (kprobes_handler_t) (u32 opcode, long addr, struct pt_regs *); + +/* architecture specific copy of original instruction */ +struct arch_specific_insn { + kprobe_opcode_t *insn; + pstate_check_t *pstate_cc; + kprobes_handler_t *handler; + /* restore address after step xol */ + unsigned long restore; +}; + +#endif diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index cef1cf398356..ace0a96e7d6e 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -192,5 +192,6 @@ static inline void spin_lock_prefetch(const void *ptr) void cpu_enable_pan(void *__unused); void cpu_enable_uao(void *__unused); +void cpu_enable_cache_maint_trap(void *__unused); #endif /* __ASM_PROCESSOR_H */ diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h new file mode 100644 index 000000000000..07b8ed037dee --- /dev/null +++ b/arch/arm64/include/asm/ptdump.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef __ASM_PTDUMP_H +#define __ASM_PTDUMP_H + +#ifdef CONFIG_ARM64_PTDUMP + +#include + +struct addr_marker { + unsigned long start_address; + char *name; +}; + +struct ptdump_info { + struct mm_struct *mm; + const struct addr_marker *markers; + unsigned long base_addr; + unsigned long max_addr; +}; + +int ptdump_register(struct ptdump_info *info, const char *name); + +#else +static inline int ptdump_register(struct ptdump_info *info, const char *name) +{ + return 0; +} +#endif /* CONFIG_ARM64_PTDUMP */ + +#endif /* __ASM_PTDUMP_H */ diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index a307eb6e7fa8..ada08b5b036d 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -46,7 +46,6 @@ #define COMPAT_PSR_MODE_UND 0x0000001b #define COMPAT_PSR_MODE_SYS 0x0000001f #define COMPAT_PSR_T_BIT 0x00000020 -#define COMPAT_PSR_E_BIT 0x00000200 #define COMPAT_PSR_F_BIT 0x00000040 #define COMPAT_PSR_I_BIT 0x00000080 #define COMPAT_PSR_A_BIT 0x00000100 @@ -74,6 +73,7 @@ #define COMPAT_PT_DATA_ADDR 0x10004 #define COMPAT_PT_TEXT_END_ADDR 0x10008 #ifndef __ASSEMBLY__ +#include /* sizeof(struct user) for AArch32 */ #define COMPAT_USER_SZ 296 @@ -117,8 +117,12 @@ struct pt_regs { }; u64 orig_x0; u64 syscallno; + u64 orig_addr_limit; + u64 unused; // maintain 16 byte alignment }; +#define MAX_REG_OFFSET offsetof(struct pt_regs, pstate) + #define arch_has_single_step() (1) #ifdef CONFIG_COMPAT @@ -144,9 +148,58 @@ struct pt_regs { #define fast_interrupts_enabled(regs) \ (!((regs)->pstate & PSR_F_BIT)) -#define user_stack_pointer(regs) \ +#define GET_USP(regs) \ (!compat_user_mode(regs) ? (regs)->sp : (regs)->compat_sp) +#define SET_USP(ptregs, value) \ + (!compat_user_mode(regs) ? ((regs)->sp = value) : ((regs)->compat_sp = value)) + +extern int regs_query_register_offset(const char *name); +extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n); + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten + * @offset: offset of the register. + * + * regs_get_register returns the value of a register whose offset from @regs. + * The @offset is the offset of the register in struct pt_regs. + * If @offset is bigger than MAX_REG_OFFSET, this returns 0. + */ +static inline u64 regs_get_register(struct pt_regs *regs, unsigned int offset) +{ + u64 val = 0; + + WARN_ON(offset & 7); + + offset >>= 3; + switch (offset) { + case 0 ... 30: + val = regs->regs[offset]; + break; + case offsetof(struct pt_regs, sp) >> 3: + val = regs->sp; + break; + case offsetof(struct pt_regs, pc) >> 3: + val = regs->pc; + break; + case offsetof(struct pt_regs, pstate) >> 3: + val = regs->pstate; + break; + default: + val = 0; + } + + return val; +} + +/* Valid only for Kernel mode traps. */ +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->sp; +} + static inline unsigned long regs_return_value(struct pt_regs *regs) { return regs->regs[0]; @@ -156,8 +209,15 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) struct task_struct; int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task); -#define instruction_pointer(regs) ((unsigned long)(regs)->pc) +#define GET_IP(regs) ((unsigned long)(regs)->pc) +#define SET_IP(regs, value) ((regs)->pc = ((u64) (value))) + +#define GET_FP(ptregs) ((unsigned long)(ptregs)->regs[29]) +#define SET_FP(ptregs, value) ((ptregs)->regs[29] = ((u64) (value))) + +#include +#undef profile_pc extern unsigned long profile_pc(struct pt_regs *regs); #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 433e50405274..022644704a93 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -124,6 +124,18 @@ static inline void cpu_panic_kernel(void) cpu_park_loop(); } +/* + * If a secondary CPU enters the kernel but fails to come online, + * (e.g. due to mismatched features), and cannot exit the kernel, + * we increment cpus_stuck_in_kernel and leave the CPU in a + * quiesecent loop within the kernel text. The memory containing + * this loop must not be re-used for anything else as the 'stuck' + * core is executing it. + * + * This function is used to inhibit features like kexec and hibernate. + */ +bool cpus_are_stuck_in_kernel(void); + #endif /* ifndef __ASSEMBLY__ */ #endif /* ifndef __ASM_SMP_H */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 751e901c8d37..cc06794b7346 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -98,11 +98,11 @@ SCTLR_ELx_SA | SCTLR_ELx_I) /* SCTLR_EL1 specific flags. */ +#define SCTLR_EL1_UCI (1 << 26) #define SCTLR_EL1_SPAN (1 << 23) #define SCTLR_EL1_SED (1 << 8) #define SCTLR_EL1_CP15BEN (1 << 5) - /* id_aa64isar0 */ #define ID_AA64ISAR0_RDM_SHIFT 28 #define ID_AA64ISAR0_ATOMICS_SHIFT 20 diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h index 0cc2f29bf9da..9cd03f3e812f 100644 --- a/arch/arm64/include/asm/traps.h +++ b/arch/arm64/include/asm/traps.h @@ -34,6 +34,8 @@ struct undef_hook { void register_undef_hook(struct undef_hook *hook); void unregister_undef_hook(struct undef_hook *hook); +void arm64_notify_segfault(struct pt_regs *regs, unsigned long addr); + #ifdef CONFIG_FUNCTION_GRAPH_TRACER static inline int __in_irqentry_text(unsigned long ptr) { diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 9e397a542756..5e834d10b291 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -21,6 +21,7 @@ /* * User space memory access functions */ +#include #include #include @@ -256,15 +257,29 @@ do { \ -EFAULT; \ }) -extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n); -extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n); +extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); +extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n); extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); +static inline unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n) +{ + kasan_check_write(to, n); + return __arch_copy_from_user(to, from, n); +} + +static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n) +{ + kasan_check_read(from, n); + return __arch_copy_to_user(to, from, n); +} + static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) { + kasan_check_write(to, n); + if (access_ok(VERIFY_READ, from, n)) - n = __copy_from_user(to, from, n); + n = __arch_copy_from_user(to, from, n); else /* security hole - plug it */ memset(to, 0, n); return n; @@ -272,8 +287,10 @@ static inline unsigned long __must_check copy_from_user(void *to, const void __u static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n) { + kasan_check_read(from, n); + if (access_ok(VERIFY_WRITE, to, n)) - n = __copy_to_user(to, from, n); + n = __arch_copy_to_user(to, from, n); return n; } diff --git a/arch/arm64/include/asm/vdso_datapage.h b/arch/arm64/include/asm/vdso_datapage.h index de66199673d7..2b9a63771eda 100644 --- a/arch/arm64/include/asm/vdso_datapage.h +++ b/arch/arm64/include/asm/vdso_datapage.h @@ -22,6 +22,8 @@ struct vdso_data { __u64 cs_cycle_last; /* Timebase at clocksource init */ + __u64 raw_time_sec; /* Raw time */ + __u64 raw_time_nsec; __u64 xtime_clock_sec; /* Kernel time */ __u64 xtime_clock_nsec; __u64 xtime_coarse_sec; /* Coarse time */ @@ -29,8 +31,10 @@ struct vdso_data { __u64 wtm_clock_sec; /* Wall to monotonic time */ __u64 wtm_clock_nsec; __u32 tb_seq_count; /* Timebase sequence counter */ - __u32 cs_mult; /* Clocksource multiplier */ - __u32 cs_shift; /* Clocksource shift */ + /* cs_* members must be adjacent and in this order (ldp accesses) */ + __u32 cs_mono_mult; /* NTP-adjusted clocksource multiplier */ + __u32 cs_shift; /* Clocksource shift (mono = raw) */ + __u32 cs_raw_mult; /* Raw clocksource multiplier */ __u32 tz_minuteswest; /* Whacky timezone stuff */ __u32 tz_dsttime; __u32 use_syscall; diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h index dcbcf8dcbefb..bbc6a8cf83f1 100644 --- a/arch/arm64/include/asm/virt.h +++ b/arch/arm64/include/asm/virt.h @@ -34,6 +34,11 @@ */ #define HVC_SET_VECTORS 1 +/* + * HVC_SOFT_RESTART - CPU soft reset, used by the cpu_soft_restart routine. + */ +#define HVC_SOFT_RESTART 2 + #define BOOT_CPU_MODE_EL1 (0xe11) #define BOOT_CPU_MODE_EL2 (0xe12) diff --git a/arch/arm64/include/asm/xen/xen-ops.h b/arch/arm64/include/asm/xen/xen-ops.h new file mode 100644 index 000000000000..ec154e719b11 --- /dev/null +++ b/arch/arm64/include/asm/xen/xen-ops.h @@ -0,0 +1,6 @@ +#ifndef _ASM_XEN_OPS_H +#define _ASM_XEN_OPS_H + +void xen_efi_runtime_setup(void); + +#endif /* _ASM_XEN_OPS_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2173149d8954..14f7b651c787 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -26,8 +26,7 @@ $(obj)/%.stub.o: $(obj)/%.o FORCE $(call if_changed,objcopy) arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ - sys_compat.o entry32.o \ - ../../arm/kernel/opcodes.o + sys_compat.o entry32.o arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o @@ -42,16 +41,15 @@ arm64-obj-$(CONFIG_EFI) += efi.o efi-entry.stub.o arm64-obj-$(CONFIG_PCI) += pci.o arm64-obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o arm64-obj-$(CONFIG_ACPI) += acpi.o +arm64-obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o arm64-obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o arm64-obj-$(CONFIG_PARAVIRT) += paravirt.o arm64-obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o arm64-obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o +arm64-obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o \ + cpu-reset.o -obj-y += $(arm64-obj-y) vdso/ +obj-y += $(arm64-obj-y) vdso/ probes/ obj-m += $(arm64-obj-m) head-y := head.o extra-y += $(head-y) vmlinux.lds - -# vDSO - this must be built first to generate the symbol offsets -$(call objectify,$(arm64-obj-y)): $(obj)/vdso/vdso-offsets.h -$(obj)/vdso/vdso-offsets.h: $(obj)/vdso diff --git a/arch/arm64/kernel/acpi_numa.c b/arch/arm64/kernel/acpi_numa.c new file mode 100644 index 000000000000..f85149cc7c71 --- /dev/null +++ b/arch/arm64/kernel/acpi_numa.c @@ -0,0 +1,112 @@ +/* + * ACPI 5.1 based NUMA setup for ARM64 + * Lots of code was borrowed from arch/x86/mm/srat.c + * + * Copyright 2004 Andi Kleen, SuSE Labs. + * Copyright (C) 2013-2016, Linaro Ltd. + * Author: Hanjun Guo + * + * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs. + * + * Called from acpi_numa_init while reading the SRAT and SLIT tables. + * Assumes all memory regions belonging to a single proximity domain + * are in one chunk. Holes between them will be included in the node. + */ + +#define pr_fmt(fmt) "ACPI: NUMA: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static int cpus_in_srat; + +struct __node_cpu_hwid { + u32 node_id; /* logical node containing this CPU */ + u64 cpu_hwid; /* MPIDR for this CPU */ +}; + +static struct __node_cpu_hwid early_node_cpu_hwid[NR_CPUS] = { +[0 ... NR_CPUS - 1] = {NUMA_NO_NODE, PHYS_CPUID_INVALID} }; + +int acpi_numa_get_nid(unsigned int cpu, u64 hwid) +{ + int i; + + for (i = 0; i < cpus_in_srat; i++) { + if (hwid == early_node_cpu_hwid[i].cpu_hwid) + return early_node_cpu_hwid[i].node_id; + } + + return NUMA_NO_NODE; +} + +/* Callback for Proximity Domain -> ACPI processor UID mapping */ +void __init acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa) +{ + int pxm, node; + phys_cpuid_t mpidr; + + if (srat_disabled()) + return; + + if (pa->header.length < sizeof(struct acpi_srat_gicc_affinity)) { + pr_err("SRAT: Invalid SRAT header length: %d\n", + pa->header.length); + bad_srat(); + return; + } + + if (!(pa->flags & ACPI_SRAT_GICC_ENABLED)) + return; + + if (cpus_in_srat >= NR_CPUS) { + pr_warn_once("SRAT: cpu_to_node_map[%d] is too small, may not be able to use all cpus\n", + NR_CPUS); + return; + } + + pxm = pa->proximity_domain; + node = acpi_map_pxm_to_node(pxm); + + if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { + pr_err("SRAT: Too many proximity domains %d\n", pxm); + bad_srat(); + return; + } + + mpidr = acpi_map_madt_entry(pa->acpi_processor_uid); + if (mpidr == PHYS_CPUID_INVALID) { + pr_err("SRAT: PXM %d with ACPI ID %d has no valid MPIDR in MADT\n", + pxm, pa->acpi_processor_uid); + bad_srat(); + return; + } + + early_node_cpu_hwid[cpus_in_srat].node_id = node; + early_node_cpu_hwid[cpus_in_srat].cpu_hwid = mpidr; + node_set(node, numa_nodes_parsed); + cpus_in_srat++; + pr_info("SRAT: PXM %d -> MPIDR 0x%Lx -> Node %d\n", + pxm, mpidr, node); +} + +int __init arm64_acpi_numa_init(void) +{ + int ret; + + ret = acpi_numa_init(); + if (ret) + return ret; + + return srat_disabled() ? -EINVAL : 0; +} diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index 678f30b05a45..78f368039c79 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -34,8 +35,8 @@ EXPORT_SYMBOL(copy_page); EXPORT_SYMBOL(clear_page); /* user mem (segment) */ -EXPORT_SYMBOL(__copy_from_user); -EXPORT_SYMBOL(__copy_to_user); +EXPORT_SYMBOL(__arch_copy_from_user); +EXPORT_SYMBOL(__arch_copy_to_user); EXPORT_SYMBOL(__clear_user); EXPORT_SYMBOL(__copy_in_user); @@ -68,6 +69,7 @@ EXPORT_SYMBOL(test_and_change_bit); #ifdef CONFIG_FUNCTION_TRACER EXPORT_SYMBOL(_mcount); +NOKPROBE_SYMBOL(_mcount); #endif /* arm-smccc */ diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index c37202c0c838..5f72475e2e3b 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -316,28 +316,6 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table) */ #define TYPE_SWPB (1 << 22) -/* - * Set up process info to signal segmentation fault - called on access error. - */ -static void set_segfault(struct pt_regs *regs, unsigned long addr) -{ - siginfo_t info; - - down_read(¤t->mm->mmap_sem); - if (find_vma(current->mm, addr) == NULL) - info.si_code = SEGV_MAPERR; - else - info.si_code = SEGV_ACCERR; - up_read(¤t->mm->mmap_sem); - - info.si_signo = SIGSEGV; - info.si_errno = 0; - info.si_addr = (void *) instruction_pointer(regs); - - pr_debug("SWP{B} emulation: access caused memory abort!\n"); - arm64_notify_die("Illegal memory access", regs, &info, 0); -} - static int emulate_swpX(unsigned int address, unsigned int *data, unsigned int type) { @@ -366,6 +344,21 @@ static int emulate_swpX(unsigned int address, unsigned int *data, return res; } +#define ARM_OPCODE_CONDITION_UNCOND 0xf + +static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr) +{ + u32 cc_bits = opcode >> 28; + + if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) { + if ((*aarch32_opcode_cond_checks[cc_bits])(psr)) + return ARM_OPCODE_CONDTEST_PASS; + else + return ARM_OPCODE_CONDTEST_FAIL; + } + return ARM_OPCODE_CONDTEST_UNCOND; +} + /* * swp_handler logs the id of calling process, dissects the instruction, sanity * checks the memory location, calls emulate_swpX for the actual operation and @@ -380,7 +373,7 @@ static int swp_handler(struct pt_regs *regs, u32 instr) type = instr & TYPE_SWPB; - switch (arm_check_condition(instr, regs->pstate)) { + switch (aarch32_check_condition(instr, regs->pstate)) { case ARM_OPCODE_CONDTEST_PASS: break; case ARM_OPCODE_CONDTEST_FAIL: @@ -430,7 +423,8 @@ ret: return 0; fault: - set_segfault(regs, address); + pr_debug("SWP{B} emulation: access caused memory abort!\n"); + arm64_notify_segfault(regs, address); return 0; } @@ -461,7 +455,7 @@ static int cp15barrier_handler(struct pt_regs *regs, u32 instr) { perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); - switch (arm_check_condition(instr, regs->pstate)) { + switch (aarch32_check_condition(instr, regs->pstate)) { case ARM_OPCODE_CONDTEST_PASS: break; case ARM_OPCODE_CONDTEST_FAIL: diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index f8e5d47f0880..05070b72fc28 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -51,6 +51,17 @@ int main(void) DEFINE(S_X5, offsetof(struct pt_regs, regs[5])); DEFINE(S_X6, offsetof(struct pt_regs, regs[6])); DEFINE(S_X7, offsetof(struct pt_regs, regs[7])); + DEFINE(S_X8, offsetof(struct pt_regs, regs[8])); + DEFINE(S_X10, offsetof(struct pt_regs, regs[10])); + DEFINE(S_X12, offsetof(struct pt_regs, regs[12])); + DEFINE(S_X14, offsetof(struct pt_regs, regs[14])); + DEFINE(S_X16, offsetof(struct pt_regs, regs[16])); + DEFINE(S_X18, offsetof(struct pt_regs, regs[18])); + DEFINE(S_X20, offsetof(struct pt_regs, regs[20])); + DEFINE(S_X22, offsetof(struct pt_regs, regs[22])); + DEFINE(S_X24, offsetof(struct pt_regs, regs[24])); + DEFINE(S_X26, offsetof(struct pt_regs, regs[26])); + DEFINE(S_X28, offsetof(struct pt_regs, regs[28])); DEFINE(S_LR, offsetof(struct pt_regs, regs[30])); DEFINE(S_SP, offsetof(struct pt_regs, sp)); #ifdef CONFIG_COMPAT @@ -60,6 +71,7 @@ int main(void) DEFINE(S_PC, offsetof(struct pt_regs, pc)); DEFINE(S_ORIG_X0, offsetof(struct pt_regs, orig_x0)); DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno)); + DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit)); DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); BLANK(); DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter)); @@ -77,6 +89,7 @@ int main(void) BLANK(); DEFINE(CLOCK_REALTIME, CLOCK_REALTIME); DEFINE(CLOCK_MONOTONIC, CLOCK_MONOTONIC); + DEFINE(CLOCK_MONOTONIC_RAW, CLOCK_MONOTONIC_RAW); DEFINE(CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC); DEFINE(CLOCK_REALTIME_COARSE, CLOCK_REALTIME_COARSE); DEFINE(CLOCK_MONOTONIC_COARSE,CLOCK_MONOTONIC_COARSE); @@ -84,6 +97,8 @@ int main(void) DEFINE(NSEC_PER_SEC, NSEC_PER_SEC); BLANK(); DEFINE(VDSO_CS_CYCLE_LAST, offsetof(struct vdso_data, cs_cycle_last)); + DEFINE(VDSO_RAW_TIME_SEC, offsetof(struct vdso_data, raw_time_sec)); + DEFINE(VDSO_RAW_TIME_NSEC, offsetof(struct vdso_data, raw_time_nsec)); DEFINE(VDSO_XTIME_CLK_SEC, offsetof(struct vdso_data, xtime_clock_sec)); DEFINE(VDSO_XTIME_CLK_NSEC, offsetof(struct vdso_data, xtime_clock_nsec)); DEFINE(VDSO_XTIME_CRS_SEC, offsetof(struct vdso_data, xtime_coarse_sec)); @@ -91,7 +106,8 @@ int main(void) DEFINE(VDSO_WTM_CLK_SEC, offsetof(struct vdso_data, wtm_clock_sec)); DEFINE(VDSO_WTM_CLK_NSEC, offsetof(struct vdso_data, wtm_clock_nsec)); DEFINE(VDSO_TB_SEQ_COUNT, offsetof(struct vdso_data, tb_seq_count)); - DEFINE(VDSO_CS_MULT, offsetof(struct vdso_data, cs_mult)); + DEFINE(VDSO_CS_MONO_MULT, offsetof(struct vdso_data, cs_mono_mult)); + DEFINE(VDSO_CS_RAW_MULT, offsetof(struct vdso_data, cs_raw_mult)); DEFINE(VDSO_CS_SHIFT, offsetof(struct vdso_data, cs_shift)); DEFINE(VDSO_TZ_MINWEST, offsetof(struct vdso_data, tz_minuteswest)); DEFINE(VDSO_TZ_DSTTIME, offsetof(struct vdso_data, tz_dsttime)); diff --git a/arch/arm64/kernel/cpu-reset.S b/arch/arm64/kernel/cpu-reset.S new file mode 100644 index 000000000000..65f42d257414 --- /dev/null +++ b/arch/arm64/kernel/cpu-reset.S @@ -0,0 +1,54 @@ +/* + * CPU reset routines + * + * Copyright (C) 2001 Deep Blue Solutions Ltd. + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2015 Huawei Futurewei Technologies. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include + +.text +.pushsection .idmap.text, "ax" + +/* + * __cpu_soft_restart(el2_switch, entry, arg0, arg1, arg2) - Helper for + * cpu_soft_restart. + * + * @el2_switch: Flag to indicate a swich to EL2 is needed. + * @entry: Location to jump to for soft reset. + * arg0: First argument passed to @entry. + * arg1: Second argument passed to @entry. + * arg2: Third argument passed to @entry. + * + * Put the CPU into the same state as it would be if it had been reset, and + * branch to what would be the reset vector. It must be executed with the + * flat identity mapping. + */ +ENTRY(__cpu_soft_restart) + /* Clear sctlr_el1 flags. */ + mrs x12, sctlr_el1 + ldr x13, =SCTLR_ELx_FLAGS + bic x12, x12, x13 + msr sctlr_el1, x12 + isb + + cbz x0, 1f // el2_switch? + mov x0, #HVC_SOFT_RESTART + hvc #0 // no return + +1: mov x18, x1 // entry + mov x0, x2 // arg0 + mov x1, x3 // arg1 + mov x2, x4 // arg2 + br x18 +ENDPROC(__cpu_soft_restart) + +.popsection diff --git a/arch/arm64/kernel/cpu-reset.h b/arch/arm64/kernel/cpu-reset.h new file mode 100644 index 000000000000..d4e9ecb264f0 --- /dev/null +++ b/arch/arm64/kernel/cpu-reset.h @@ -0,0 +1,34 @@ +/* + * CPU reset routines + * + * Copyright (C) 2015 Huawei Futurewei Technologies. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ARM64_CPU_RESET_H +#define _ARM64_CPU_RESET_H + +#include + +void __cpu_soft_restart(unsigned long el2_switch, unsigned long entry, + unsigned long arg0, unsigned long arg1, unsigned long arg2); + +static inline void __noreturn cpu_soft_restart(unsigned long el2_switch, + unsigned long entry, unsigned long arg0, unsigned long arg1, + unsigned long arg2) +{ + typeof(__cpu_soft_restart) *restart; + + el2_switch = el2_switch && !is_kernel_in_hyp_mode() && + is_hyp_mode_available(); + restart = (void *)virt_to_phys(__cpu_soft_restart); + + cpu_install_idmap(); + restart(el2_switch, entry, arg0, arg1, arg2); + unreachable(); +} + +#endif diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index d42789499f17..82b0fc2e637b 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -46,6 +46,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .desc = "ARM errata 826319, 827319, 824069", .capability = ARM64_WORKAROUND_CLEAN_CACHE, MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x02), + .enable = cpu_enable_cache_maint_trap, }, #endif #ifdef CONFIG_ARM64_ERRATUM_819472 @@ -54,6 +55,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .desc = "ARM errata 819472", .capability = ARM64_WORKAROUND_CLEAN_CACHE, MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x01), + .enable = cpu_enable_cache_maint_trap, }, #endif #ifdef CONFIG_ARM64_ERRATUM_832075 @@ -98,6 +100,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = { MIDR_RANGE(MIDR_THUNDERX, 0x00, (1 << MIDR_VARIANT_SHIFT) | 1), }, + { + /* Cavium ThunderX, T81 pass 1.0 */ + .desc = "Cavium erratum 27456", + .capability = ARM64_WORKAROUND_CAVIUM_27456, + MIDR_RANGE(MIDR_THUNDERX_81XX, 0x00, 0x00), + }, #endif { } @@ -127,3 +135,8 @@ void check_local_cpu_errata(void) { update_cpu_capabilities(arm64_errata, "enabling workaround for"); } + +void __init enable_errata_workarounds(void) +{ + enable_cpu_capabilities(arm64_errata); +} diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 811773d1c1d0..916d27ad79c1 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -913,8 +913,7 @@ void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, * Run through the enabled capabilities and enable() it on all active * CPUs */ -static void __init -enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) +void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) { for (; caps->matches; caps++) if (caps->enable && cpus_have_cap(caps->capability)) @@ -1036,6 +1035,7 @@ void __init setup_cpu_features(void) /* Set the CPU feature capabilies */ setup_feature_capabilities(); + enable_errata_workarounds(); setup_elf_hwcaps(arm64_elf_hwcaps); if (system_supports_32bit_el0()) diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c index e11857fce05f..75a0f8acef66 100644 --- a/arch/arm64/kernel/cpuidle.c +++ b/arch/arm64/kernel/cpuidle.c @@ -9,13 +9,16 @@ * published by the Free Software Foundation. */ +#include +#include +#include #include #include #include #include -int __init arm_cpuidle_init(unsigned int cpu) +int arm_cpuidle_init(unsigned int cpu) { int ret = -EOPNOTSUPP; @@ -39,3 +42,18 @@ int arm_cpuidle_suspend(int index) return cpu_ops[cpu]->cpu_suspend(index); } + +#ifdef CONFIG_ACPI + +#include + +int acpi_processor_ffh_lpi_probe(unsigned int cpu) +{ + return arm_cpuidle_init(cpu); +} + +int acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi) +{ + return CPU_PM_CPU_IDLE_ENTER(arm_cpuidle_suspend, lpi->index); +} +#endif diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index c173d329397f..ed1b84fe6925 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -183,6 +183,123 @@ const struct seq_operations cpuinfo_op = { .show = c_show }; + +static struct kobj_type cpuregs_kobj_type = { + .sysfs_ops = &kobj_sysfs_ops, +}; + +/* + * The ARM ARM uses the phrase "32-bit register" to describe a register + * whose upper 32 bits are RES0 (per C5.1.1, ARM DDI 0487A.i), however + * no statement is made as to whether the upper 32 bits will or will not + * be made use of in future, and between ARM DDI 0487A.c and ARM DDI + * 0487A.d CLIDR_EL1 was expanded from 32-bit to 64-bit. + * + * Thus, while both MIDR_EL1 and REVIDR_EL1 are described as 32-bit + * registers, we expose them both as 64 bit values to cater for possible + * future expansion without an ABI break. + */ +#define kobj_to_cpuinfo(kobj) container_of(kobj, struct cpuinfo_arm64, kobj) +#define CPUREGS_ATTR_RO(_name, _field) \ + static ssize_t _name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ + { \ + struct cpuinfo_arm64 *info = kobj_to_cpuinfo(kobj); \ + \ + if (info->reg_midr) \ + return sprintf(buf, "0x%016x\n", info->reg_##_field); \ + else \ + return 0; \ + } \ + static struct kobj_attribute cpuregs_attr_##_name = __ATTR_RO(_name) + +CPUREGS_ATTR_RO(midr_el1, midr); +CPUREGS_ATTR_RO(revidr_el1, revidr); + +static struct attribute *cpuregs_id_attrs[] = { + &cpuregs_attr_midr_el1.attr, + &cpuregs_attr_revidr_el1.attr, + NULL +}; + +static struct attribute_group cpuregs_attr_group = { + .attrs = cpuregs_id_attrs, + .name = "identification" +}; + +static int cpuid_add_regs(int cpu) +{ + int rc; + struct device *dev; + struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu); + + dev = get_cpu_device(cpu); + if (!dev) { + rc = -ENODEV; + goto out; + } + rc = kobject_add(&info->kobj, &dev->kobj, "regs"); + if (rc) + goto out; + rc = sysfs_create_group(&info->kobj, &cpuregs_attr_group); + if (rc) + kobject_del(&info->kobj); +out: + return rc; +} + +static int cpuid_remove_regs(int cpu) +{ + struct device *dev; + struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu); + + dev = get_cpu_device(cpu); + if (!dev) + return -ENODEV; + if (info->kobj.parent) { + sysfs_remove_group(&info->kobj, &cpuregs_attr_group); + kobject_del(&info->kobj); + } + + return 0; +} + +static int cpuid_callback(struct notifier_block *nb, + unsigned long action, void *hcpu) +{ + int rc = 0; + unsigned long cpu = (unsigned long)hcpu; + + switch (action & ~CPU_TASKS_FROZEN) { + case CPU_ONLINE: + rc = cpuid_add_regs(cpu); + break; + case CPU_DEAD: + rc = cpuid_remove_regs(cpu); + break; + } + + return notifier_from_errno(rc); +} + +static int __init cpuinfo_regs_init(void) +{ + int cpu; + + cpu_notifier_register_begin(); + + for_each_possible_cpu(cpu) { + struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu); + + kobject_init(&info->kobj, &cpuregs_kobj_type); + if (cpu_online(cpu)) + cpuid_add_regs(cpu); + } + __hotcpu_notifier(cpuid_callback, 0); + + cpu_notifier_register_done(); + return 0; +} static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) { unsigned int cpu = smp_processor_id(); @@ -212,6 +329,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_ctr = read_cpuid_cachetype(); info->reg_dczid = read_cpuid(DCZID_EL0); info->reg_midr = read_cpuid_id(); + info->reg_revidr = read_cpuid(REVIDR_EL1); info->reg_id_aa64dfr0 = read_cpuid(ID_AA64DFR0_EL1); info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1); @@ -264,3 +382,5 @@ void __init cpuinfo_store_boot_cpu(void) boot_cpu_data = *info; init_cpu_features(&boot_cpu_data); } + +device_initcall(cpuinfo_regs_init); diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 4fbf3c54275c..91fff48d0f57 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -48,6 +49,7 @@ static void mdscr_write(u32 mdscr) asm volatile("msr mdscr_el1, %0" :: "r" (mdscr)); local_dbg_restore(flags); } +NOKPROBE_SYMBOL(mdscr_write); static u32 mdscr_read(void) { @@ -55,6 +57,7 @@ static u32 mdscr_read(void) asm volatile("mrs %0, mdscr_el1" : "=r" (mdscr)); return mdscr; } +NOKPROBE_SYMBOL(mdscr_read); /* * Allow root to disable self-hosted debug from userspace. @@ -103,6 +106,7 @@ void enable_debug_monitors(enum dbg_active_el el) mdscr_write(mdscr); } } +NOKPROBE_SYMBOL(enable_debug_monitors); void disable_debug_monitors(enum dbg_active_el el) { @@ -123,6 +127,7 @@ void disable_debug_monitors(enum dbg_active_el el) mdscr_write(mdscr); } } +NOKPROBE_SYMBOL(disable_debug_monitors); /* * OS lock clearing. @@ -151,7 +156,6 @@ static int debug_monitors_init(void) /* Clear the OS lock. */ on_each_cpu(clear_os_lock, NULL, 1); isb(); - local_dbg_enable(); /* Register hotplug handler. */ __register_cpu_notifier(&os_lock_nb); @@ -166,22 +170,15 @@ postcore_initcall(debug_monitors_init); */ static void set_regs_spsr_ss(struct pt_regs *regs) { - unsigned long spsr; - - spsr = regs->pstate; - spsr &= ~DBG_SPSR_SS; - spsr |= DBG_SPSR_SS; - regs->pstate = spsr; + regs->pstate |= DBG_SPSR_SS; } +NOKPROBE_SYMBOL(set_regs_spsr_ss); static void clear_regs_spsr_ss(struct pt_regs *regs) { - unsigned long spsr; - - spsr = regs->pstate; - spsr &= ~DBG_SPSR_SS; - regs->pstate = spsr; + regs->pstate &= ~DBG_SPSR_SS; } +NOKPROBE_SYMBOL(clear_regs_spsr_ss); /* EL1 Single Step Handler hooks */ static LIST_HEAD(step_hook); @@ -225,6 +222,7 @@ static int call_step_hook(struct pt_regs *regs, unsigned int esr) return retval; } +NOKPROBE_SYMBOL(call_step_hook); static void send_user_sigtrap(int si_code) { @@ -266,6 +264,10 @@ static int single_step_handler(unsigned long addr, unsigned int esr, */ user_rewind_single_step(current); } else { +#ifdef CONFIG_KPROBES + if (kprobe_single_step_handler(regs, esr) == DBG_HOOK_HANDLED) + return 0; +#endif if (call_step_hook(regs, esr) == DBG_HOOK_HANDLED) return 0; @@ -279,6 +281,7 @@ static int single_step_handler(unsigned long addr, unsigned int esr, return 0; } +NOKPROBE_SYMBOL(single_step_handler); /* * Breakpoint handler is re-entrant as another breakpoint can @@ -316,19 +319,28 @@ static int call_break_hook(struct pt_regs *regs, unsigned int esr) return fn ? fn(regs, esr) : DBG_HOOK_ERROR; } +NOKPROBE_SYMBOL(call_break_hook); static int brk_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs) { if (user_mode(regs)) { send_user_sigtrap(TRAP_BRKPT); - } else if (call_break_hook(regs, esr) != DBG_HOOK_HANDLED) { - pr_warning("Unexpected kernel BRK exception at EL1\n"); + } +#ifdef CONFIG_KPROBES + else if ((esr & BRK64_ESR_MASK) == BRK64_ESR_KPROBES) { + if (kprobe_breakpoint_handler(regs, esr) != DBG_HOOK_HANDLED) + return -EFAULT; + } +#endif + else if (call_break_hook(regs, esr) != DBG_HOOK_HANDLED) { + pr_warn("Unexpected kernel BRK exception at EL1\n"); return -EFAULT; } return 0; } +NOKPROBE_SYMBOL(brk_handler); int aarch32_break_handler(struct pt_regs *regs) { @@ -365,6 +377,7 @@ int aarch32_break_handler(struct pt_regs *regs) send_user_sigtrap(TRAP_BRKPT); return 0; } +NOKPROBE_SYMBOL(aarch32_break_handler); static int __init debug_traps_init(void) { @@ -386,6 +399,7 @@ void user_rewind_single_step(struct task_struct *task) if (test_ti_thread_flag(task_thread_info(task), TIF_SINGLESTEP)) set_regs_spsr_ss(task_pt_regs(task)); } +NOKPROBE_SYMBOL(user_rewind_single_step); void user_fastforward_single_step(struct task_struct *task) { @@ -401,6 +415,7 @@ void kernel_enable_single_step(struct pt_regs *regs) mdscr_write(mdscr_read() | DBG_MDSCR_SS); enable_debug_monitors(DBG_ACTIVE_EL1); } +NOKPROBE_SYMBOL(kernel_enable_single_step); void kernel_disable_single_step(void) { @@ -408,12 +423,14 @@ void kernel_disable_single_step(void) mdscr_write(mdscr_read() & ~DBG_MDSCR_SS); disable_debug_monitors(DBG_ACTIVE_EL1); } +NOKPROBE_SYMBOL(kernel_disable_single_step); int kernel_active_single_step(void) { WARN_ON(!irqs_disabled()); return mdscr_read() & DBG_MDSCR_SS; } +NOKPROBE_SYMBOL(kernel_active_single_step); /* ptrace API */ void user_enable_single_step(struct task_struct *task) @@ -421,8 +438,10 @@ void user_enable_single_step(struct task_struct *task) set_ti_thread_flag(task_thread_info(task), TIF_SINGLESTEP); set_regs_spsr_ss(task_pt_regs(task)); } +NOKPROBE_SYMBOL(user_enable_single_step); void user_disable_single_step(struct task_struct *task) { clear_ti_thread_flag(task_thread_info(task), TIF_SINGLESTEP); } +NOKPROBE_SYMBOL(user_disable_single_step); diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 78f52488f9ff..ba9bee389fd5 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -62,13 +62,61 @@ struct screen_info screen_info __section(.data); int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) { pteval_t prot_val = create_mapping_protection(md); + bool allow_block_mappings = (md->type != EFI_RUNTIME_SERVICES_CODE && + md->type != EFI_RUNTIME_SERVICES_DATA); + + if (!PAGE_ALIGNED(md->phys_addr) || + !PAGE_ALIGNED(md->num_pages << EFI_PAGE_SHIFT)) { + /* + * If the end address of this region is not aligned to page + * size, the mapping is rounded up, and may end up sharing a + * page frame with the next UEFI memory region. If we create + * a block entry now, we may need to split it again when mapping + * the next region, and support for that is going to be removed + * from the MMU routines. So avoid block mappings altogether in + * that case. + */ + allow_block_mappings = false; + } create_pgd_mapping(mm, md->phys_addr, md->virt_addr, md->num_pages << EFI_PAGE_SHIFT, - __pgprot(prot_val | PTE_NG)); + __pgprot(prot_val | PTE_NG), allow_block_mappings); + return 0; +} + +static int __init set_permissions(pte_t *ptep, pgtable_t token, + unsigned long addr, void *data) +{ + efi_memory_desc_t *md = data; + pte_t pte = *ptep; + + if (md->attribute & EFI_MEMORY_RO) + pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); + if (md->attribute & EFI_MEMORY_XP) + pte = set_pte_bit(pte, __pgprot(PTE_PXN)); + set_pte(ptep, pte); return 0; } +int __init efi_set_mapping_permissions(struct mm_struct *mm, + efi_memory_desc_t *md) +{ + BUG_ON(md->type != EFI_RUNTIME_SERVICES_CODE && + md->type != EFI_RUNTIME_SERVICES_DATA); + + /* + * Calling apply_to_page_range() is only safe on regions that are + * guaranteed to be mapped down to pages. Since we are only called + * for regions that have been mapped using efi_create_mapping() above + * (and this is checked by the generic Memory Attributes table parsing + * routines), there is no need to check that again here. + */ + return apply_to_page_range(mm, md->virt_addr, + md->num_pages << EFI_PAGE_SHIFT, + set_permissions, md); +} + static int __init arm64_dmi_init(void) { /* diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 12e8d2bcb3f9..96e4a2b64cc1 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -97,7 +98,14 @@ mov x29, xzr // fp pointed to user-space .else add x21, sp, #S_FRAME_SIZE - .endif + get_thread_info tsk + /* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */ + ldr x20, [tsk, #TI_ADDR_LIMIT] + str x20, [sp, #S_ORIG_ADDR_LIMIT] + mov x20, #TASK_SIZE_64 + str x20, [tsk, #TI_ADDR_LIMIT] + ALTERNATIVE(nop, SET_PSTATE_UAO(0), ARM64_HAS_UAO, CONFIG_ARM64_UAO) + .endif /* \el == 0 */ mrs x22, elr_el1 mrs x23, spsr_el1 stp lr, x21, [sp, #S_LR] @@ -128,6 +136,14 @@ .endm .macro kernel_exit, el + .if \el != 0 + /* Restore the task's original addr_limit. */ + ldr x20, [sp, #S_ORIG_ADDR_LIMIT] + str x20, [tsk, #TI_ADDR_LIMIT] + + /* No need to restore UAO, it will be restored from SPSR_EL1 */ + .endif + ldp x21, x22, [sp, #S_PC] // load ELR, SPSR .if \el == 0 ct_user_enter @@ -242,6 +258,7 @@ tsk .req x28 // current thread_info /* * Exception vectors. */ + .pushsection ".entry.text", "ax" .align 11 ENTRY(vectors) @@ -406,7 +423,6 @@ el1_irq: bl trace_hardirqs_off #endif - get_thread_info tsk irq_handler #ifdef CONFIG_PREEMPT @@ -451,7 +467,7 @@ el0_sync: cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception b.eq el0_fpsimd_exc cmp x24, #ESR_ELx_EC_SYS64 // configurable trap - b.eq el0_undef + b.eq el0_sys cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception b.eq el0_sp_pc cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception @@ -532,7 +548,7 @@ el0_ia: enable_dbg_and_irq ct_user_exit mov x0, x26 - orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts + mov x1, x25 mov x2, sp bl do_mem_abort b ret_to_user @@ -579,6 +595,16 @@ el0_undef: mov x0, sp bl do_undefinstr b ret_to_user +el0_sys: + /* + * System instructions, for trapped cache maintenance instructions + */ + enable_dbg_and_irq + ct_user_exit + mov x0, x25 + mov x1, sp + bl do_sysinstr + b ret_to_user el0_dbg: /* * Debug exception handling @@ -774,6 +800,8 @@ __ni_sys_trace: bl do_ni_syscall b __sys_trace_return + .popsection // .entry.text + /* * Special system call wrappers. */ diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index f8df75d740f4..21ab5df9fa76 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -236,6 +237,11 @@ int swsusp_arch_suspend(void) unsigned long flags; struct sleep_stack_data state; + if (cpus_are_stuck_in_kernel()) { + pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); + return -EBUSY; + } + local_dbg_save(flags); if (__cpu_suspend_enter(&state)) { diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index ce21aa88263f..26a6bf77d272 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -127,6 +128,7 @@ static u64 read_wb_reg(int reg, int n) return val; } +NOKPROBE_SYMBOL(read_wb_reg); static void write_wb_reg(int reg, int n, u64 val) { @@ -140,6 +142,7 @@ static void write_wb_reg(int reg, int n, u64 val) } isb(); } +NOKPROBE_SYMBOL(write_wb_reg); /* * Convert a breakpoint privilege level to the corresponding exception @@ -157,6 +160,7 @@ static enum dbg_active_el debug_exception_level(int privilege) return -EINVAL; } } +NOKPROBE_SYMBOL(debug_exception_level); enum hw_breakpoint_ops { HW_BREAKPOINT_INSTALL, @@ -575,6 +579,7 @@ static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable) write_wb_reg(reg, i, ctrl); } } +NOKPROBE_SYMBOL(toggle_bp_registers); /* * Debug exception handlers. @@ -654,6 +659,7 @@ unlock: return 0; } +NOKPROBE_SYMBOL(breakpoint_handler); static int watchpoint_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs) @@ -756,6 +762,7 @@ unlock: return 0; } +NOKPROBE_SYMBOL(watchpoint_handler); /* * Handle single-step exception. @@ -813,6 +820,7 @@ int reinstall_suspended_bps(struct pt_regs *regs) return !handled_exception; } +NOKPROBE_SYMBOL(reinstall_suspended_bps); /* * Context-switcher for restoring suspended breakpoints. diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 8727f4490772..d3b5f75e652e 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -71,8 +71,16 @@ el1_sync: msr vbar_el2, x1 b 9f +2: cmp x0, #HVC_SOFT_RESTART + b.ne 3f + mov x0, x2 + mov x2, x4 + mov x4, x1 + mov x1, x3 + br x4 // no return + /* Someone called kvm_call_hyp() against the hyp-stub... */ -2: mov x0, #ARM_EXCEPTION_HYP_GONE +3: mov x0, #ARM_EXCEPTION_HYP_GONE 9: eret ENDPROC(el1_sync) diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c index 368c08290dd8..63f9432d05e8 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/kernel/insn.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #define AARCH64_INSN_SF_BIT BIT(31) @@ -162,6 +163,32 @@ static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn) aarch64_insn_is_nop(insn); } +bool __kprobes aarch64_insn_uses_literal(u32 insn) +{ + /* ldr/ldrsw (literal), prfm */ + + return aarch64_insn_is_ldr_lit(insn) || + aarch64_insn_is_ldrsw_lit(insn) || + aarch64_insn_is_adr_adrp(insn) || + aarch64_insn_is_prfm_lit(insn); +} + +bool __kprobes aarch64_insn_is_branch(u32 insn) +{ + /* b, bl, cb*, tb*, b.cond, br, blr */ + + return aarch64_insn_is_b(insn) || + aarch64_insn_is_bl(insn) || + aarch64_insn_is_cbz(insn) || + aarch64_insn_is_cbnz(insn) || + aarch64_insn_is_tbz(insn) || + aarch64_insn_is_tbnz(insn) || + aarch64_insn_is_ret(insn) || + aarch64_insn_is_br(insn) || + aarch64_insn_is_blr(insn) || + aarch64_insn_is_bcond(insn); +} + /* * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a * Section B2.6.5 "Concurrent modification and execution of instructions": @@ -1175,6 +1202,14 @@ u32 aarch64_set_branch_offset(u32 insn, s32 offset) BUG(); } +/* + * Extract the Op/CR data from a msr/mrs instruction. + */ +u32 aarch64_insn_extract_system_reg(u32 insn) +{ + return (insn & 0x1FFFE0) >> 5; +} + bool aarch32_insn_is_wide(u32 insn) { return insn >= 0xe800; @@ -1200,3 +1235,101 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn) { return insn & CRM_MASK; } + +static bool __kprobes __check_eq(unsigned long pstate) +{ + return (pstate & PSR_Z_BIT) != 0; +} + +static bool __kprobes __check_ne(unsigned long pstate) +{ + return (pstate & PSR_Z_BIT) == 0; +} + +static bool __kprobes __check_cs(unsigned long pstate) +{ + return (pstate & PSR_C_BIT) != 0; +} + +static bool __kprobes __check_cc(unsigned long pstate) +{ + return (pstate & PSR_C_BIT) == 0; +} + +static bool __kprobes __check_mi(unsigned long pstate) +{ + return (pstate & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_pl(unsigned long pstate) +{ + return (pstate & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_vs(unsigned long pstate) +{ + return (pstate & PSR_V_BIT) != 0; +} + +static bool __kprobes __check_vc(unsigned long pstate) +{ + return (pstate & PSR_V_BIT) == 0; +} + +static bool __kprobes __check_hi(unsigned long pstate) +{ + pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (pstate & PSR_C_BIT) != 0; +} + +static bool __kprobes __check_ls(unsigned long pstate) +{ + pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (pstate & PSR_C_BIT) == 0; +} + +static bool __kprobes __check_ge(unsigned long pstate) +{ + pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (pstate & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_lt(unsigned long pstate) +{ + pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (pstate & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_gt(unsigned long pstate) +{ + /*PSR_N_BIT ^= PSR_V_BIT */ + unsigned long temp = pstate ^ (pstate << 3); + + temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ + return (temp & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_le(unsigned long pstate) +{ + /*PSR_N_BIT ^= PSR_V_BIT */ + unsigned long temp = pstate ^ (pstate << 3); + + temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ + return (temp & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_al(unsigned long pstate) +{ + return true; +} + +/* + * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that + * it behaves identically to 0b1110 ("al"). + */ +pstate_check_t * const aarch32_opcode_cond_checks[16] = { + __check_eq, __check_ne, __check_cs, __check_cc, + __check_mi, __check_pl, __check_vs, __check_vc, + __check_hi, __check_ls, __check_ge, __check_lt, + __check_gt, __check_le, __check_al, __check_al +}; diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c index b5f063e5eff7..8c57f6496e56 100644 --- a/arch/arm64/kernel/kgdb.c +++ b/arch/arm64/kernel/kgdb.c @@ -22,6 +22,7 @@ #include #include #include +#include #include struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { @@ -230,6 +231,7 @@ static int kgdb_brk_fn(struct pt_regs *regs, unsigned int esr) kgdb_handle_exception(1, SIGTRAP, 0, regs); return 0; } +NOKPROBE_SYMBOL(kgdb_brk_fn) static int kgdb_compiled_brk_fn(struct pt_regs *regs, unsigned int esr) { @@ -238,12 +240,14 @@ static int kgdb_compiled_brk_fn(struct pt_regs *regs, unsigned int esr) return 0; } +NOKPROBE_SYMBOL(kgdb_compiled_brk_fn); static int kgdb_step_brk_fn(struct pt_regs *regs, unsigned int esr) { kgdb_handle_exception(1, SIGTRAP, 0, regs); return 0; } +NOKPROBE_SYMBOL(kgdb_step_brk_fn); static struct break_hook kgdb_brkpt_hook = { .esr_mask = 0xffffffff, diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c new file mode 100644 index 000000000000..bc96c8a7fc79 --- /dev/null +++ b/arch/arm64/kernel/machine_kexec.c @@ -0,0 +1,212 @@ +/* + * kexec for arm64 + * + * Copyright (C) Linaro. + * Copyright (C) Huawei Futurewei Technologies. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +#include +#include +#include + +#include "cpu-reset.h" + +/* Global variables for the arm64_relocate_new_kernel routine. */ +extern const unsigned char arm64_relocate_new_kernel[]; +extern const unsigned long arm64_relocate_new_kernel_size; + +static unsigned long kimage_start; + +/** + * kexec_image_info - For debugging output. + */ +#define kexec_image_info(_i) _kexec_image_info(__func__, __LINE__, _i) +static void _kexec_image_info(const char *func, int line, + const struct kimage *kimage) +{ + unsigned long i; + + pr_debug("%s:%d:\n", func, line); + pr_debug(" kexec kimage info:\n"); + pr_debug(" type: %d\n", kimage->type); + pr_debug(" start: %lx\n", kimage->start); + pr_debug(" head: %lx\n", kimage->head); + pr_debug(" nr_segments: %lu\n", kimage->nr_segments); + + for (i = 0; i < kimage->nr_segments; i++) { + pr_debug(" segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n", + i, + kimage->segment[i].mem, + kimage->segment[i].mem + kimage->segment[i].memsz, + kimage->segment[i].memsz, + kimage->segment[i].memsz / PAGE_SIZE); + } +} + +void machine_kexec_cleanup(struct kimage *kimage) +{ + /* Empty routine needed to avoid build errors. */ +} + +/** + * machine_kexec_prepare - Prepare for a kexec reboot. + * + * Called from the core kexec code when a kernel image is loaded. + * Forbid loading a kexec kernel if we have no way of hotplugging cpus or cpus + * are stuck in the kernel. This avoids a panic once we hit machine_kexec(). + */ +int machine_kexec_prepare(struct kimage *kimage) +{ + kimage_start = kimage->start; + + kexec_image_info(kimage); + + if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) { + pr_err("Can't kexec: CPUs are stuck in the kernel.\n"); + return -EBUSY; + } + + return 0; +} + +/** + * kexec_list_flush - Helper to flush the kimage list and source pages to PoC. + */ +static void kexec_list_flush(struct kimage *kimage) +{ + kimage_entry_t *entry; + + for (entry = &kimage->head; ; entry++) { + unsigned int flag; + void *addr; + + /* flush the list entries. */ + __flush_dcache_area(entry, sizeof(kimage_entry_t)); + + flag = *entry & IND_FLAGS; + if (flag == IND_DONE) + break; + + addr = phys_to_virt(*entry & PAGE_MASK); + + switch (flag) { + case IND_INDIRECTION: + /* Set entry point just before the new list page. */ + entry = (kimage_entry_t *)addr - 1; + break; + case IND_SOURCE: + /* flush the source pages. */ + __flush_dcache_area(addr, PAGE_SIZE); + break; + case IND_DESTINATION: + break; + default: + BUG(); + } + } +} + +/** + * kexec_segment_flush - Helper to flush the kimage segments to PoC. + */ +static void kexec_segment_flush(const struct kimage *kimage) +{ + unsigned long i; + + pr_debug("%s:\n", __func__); + + for (i = 0; i < kimage->nr_segments; i++) { + pr_debug(" segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n", + i, + kimage->segment[i].mem, + kimage->segment[i].mem + kimage->segment[i].memsz, + kimage->segment[i].memsz, + kimage->segment[i].memsz / PAGE_SIZE); + + __flush_dcache_area(phys_to_virt(kimage->segment[i].mem), + kimage->segment[i].memsz); + } +} + +/** + * machine_kexec - Do the kexec reboot. + * + * Called from the core kexec code for a sys_reboot with LINUX_REBOOT_CMD_KEXEC. + */ +void machine_kexec(struct kimage *kimage) +{ + phys_addr_t reboot_code_buffer_phys; + void *reboot_code_buffer; + + /* + * New cpus may have become stuck_in_kernel after we loaded the image. + */ + BUG_ON(cpus_are_stuck_in_kernel() || (num_online_cpus() > 1)); + + reboot_code_buffer_phys = page_to_phys(kimage->control_code_page); + reboot_code_buffer = phys_to_virt(reboot_code_buffer_phys); + + kexec_image_info(kimage); + + pr_debug("%s:%d: control_code_page: %p\n", __func__, __LINE__, + kimage->control_code_page); + pr_debug("%s:%d: reboot_code_buffer_phys: %pa\n", __func__, __LINE__, + &reboot_code_buffer_phys); + pr_debug("%s:%d: reboot_code_buffer: %p\n", __func__, __LINE__, + reboot_code_buffer); + pr_debug("%s:%d: relocate_new_kernel: %p\n", __func__, __LINE__, + arm64_relocate_new_kernel); + pr_debug("%s:%d: relocate_new_kernel_size: 0x%lx(%lu) bytes\n", + __func__, __LINE__, arm64_relocate_new_kernel_size, + arm64_relocate_new_kernel_size); + + /* + * Copy arm64_relocate_new_kernel to the reboot_code_buffer for use + * after the kernel is shut down. + */ + memcpy(reboot_code_buffer, arm64_relocate_new_kernel, + arm64_relocate_new_kernel_size); + + /* Flush the reboot_code_buffer in preparation for its execution. */ + __flush_dcache_area(reboot_code_buffer, arm64_relocate_new_kernel_size); + flush_icache_range((uintptr_t)reboot_code_buffer, + arm64_relocate_new_kernel_size); + + /* Flush the kimage list and its buffers. */ + kexec_list_flush(kimage); + + /* Flush the new image if already in place. */ + if (kimage->head & IND_DONE) + kexec_segment_flush(kimage); + + pr_info("Bye!\n"); + + /* Disable all DAIF exceptions. */ + asm volatile ("msr daifset, #0xf" : : : "memory"); + + /* + * cpu_soft_restart will shutdown the MMU, disable data caches, then + * transfer control to the reboot_code_buffer which contains a copy of + * the arm64_relocate_new_kernel routine. arm64_relocate_new_kernel + * uses physical addressing to relocate the new image to its final + * position and transfers control to the image entry point when the + * relocation is complete. + */ + + cpu_soft_restart(1, reboot_code_buffer_phys, kimage->head, + kimage_start, 0); + + BUG(); /* Should never get here. */ +} + +void machine_crash_shutdown(struct pt_regs *regs) +{ + /* Empty routine needed to avoid build errors. */ +} diff --git a/arch/arm64/kernel/probes/Makefile b/arch/arm64/kernel/probes/Makefile new file mode 100644 index 000000000000..ce06312e3d34 --- /dev/null +++ b/arch/arm64/kernel/probes/Makefile @@ -0,0 +1,3 @@ +obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o \ + kprobes_trampoline.o \ + simulate-insn.o diff --git a/arch/arm64/kernel/probes/decode-insn.c b/arch/arm64/kernel/probes/decode-insn.c new file mode 100644 index 000000000000..37e47a9d617e --- /dev/null +++ b/arch/arm64/kernel/probes/decode-insn.c @@ -0,0 +1,174 @@ +/* + * arch/arm64/kernel/probes/decode-insn.c + * + * Copyright (C) 2013 Linaro Limited. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include + +#include "decode-insn.h" +#include "simulate-insn.h" + +static bool __kprobes aarch64_insn_is_steppable(u32 insn) +{ + /* + * Branch instructions will write a new value into the PC which is + * likely to be relative to the XOL address and therefore invalid. + * Deliberate generation of an exception during stepping is also not + * currently safe. Lastly, MSR instructions can do any number of nasty + * things we can't handle during single-stepping. + */ + if (aarch64_get_insn_class(insn) == AARCH64_INSN_CLS_BR_SYS) { + if (aarch64_insn_is_branch(insn) || + aarch64_insn_is_msr_imm(insn) || + aarch64_insn_is_msr_reg(insn) || + aarch64_insn_is_exception(insn) || + aarch64_insn_is_eret(insn)) + return false; + + /* + * The MRS instruction may not return a correct value when + * executing in the single-stepping environment. We do make one + * exception, for reading the DAIF bits. + */ + if (aarch64_insn_is_mrs(insn)) + return aarch64_insn_extract_system_reg(insn) + != AARCH64_INSN_SPCLREG_DAIF; + + /* + * The HINT instruction is is problematic when single-stepping, + * except for the NOP case. + */ + if (aarch64_insn_is_hint(insn)) + return aarch64_insn_is_nop(insn); + + return true; + } + + /* + * Instructions which load PC relative literals are not going to work + * when executed from an XOL slot. Instructions doing an exclusive + * load/store are not going to complete successfully when single-step + * exception handling happens in the middle of the sequence. + */ + if (aarch64_insn_uses_literal(insn) || + aarch64_insn_is_exclusive(insn)) + return false; + + return true; +} + +/* Return: + * INSN_REJECTED If instruction is one not allowed to kprobe, + * INSN_GOOD If instruction is supported and uses instruction slot, + * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. + */ +static enum kprobe_insn __kprobes +arm_probe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* + * Instructions reading or modifying the PC won't work from the XOL + * slot. + */ + if (aarch64_insn_is_steppable(insn)) + return INSN_GOOD; + + if (aarch64_insn_is_bcond(insn)) { + asi->handler = simulate_b_cond; + } else if (aarch64_insn_is_cbz(insn) || + aarch64_insn_is_cbnz(insn)) { + asi->handler = simulate_cbz_cbnz; + } else if (aarch64_insn_is_tbz(insn) || + aarch64_insn_is_tbnz(insn)) { + asi->handler = simulate_tbz_tbnz; + } else if (aarch64_insn_is_adr_adrp(insn)) { + asi->handler = simulate_adr_adrp; + } else if (aarch64_insn_is_b(insn) || + aarch64_insn_is_bl(insn)) { + asi->handler = simulate_b_bl; + } else if (aarch64_insn_is_br(insn) || + aarch64_insn_is_blr(insn) || + aarch64_insn_is_ret(insn)) { + asi->handler = simulate_br_blr_ret; + } else if (aarch64_insn_is_ldr_lit(insn)) { + asi->handler = simulate_ldr_literal; + } else if (aarch64_insn_is_ldrsw_lit(insn)) { + asi->handler = simulate_ldrsw_literal; + } else { + /* + * Instruction cannot be stepped out-of-line and we don't + * (yet) simulate it. + */ + return INSN_REJECTED; + } + + return INSN_GOOD_NO_SLOT; +} + +static bool __kprobes +is_probed_address_atomic(kprobe_opcode_t *scan_start, kprobe_opcode_t *scan_end) +{ + while (scan_start > scan_end) { + /* + * atomic region starts from exclusive load and ends with + * exclusive store. + */ + if (aarch64_insn_is_store_ex(le32_to_cpu(*scan_start))) + return false; + else if (aarch64_insn_is_load_ex(le32_to_cpu(*scan_start))) + return true; + scan_start--; + } + + return false; +} + +enum kprobe_insn __kprobes +arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi) +{ + enum kprobe_insn decoded; + kprobe_opcode_t insn = le32_to_cpu(*addr); + kprobe_opcode_t *scan_start = addr - 1; + kprobe_opcode_t *scan_end = addr - MAX_ATOMIC_CONTEXT_SIZE; +#if defined(CONFIG_MODULES) && defined(MODULES_VADDR) + struct module *mod; +#endif + + if (addr >= (kprobe_opcode_t *)_text && + scan_end < (kprobe_opcode_t *)_text) + scan_end = (kprobe_opcode_t *)_text; +#if defined(CONFIG_MODULES) && defined(MODULES_VADDR) + else { + preempt_disable(); + mod = __module_address((unsigned long)addr); + if (mod && within_module_init((unsigned long)addr, mod) && + !within_module_init((unsigned long)scan_end, mod)) + scan_end = (kprobe_opcode_t *)mod->init_layout.base; + else if (mod && within_module_core((unsigned long)addr, mod) && + !within_module_core((unsigned long)scan_end, mod)) + scan_end = (kprobe_opcode_t *)mod->core_layout.base; + preempt_enable(); + } +#endif + decoded = arm_probe_decode_insn(insn, asi); + + if (decoded == INSN_REJECTED || + is_probed_address_atomic(scan_start, scan_end)) + return INSN_REJECTED; + + return decoded; +} diff --git a/arch/arm64/kernel/probes/decode-insn.h b/arch/arm64/kernel/probes/decode-insn.h new file mode 100644 index 000000000000..d438289646a6 --- /dev/null +++ b/arch/arm64/kernel/probes/decode-insn.h @@ -0,0 +1,35 @@ +/* + * arch/arm64/kernel/probes/decode-insn.h + * + * Copyright (C) 2013 Linaro Limited. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#ifndef _ARM_KERNEL_KPROBES_ARM64_H +#define _ARM_KERNEL_KPROBES_ARM64_H + +/* + * ARM strongly recommends a limit of 128 bytes between LoadExcl and + * StoreExcl instructions in a single thread of execution. So keep the + * max atomic context size as 32. + */ +#define MAX_ATOMIC_CONTEXT_SIZE (128 / sizeof(kprobe_opcode_t)) + +enum kprobe_insn { + INSN_REJECTED, + INSN_GOOD_NO_SLOT, + INSN_GOOD, +}; + +enum kprobe_insn __kprobes +arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi); + +#endif /* _ARM_KERNEL_KPROBES_ARM64_H */ diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c new file mode 100644 index 000000000000..bf9768588288 --- /dev/null +++ b/arch/arm64/kernel/probes/kprobes.c @@ -0,0 +1,686 @@ +/* + * arch/arm64/kernel/probes/kprobes.c + * + * Kprobes support for ARM64 + * + * Copyright (C) 2013 Linaro Limited. + * Author: Sandeepa Prabhu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "decode-insn.h" + +DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +static void __kprobes +post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *); + +static inline unsigned long min_stack_size(unsigned long addr) +{ + unsigned long size; + + if (on_irq_stack(addr, raw_smp_processor_id())) + size = IRQ_STACK_PTR(raw_smp_processor_id()) - addr; + else + size = (unsigned long)current_thread_info() + THREAD_START_SP - addr; + + return min(size, FIELD_SIZEOF(struct kprobe_ctlblk, jprobes_stack)); +} + +static void __kprobes arch_prepare_ss_slot(struct kprobe *p) +{ + /* prepare insn slot */ + p->ainsn.insn[0] = cpu_to_le32(p->opcode); + + flush_icache_range((uintptr_t) (p->ainsn.insn), + (uintptr_t) (p->ainsn.insn) + + MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); + + /* + * Needs restoring of return address after stepping xol. + */ + p->ainsn.restore = (unsigned long) p->addr + + sizeof(kprobe_opcode_t); +} + +static void __kprobes arch_prepare_simulate(struct kprobe *p) +{ + /* This instructions is not executed xol. No need to adjust the PC */ + p->ainsn.restore = 0; +} + +static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (p->ainsn.handler) + p->ainsn.handler((u32)p->opcode, (long)p->addr, regs); + + /* single step simulated, now go for post processing */ + post_kprobe_handler(kcb, regs); +} + +int __kprobes arch_prepare_kprobe(struct kprobe *p) +{ + unsigned long probe_addr = (unsigned long)p->addr; + extern char __start_rodata[]; + extern char __end_rodata[]; + + if (probe_addr & 0x3) + return -EINVAL; + + /* copy instruction */ + p->opcode = le32_to_cpu(*p->addr); + + if (in_exception_text(probe_addr)) + return -EINVAL; + if (probe_addr >= (unsigned long) __start_rodata && + probe_addr <= (unsigned long) __end_rodata) + return -EINVAL; + + /* decode instruction */ + switch (arm_kprobe_decode_insn(p->addr, &p->ainsn)) { + case INSN_REJECTED: /* insn not supported */ + return -EINVAL; + + case INSN_GOOD_NO_SLOT: /* insn need simulation */ + p->ainsn.insn = NULL; + break; + + case INSN_GOOD: /* instruction uses slot */ + p->ainsn.insn = get_insn_slot(); + if (!p->ainsn.insn) + return -ENOMEM; + break; + }; + + /* prepare the instruction */ + if (p->ainsn.insn) + arch_prepare_ss_slot(p); + else + arch_prepare_simulate(p); + + return 0; +} + +static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode) +{ + void *addrs[1]; + u32 insns[1]; + + addrs[0] = (void *)addr; + insns[0] = (u32)opcode; + + return aarch64_insn_patch_text(addrs, insns, 1); +} + +/* arm kprobe: install breakpoint in text */ +void __kprobes arch_arm_kprobe(struct kprobe *p) +{ + patch_text(p->addr, BRK64_OPCODE_KPROBES); +} + +/* disarm kprobe: remove breakpoint from text */ +void __kprobes arch_disarm_kprobe(struct kprobe *p) +{ + patch_text(p->addr, p->opcode); +} + +void __kprobes arch_remove_kprobe(struct kprobe *p) +{ + if (p->ainsn.insn) { + free_insn_slot(p->ainsn.insn, 0); + p->ainsn.insn = NULL; + } +} + +static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; +} + +static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); + kcb->kprobe_status = kcb->prev_kprobe.status; +} + +static void __kprobes set_current_kprobe(struct kprobe *p) +{ + __this_cpu_write(current_kprobe, p); +} + +/* + * The D-flag (Debug mask) is set (masked) upon debug exception entry. + * Kprobes needs to clear (unmask) D-flag -ONLY- in case of recursive + * probe i.e. when probe hit from kprobe handler context upon + * executing the pre/post handlers. In this case we return with + * D-flag clear so that single-stepping can be carried-out. + * + * Leave D-flag set in all other cases. + */ +static void __kprobes +spsr_set_debug_flag(struct pt_regs *regs, int mask) +{ + unsigned long spsr = regs->pstate; + + if (mask) + spsr |= PSR_D_BIT; + else + spsr &= ~PSR_D_BIT; + + regs->pstate = spsr; +} + +/* + * Interrupts need to be disabled before single-step mode is set, and not + * reenabled until after single-step mode ends. + * Without disabling interrupt on local CPU, there is a chance of + * interrupt occurrence in the period of exception return and start of + * out-of-line single-step, that result in wrongly single stepping + * into the interrupt handler. + */ +static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + kcb->saved_irqflag = regs->pstate; + regs->pstate |= PSR_I_BIT; +} + +static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + if (kcb->saved_irqflag & PSR_I_BIT) + regs->pstate |= PSR_I_BIT; + else + regs->pstate &= ~PSR_I_BIT; +} + +static void __kprobes +set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr) +{ + kcb->ss_ctx.ss_pending = true; + kcb->ss_ctx.match_addr = addr + sizeof(kprobe_opcode_t); +} + +static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb) +{ + kcb->ss_ctx.ss_pending = false; + kcb->ss_ctx.match_addr = 0; +} + +static void __kprobes setup_singlestep(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb, int reenter) +{ + unsigned long slot; + + if (reenter) { + save_previous_kprobe(kcb); + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_REENTER; + } else { + kcb->kprobe_status = KPROBE_HIT_SS; + } + + + if (p->ainsn.insn) { + /* prepare for single stepping */ + slot = (unsigned long)p->ainsn.insn; + + set_ss_context(kcb, slot); /* mark pending ss */ + + if (kcb->kprobe_status == KPROBE_REENTER) + spsr_set_debug_flag(regs, 0); + else + WARN_ON(regs->pstate & PSR_D_BIT); + + /* IRQs and single stepping do not mix well. */ + kprobes_save_local_irqflag(kcb, regs); + kernel_enable_single_step(regs); + instruction_pointer_set(regs, slot); + } else { + /* insn simulation */ + arch_simulate_insn(p, regs); + } +} + +static int __kprobes reenter_kprobe(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + switch (kcb->kprobe_status) { + case KPROBE_HIT_SSDONE: + case KPROBE_HIT_ACTIVE: + kprobes_inc_nmissed_count(p); + setup_singlestep(p, regs, kcb, 1); + break; + case KPROBE_HIT_SS: + case KPROBE_REENTER: + pr_warn("Unrecoverable kprobe detected at %p.\n", p->addr); + dump_kprobe(p); + BUG(); + break; + default: + WARN_ON(1); + return 0; + } + + return 1; +} + +static void __kprobes +post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs) +{ + struct kprobe *cur = kprobe_running(); + + if (!cur) + return; + + /* return addr restore if non-branching insn */ + if (cur->ainsn.restore != 0) + instruction_pointer_set(regs, cur->ainsn.restore); + + /* restore back original saved kprobe variables and continue */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + return; + } + /* call post handler */ + kcb->kprobe_status = KPROBE_HIT_SSDONE; + if (cur->post_handler) { + /* post_handler can hit breakpoint and single step + * again, so we enable D-flag for recursive exception. + */ + cur->post_handler(cur, regs, 0); + } + + reset_current_kprobe(); +} + +int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + switch (kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_REENTER: + /* + * We are here because the instruction being single + * stepped caused a page fault. We reset the current + * kprobe and the ip points back to the probe address + * and allow the page fault handler to continue as a + * normal page fault. + */ + instruction_pointer_set(regs, (unsigned long) cur->addr); + if (!instruction_pointer(regs)) + BUG(); + + kernel_disable_single_step(); + if (kcb->kprobe_status == KPROBE_REENTER) + spsr_set_debug_flag(regs, 1); + + if (kcb->kprobe_status == KPROBE_REENTER) + restore_previous_kprobe(kcb); + else + reset_current_kprobe(); + + break; + case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SSDONE: + /* + * We increment the nmissed count for accounting, + * we can also use npre/npostfault count for accounting + * these specific fault cases. + */ + kprobes_inc_nmissed_count(cur); + + /* + * We come here because instructions in the pre/post + * handler caused the page_fault, this could happen + * if handler tries to access user space by + * copy_from_user(), get_user() etc. Let the + * user-specified handler try to fix it first. + */ + if (cur->fault_handler && cur->fault_handler(cur, regs, fsr)) + return 1; + + /* + * In case the user-specified fault handler returned + * zero, try to fix up. + */ + if (fixup_exception(regs)) + return 1; + } + return 0; +} + +int __kprobes kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} + +static void __kprobes kprobe_handler(struct pt_regs *regs) +{ + struct kprobe *p, *cur_kprobe; + struct kprobe_ctlblk *kcb; + unsigned long addr = instruction_pointer(regs); + + kcb = get_kprobe_ctlblk(); + cur_kprobe = kprobe_running(); + + p = get_kprobe((kprobe_opcode_t *) addr); + + if (p) { + if (cur_kprobe) { + if (reenter_kprobe(p, regs, kcb)) + return; + } else { + /* Probe hit */ + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + + /* + * If we have no pre-handler or it returned 0, we + * continue with normal processing. If we have a + * pre-handler and it returned non-zero, it prepped + * for calling the break_handler below on re-entry, + * so get out doing nothing more here. + * + * pre_handler can hit a breakpoint and can step thru + * before return, keep PSTATE D-flag enabled until + * pre_handler return back. + */ + if (!p->pre_handler || !p->pre_handler(p, regs)) { + setup_singlestep(p, regs, kcb, 0); + return; + } + } + } else if ((le32_to_cpu(*(kprobe_opcode_t *) addr) == + BRK64_OPCODE_KPROBES) && cur_kprobe) { + /* We probably hit a jprobe. Call its break handler. */ + if (cur_kprobe->break_handler && + cur_kprobe->break_handler(cur_kprobe, regs)) { + setup_singlestep(cur_kprobe, regs, kcb, 0); + return; + } + } + /* + * The breakpoint instruction was removed right + * after we hit it. Another cpu has removed + * either a probepoint or a debugger breakpoint + * at this address. In either case, no further + * handling of this interrupt is appropriate. + * Return back to original instruction, and continue. + */ +} + +static int __kprobes +kprobe_ss_hit(struct kprobe_ctlblk *kcb, unsigned long addr) +{ + if ((kcb->ss_ctx.ss_pending) + && (kcb->ss_ctx.match_addr == addr)) { + clear_ss_context(kcb); /* clear pending ss */ + return DBG_HOOK_HANDLED; + } + /* not ours, kprobes should ignore it */ + return DBG_HOOK_ERROR; +} + +int __kprobes +kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + int retval; + + /* return error if this is not our step */ + retval = kprobe_ss_hit(kcb, instruction_pointer(regs)); + + if (retval == DBG_HOOK_HANDLED) { + kprobes_restore_local_irqflag(kcb, regs); + kernel_disable_single_step(); + + if (kcb->kprobe_status == KPROBE_REENTER) + spsr_set_debug_flag(regs, 1); + + post_kprobe_handler(kcb, regs); + } + + return retval; +} + +int __kprobes +kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr) +{ + kprobe_handler(regs); + return DBG_HOOK_HANDLED; +} + +int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + struct jprobe *jp = container_of(p, struct jprobe, kp); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + long stack_ptr = kernel_stack_pointer(regs); + + kcb->jprobe_saved_regs = *regs; + /* + * As Linus pointed out, gcc assumes that the callee + * owns the argument space and could overwrite it, e.g. + * tailcall optimization. So, to be absolutely safe + * we also save and restore enough stack bytes to cover + * the argument area. + */ + kasan_disable_current(); + memcpy(kcb->jprobes_stack, (void *)stack_ptr, + min_stack_size(stack_ptr)); + kasan_enable_current(); + + instruction_pointer_set(regs, (unsigned long) jp->entry); + preempt_disable(); + pause_graph_tracing(); + return 1; +} + +void __kprobes jprobe_return(void) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + /* + * Jprobe handler return by entering break exception, + * encoded same as kprobe, but with following conditions + * -a special PC to identify it from the other kprobes. + * -restore stack addr to original saved pt_regs + */ + asm volatile(" mov sp, %0 \n" + "jprobe_return_break: brk %1 \n" + : + : "r" (kcb->jprobe_saved_regs.sp), + "I" (BRK64_ESR_KPROBES) + : "memory"); + + unreachable(); +} + +int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + long stack_addr = kcb->jprobe_saved_regs.sp; + long orig_sp = kernel_stack_pointer(regs); + struct jprobe *jp = container_of(p, struct jprobe, kp); + extern const char jprobe_return_break[]; + + if (instruction_pointer(regs) != (u64) jprobe_return_break) + return 0; + + if (orig_sp != stack_addr) { + struct pt_regs *saved_regs = + (struct pt_regs *)kcb->jprobe_saved_regs.sp; + pr_err("current sp %lx does not match saved sp %lx\n", + orig_sp, stack_addr); + pr_err("Saved registers for jprobe %p\n", jp); + show_regs(saved_regs); + pr_err("Current registers\n"); + show_regs(regs); + BUG(); + } + unpause_graph_tracing(); + *regs = kcb->jprobe_saved_regs; + kasan_disable_current(); + memcpy((void *)stack_addr, kcb->jprobes_stack, + min_stack_size(stack_addr)); + kasan_enable_current(); + preempt_enable_no_resched(); + return 1; +} + +bool arch_within_kprobe_blacklist(unsigned long addr) +{ + extern char __idmap_text_start[], __idmap_text_end[]; + extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[]; + + if ((addr >= (unsigned long)__kprobes_text_start && + addr < (unsigned long)__kprobes_text_end) || + (addr >= (unsigned long)__entry_text_start && + addr < (unsigned long)__entry_text_end) || + (addr >= (unsigned long)__idmap_text_start && + addr < (unsigned long)__idmap_text_end) || + !!search_exception_tables(addr)) + return true; + + if (!is_kernel_in_hyp_mode()) { + if ((addr >= (unsigned long)__hyp_text_start && + addr < (unsigned long)__hyp_text_end) || + (addr >= (unsigned long)__hyp_idmap_text_start && + addr < (unsigned long)__hyp_idmap_text_end)) + return true; + } + + return false; +} + +void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) +{ + struct kretprobe_instance *ri = NULL; + struct hlist_head *head, empty_rp; + struct hlist_node *tmp; + unsigned long flags, orig_ret_address = 0; + unsigned long trampoline_address = + (unsigned long)&kretprobe_trampoline; + kprobe_opcode_t *correct_ret_addr = NULL; + + INIT_HLIST_HEAD(&empty_rp); + kretprobe_hash_lock(current, &head, &flags); + + /* + * It is possible to have multiple instances associated with a given + * task either because multiple functions in the call path have + * return probes installed on them, and/or more than one + * return probe was registered for a target function. + * + * We can handle this because: + * - instances are always pushed into the head of the list + * - when multiple return probes are registered for the same + * function, the (chronologically) first instance's ret_addr + * will be the real return address, and all the rest will + * point to kretprobe_trampoline. + */ + hlist_for_each_entry_safe(ri, tmp, head, hlist) { + if (ri->task != current) + /* another task is sharing our hash bucket */ + continue; + + orig_ret_address = (unsigned long)ri->ret_addr; + + if (orig_ret_address != trampoline_address) + /* + * This is the real return address. Any other + * instances associated with this task are for + * other calls deeper on the call stack + */ + break; + } + + kretprobe_assert(ri, orig_ret_address, trampoline_address); + + correct_ret_addr = ri->ret_addr; + hlist_for_each_entry_safe(ri, tmp, head, hlist) { + if (ri->task != current) + /* another task is sharing our hash bucket */ + continue; + + orig_ret_address = (unsigned long)ri->ret_addr; + if (ri->rp && ri->rp->handler) { + __this_cpu_write(current_kprobe, &ri->rp->kp); + get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE; + ri->ret_addr = correct_ret_addr; + ri->rp->handler(ri, regs); + __this_cpu_write(current_kprobe, NULL); + } + + recycle_rp_inst(ri, &empty_rp); + + if (orig_ret_address != trampoline_address) + /* + * This is the real return address. Any other + * instances associated with this task are for + * other calls deeper on the call stack + */ + break; + } + + kretprobe_hash_unlock(current, &flags); + + hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { + hlist_del(&ri->hlist); + kfree(ri); + } + return (void *)orig_ret_address; +} + +void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *)regs->regs[30]; + + /* replace return addr (x30) with trampoline */ + regs->regs[30] = (long)&kretprobe_trampoline; +} + +int __kprobes arch_trampoline_kprobe(struct kprobe *p) +{ + return 0; +} + +int __init arch_init_kprobes(void) +{ + return 0; +} diff --git a/arch/arm64/kernel/probes/kprobes_trampoline.S b/arch/arm64/kernel/probes/kprobes_trampoline.S new file mode 100644 index 000000000000..5d6e7f14638c --- /dev/null +++ b/arch/arm64/kernel/probes/kprobes_trampoline.S @@ -0,0 +1,81 @@ +/* + * trampoline entry and return code for kretprobes. + */ + +#include +#include +#include + + .text + + .macro save_all_base_regs + stp x0, x1, [sp, #S_X0] + stp x2, x3, [sp, #S_X2] + stp x4, x5, [sp, #S_X4] + stp x6, x7, [sp, #S_X6] + stp x8, x9, [sp, #S_X8] + stp x10, x11, [sp, #S_X10] + stp x12, x13, [sp, #S_X12] + stp x14, x15, [sp, #S_X14] + stp x16, x17, [sp, #S_X16] + stp x18, x19, [sp, #S_X18] + stp x20, x21, [sp, #S_X20] + stp x22, x23, [sp, #S_X22] + stp x24, x25, [sp, #S_X24] + stp x26, x27, [sp, #S_X26] + stp x28, x29, [sp, #S_X28] + add x0, sp, #S_FRAME_SIZE + stp lr, x0, [sp, #S_LR] + /* + * Construct a useful saved PSTATE + */ + mrs x0, nzcv + mrs x1, daif + orr x0, x0, x1 + mrs x1, CurrentEL + orr x0, x0, x1 + mrs x1, SPSel + orr x0, x0, x1 + stp xzr, x0, [sp, #S_PC] + .endm + + .macro restore_all_base_regs + ldr x0, [sp, #S_PSTATE] + and x0, x0, #(PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT) + msr nzcv, x0 + ldp x0, x1, [sp, #S_X0] + ldp x2, x3, [sp, #S_X2] + ldp x4, x5, [sp, #S_X4] + ldp x6, x7, [sp, #S_X6] + ldp x8, x9, [sp, #S_X8] + ldp x10, x11, [sp, #S_X10] + ldp x12, x13, [sp, #S_X12] + ldp x14, x15, [sp, #S_X14] + ldp x16, x17, [sp, #S_X16] + ldp x18, x19, [sp, #S_X18] + ldp x20, x21, [sp, #S_X20] + ldp x22, x23, [sp, #S_X22] + ldp x24, x25, [sp, #S_X24] + ldp x26, x27, [sp, #S_X26] + ldp x28, x29, [sp, #S_X28] + .endm + +ENTRY(kretprobe_trampoline) + sub sp, sp, #S_FRAME_SIZE + + save_all_base_regs + + mov x0, sp + bl trampoline_probe_handler + /* + * Replace trampoline address in lr with actual orig_ret_addr return + * address. + */ + mov lr, x0 + + restore_all_base_regs + + add sp, sp, #S_FRAME_SIZE + ret + +ENDPROC(kretprobe_trampoline) diff --git a/arch/arm64/kernel/probes/simulate-insn.c b/arch/arm64/kernel/probes/simulate-insn.c new file mode 100644 index 000000000000..8977ce9d009d --- /dev/null +++ b/arch/arm64/kernel/probes/simulate-insn.c @@ -0,0 +1,217 @@ +/* + * arch/arm64/kernel/probes/simulate-insn.c + * + * Copyright (C) 2013 Linaro Limited. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include + +#include "simulate-insn.h" + +#define sign_extend(x, signbit) \ + ((x) | (0 - ((x) & (1 << (signbit))))) + +#define bbl_displacement(insn) \ + sign_extend(((insn) & 0x3ffffff) << 2, 27) + +#define bcond_displacement(insn) \ + sign_extend(((insn >> 5) & 0x7ffff) << 2, 20) + +#define cbz_displacement(insn) \ + sign_extend(((insn >> 5) & 0x7ffff) << 2, 20) + +#define tbz_displacement(insn) \ + sign_extend(((insn >> 5) & 0x3fff) << 2, 15) + +#define ldr_displacement(insn) \ + sign_extend(((insn >> 5) & 0x7ffff) << 2, 20) + +static inline void set_x_reg(struct pt_regs *regs, int reg, u64 val) +{ + if (reg < 31) + regs->regs[reg] = val; +} + +static inline void set_w_reg(struct pt_regs *regs, int reg, u64 val) +{ + if (reg < 31) + regs->regs[reg] = lower_32_bits(val); +} + +static inline u64 get_x_reg(struct pt_regs *regs, int reg) +{ + if (reg < 31) + return regs->regs[reg]; + else + return 0; +} + +static inline u32 get_w_reg(struct pt_regs *regs, int reg) +{ + if (reg < 31) + return lower_32_bits(regs->regs[reg]); + else + return 0; +} + +static bool __kprobes check_cbz(u32 opcode, struct pt_regs *regs) +{ + int xn = opcode & 0x1f; + + return (opcode & (1 << 31)) ? + (get_x_reg(regs, xn) == 0) : (get_w_reg(regs, xn) == 0); +} + +static bool __kprobes check_cbnz(u32 opcode, struct pt_regs *regs) +{ + int xn = opcode & 0x1f; + + return (opcode & (1 << 31)) ? + (get_x_reg(regs, xn) != 0) : (get_w_reg(regs, xn) != 0); +} + +static bool __kprobes check_tbz(u32 opcode, struct pt_regs *regs) +{ + int xn = opcode & 0x1f; + int bit_pos = ((opcode & (1 << 31)) >> 26) | ((opcode >> 19) & 0x1f); + + return ((get_x_reg(regs, xn) >> bit_pos) & 0x1) == 0; +} + +static bool __kprobes check_tbnz(u32 opcode, struct pt_regs *regs) +{ + int xn = opcode & 0x1f; + int bit_pos = ((opcode & (1 << 31)) >> 26) | ((opcode >> 19) & 0x1f); + + return ((get_x_reg(regs, xn) >> bit_pos) & 0x1) != 0; +} + +/* + * instruction simulation functions + */ +void __kprobes +simulate_adr_adrp(u32 opcode, long addr, struct pt_regs *regs) +{ + long imm, xn, val; + + xn = opcode & 0x1f; + imm = ((opcode >> 3) & 0x1ffffc) | ((opcode >> 29) & 0x3); + imm = sign_extend(imm, 20); + if (opcode & 0x80000000) + val = (imm<<12) + (addr & 0xfffffffffffff000); + else + val = imm + addr; + + set_x_reg(regs, xn, val); + + instruction_pointer_set(regs, instruction_pointer(regs) + 4); +} + +void __kprobes +simulate_b_bl(u32 opcode, long addr, struct pt_regs *regs) +{ + int disp = bbl_displacement(opcode); + + /* Link register is x30 */ + if (opcode & (1 << 31)) + set_x_reg(regs, 30, addr + 4); + + instruction_pointer_set(regs, addr + disp); +} + +void __kprobes +simulate_b_cond(u32 opcode, long addr, struct pt_regs *regs) +{ + int disp = 4; + + if (aarch32_opcode_cond_checks[opcode & 0xf](regs->pstate & 0xffffffff)) + disp = bcond_displacement(opcode); + + instruction_pointer_set(regs, addr + disp); +} + +void __kprobes +simulate_br_blr_ret(u32 opcode, long addr, struct pt_regs *regs) +{ + int xn = (opcode >> 5) & 0x1f; + + /* update pc first in case we're doing a "blr lr" */ + instruction_pointer_set(regs, get_x_reg(regs, xn)); + + /* Link register is x30 */ + if (((opcode >> 21) & 0x3) == 1) + set_x_reg(regs, 30, addr + 4); +} + +void __kprobes +simulate_cbz_cbnz(u32 opcode, long addr, struct pt_regs *regs) +{ + int disp = 4; + + if (opcode & (1 << 24)) { + if (check_cbnz(opcode, regs)) + disp = cbz_displacement(opcode); + } else { + if (check_cbz(opcode, regs)) + disp = cbz_displacement(opcode); + } + instruction_pointer_set(regs, addr + disp); +} + +void __kprobes +simulate_tbz_tbnz(u32 opcode, long addr, struct pt_regs *regs) +{ + int disp = 4; + + if (opcode & (1 << 24)) { + if (check_tbnz(opcode, regs)) + disp = tbz_displacement(opcode); + } else { + if (check_tbz(opcode, regs)) + disp = tbz_displacement(opcode); + } + instruction_pointer_set(regs, addr + disp); +} + +void __kprobes +simulate_ldr_literal(u32 opcode, long addr, struct pt_regs *regs) +{ + u64 *load_addr; + int xn = opcode & 0x1f; + int disp; + + disp = ldr_displacement(opcode); + load_addr = (u64 *) (addr + disp); + + if (opcode & (1 << 30)) /* x0-x30 */ + set_x_reg(regs, xn, *load_addr); + else /* w0-w30 */ + set_w_reg(regs, xn, *load_addr); + + instruction_pointer_set(regs, instruction_pointer(regs) + 4); +} + +void __kprobes +simulate_ldrsw_literal(u32 opcode, long addr, struct pt_regs *regs) +{ + s32 *load_addr; + int xn = opcode & 0x1f; + int disp; + + disp = ldr_displacement(opcode); + load_addr = (s32 *) (addr + disp); + + set_x_reg(regs, xn, *load_addr); + + instruction_pointer_set(regs, instruction_pointer(regs) + 4); +} diff --git a/arch/arm64/kernel/probes/simulate-insn.h b/arch/arm64/kernel/probes/simulate-insn.h new file mode 100644 index 000000000000..050bde683c2d --- /dev/null +++ b/arch/arm64/kernel/probes/simulate-insn.h @@ -0,0 +1,28 @@ +/* + * arch/arm64/kernel/probes/simulate-insn.h + * + * Copyright (C) 2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#ifndef _ARM_KERNEL_KPROBES_SIMULATE_INSN_H +#define _ARM_KERNEL_KPROBES_SIMULATE_INSN_H + +void simulate_adr_adrp(u32 opcode, long addr, struct pt_regs *regs); +void simulate_b_bl(u32 opcode, long addr, struct pt_regs *regs); +void simulate_b_cond(u32 opcode, long addr, struct pt_regs *regs); +void simulate_br_blr_ret(u32 opcode, long addr, struct pt_regs *regs); +void simulate_cbz_cbnz(u32 opcode, long addr, struct pt_regs *regs); +void simulate_tbz_tbnz(u32 opcode, long addr, struct pt_regs *regs); +void simulate_ldr_literal(u32 opcode, long addr, struct pt_regs *regs); +void simulate_ldrsw_literal(u32 opcode, long addr, struct pt_regs *regs); + +#endif /* _ARM_KERNEL_KPROBES_SIMULATE_INSN_H */ diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 3f6cd5c5234f..030c1d5aa46d 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -48,6 +48,107 @@ #define CREATE_TRACE_POINTS #include +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} +#define GPR_OFFSET_NAME(r) \ + {.name = "x" #r, .offset = offsetof(struct pt_regs, regs[r])} + +static const struct pt_regs_offset regoffset_table[] = { + GPR_OFFSET_NAME(0), + GPR_OFFSET_NAME(1), + GPR_OFFSET_NAME(2), + GPR_OFFSET_NAME(3), + GPR_OFFSET_NAME(4), + GPR_OFFSET_NAME(5), + GPR_OFFSET_NAME(6), + GPR_OFFSET_NAME(7), + GPR_OFFSET_NAME(8), + GPR_OFFSET_NAME(9), + GPR_OFFSET_NAME(10), + GPR_OFFSET_NAME(11), + GPR_OFFSET_NAME(12), + GPR_OFFSET_NAME(13), + GPR_OFFSET_NAME(14), + GPR_OFFSET_NAME(15), + GPR_OFFSET_NAME(16), + GPR_OFFSET_NAME(17), + GPR_OFFSET_NAME(18), + GPR_OFFSET_NAME(19), + GPR_OFFSET_NAME(20), + GPR_OFFSET_NAME(21), + GPR_OFFSET_NAME(22), + GPR_OFFSET_NAME(23), + GPR_OFFSET_NAME(24), + GPR_OFFSET_NAME(25), + GPR_OFFSET_NAME(26), + GPR_OFFSET_NAME(27), + GPR_OFFSET_NAME(28), + GPR_OFFSET_NAME(29), + GPR_OFFSET_NAME(30), + {.name = "lr", .offset = offsetof(struct pt_regs, regs[30])}, + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(pc), + REG_OFFSET_NAME(pstate), + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +{ + return ((addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) || + on_irq_stack(addr, raw_smp_processor_id()); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} + /* * TODO: does not yet catch signals sent when the child dies. * in exit.c or in signal.c. diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S new file mode 100644 index 000000000000..51b73cdde287 --- /dev/null +++ b/arch/arm64/kernel/relocate_kernel.S @@ -0,0 +1,130 @@ +/* + * kexec for arm64 + * + * Copyright (C) Linaro. + * Copyright (C) Huawei Futurewei Technologies. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +#include +#include +#include +#include + +/* + * arm64_relocate_new_kernel - Put a 2nd stage image in place and boot it. + * + * The memory that the old kernel occupies may be overwritten when coping the + * new image to its final location. To assure that the + * arm64_relocate_new_kernel routine which does that copy is not overwritten, + * all code and data needed by arm64_relocate_new_kernel must be between the + * symbols arm64_relocate_new_kernel and arm64_relocate_new_kernel_end. The + * machine_kexec() routine will copy arm64_relocate_new_kernel to the kexec + * control_code_page, a special page which has been set up to be preserved + * during the copy operation. + */ +ENTRY(arm64_relocate_new_kernel) + + /* Setup the list loop variables. */ + mov x17, x1 /* x17 = kimage_start */ + mov x16, x0 /* x16 = kimage_head */ + dcache_line_size x15, x0 /* x15 = dcache line size */ + mov x14, xzr /* x14 = entry ptr */ + mov x13, xzr /* x13 = copy dest */ + + /* Clear the sctlr_el2 flags. */ + mrs x0, CurrentEL + cmp x0, #CurrentEL_EL2 + b.ne 1f + mrs x0, sctlr_el2 + ldr x1, =SCTLR_ELx_FLAGS + bic x0, x0, x1 + msr sctlr_el2, x0 + isb +1: + + /* Check if the new image needs relocation. */ + tbnz x16, IND_DONE_BIT, .Ldone + +.Lloop: + and x12, x16, PAGE_MASK /* x12 = addr */ + + /* Test the entry flags. */ +.Ltest_source: + tbz x16, IND_SOURCE_BIT, .Ltest_indirection + + /* Invalidate dest page to PoC. */ + mov x0, x13 + add x20, x0, #PAGE_SIZE + sub x1, x15, #1 + bic x0, x0, x1 +2: dc ivac, x0 + add x0, x0, x15 + cmp x0, x20 + b.lo 2b + dsb sy + + mov x20, x13 + mov x21, x12 + copy_page x20, x21, x0, x1, x2, x3, x4, x5, x6, x7 + + /* dest += PAGE_SIZE */ + add x13, x13, PAGE_SIZE + b .Lnext + +.Ltest_indirection: + tbz x16, IND_INDIRECTION_BIT, .Ltest_destination + + /* ptr = addr */ + mov x14, x12 + b .Lnext + +.Ltest_destination: + tbz x16, IND_DESTINATION_BIT, .Lnext + + /* dest = addr */ + mov x13, x12 + +.Lnext: + /* entry = *ptr++ */ + ldr x16, [x14], #8 + + /* while (!(entry & DONE)) */ + tbz x16, IND_DONE_BIT, .Lloop + +.Ldone: + /* wait for writes from copy_page to finish */ + dsb nsh + ic iallu + dsb nsh + isb + + /* Start new image. */ + mov x0, xzr + mov x1, xzr + mov x2, xzr + mov x3, xzr + br x17 + +ENDPROC(arm64_relocate_new_kernel) + +.ltorg + +.align 3 /* To keep the 64-bit values below naturally aligned. */ + +.Lcopy_end: +.org KEXEC_CONTROL_PAGE_SIZE + +/* + * arm64_relocate_new_kernel_size - Number of bytes to copy to the + * control_code_page. + */ +.globl arm64_relocate_new_kernel_size +arm64_relocate_new_kernel_size: + .quad .Lcopy_end - arm64_relocate_new_kernel diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 3279defabaa2..2981f1bdd073 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -202,7 +202,7 @@ static void __init request_standard_resources(void) struct resource *res; kernel_code.start = virt_to_phys(_text); - kernel_code.end = virt_to_phys(_etext - 1); + kernel_code.end = virt_to_phys(__init_begin - 1); kernel_data.start = virt_to_phys(_sdata); kernel_data.end = virt_to_phys(_end - 1); @@ -257,14 +257,17 @@ void __init setup_arch(char **cmdline_p) */ cpu_uninstall_idmap(); + xen_early_init(); efi_init(); arm64_memblock_init(); + paging_init(); + + acpi_table_upgrade(); + /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); - paging_init(); - if (acpi_disabled) unflatten_device_tree(); @@ -281,8 +284,6 @@ void __init setup_arch(char **cmdline_p) else psci_acpi_init(); - xen_early_init(); - cpu_read_bootcpu_ops(); smp_init_cpus(); smp_build_mpidr_hash(); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 678e0842cb3b..76a6d9263908 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -267,7 +267,6 @@ asmlinkage void secondary_start_kernel(void) set_cpu_online(cpu, true); complete(&cpu_running); - local_dbg_enable(); local_irq_enable(); local_async_enable(); @@ -437,9 +436,9 @@ void __init smp_cpus_done(unsigned int max_cpus) void __init smp_prepare_boot_cpu(void) { + set_my_cpu_offset(per_cpu_offset(smp_processor_id())); cpuinfo_store_boot_cpu(); save_boot_cpu_run_el(); - set_my_cpu_offset(per_cpu_offset(smp_processor_id())); } static u64 __init of_get_cpu_mpidr(struct device_node *dn) @@ -560,6 +559,8 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) */ acpi_set_mailbox_entry(cpu_count, processor); + early_map_cpu_to_node(cpu_count, acpi_numa_get_nid(cpu_count, hwid)); + cpu_count++; } @@ -693,6 +694,13 @@ void __init smp_prepare_cpus(unsigned int max_cpus) smp_store_cpu_info(smp_processor_id()); + /* + * If UP is mandated by "nosmp" (which implies "maxcpus=0"), don't set + * secondary CPUs present. + */ + if (max_cpus == 0) + return; + /* * Initialise the present map (which describes the set of CPUs * actually populated at the present time) and release the @@ -909,3 +917,21 @@ int setup_profiling_timer(unsigned int multiplier) { return -EINVAL; } + +static bool have_cpu_die(void) +{ +#ifdef CONFIG_HOTPLUG_CPU + int any_cpu = raw_smp_processor_id(); + + if (cpu_ops[any_cpu]->cpu_die) + return true; +#endif + return false; +} + +bool cpus_are_stuck_in_kernel(void) +{ + bool smp_spin_tables = (num_possible_cpus() > 1 && !have_cpu_die()); + + return !!cpus_stuck_in_kernel || smp_spin_tables; +} diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 2a43012616b7..e04f83873af7 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -41,6 +41,7 @@ #include #include #include +#include static const char *handler[]= { "Synchronous Abort", @@ -52,15 +53,14 @@ static const char *handler[]= { int show_unhandled_signals = 1; /* - * Dump out the contents of some memory nicely... + * Dump out the contents of some kernel memory nicely... */ static void dump_mem(const char *lvl, const char *str, unsigned long bottom, - unsigned long top, bool compat) + unsigned long top) { unsigned long first; mm_segment_t fs; int i; - unsigned int width = compat ? 4 : 8; /* * We need to switch to kernel mode so that we can use __get_user @@ -78,22 +78,15 @@ static void dump_mem(const char *lvl, const char *str, unsigned long bottom, memset(str, ' ', sizeof(str)); str[sizeof(str) - 1] = '\0'; - for (p = first, i = 0; i < (32 / width) - && p < top; i++, p += width) { + for (p = first, i = 0; i < (32 / 8) + && p < top; i++, p += 8) { if (p >= bottom && p < top) { unsigned long val; - if (width == 8) { - if (__get_user(val, (unsigned long *)p) == 0) - sprintf(str + i * 17, " %016lx", val); - else - sprintf(str + i * 17, " ????????????????"); - } else { - if (__get_user(val, (unsigned int *)p) == 0) - sprintf(str + i * 9, " %08lx", val); - else - sprintf(str + i * 9, " ????????"); - } + if (__get_user(val, (unsigned long *)p) == 0) + sprintf(str + i * 17, " %016lx", val); + else + sprintf(str + i * 17, " ????????????????"); } } printk("%s%04lx:%s\n", lvl, first & 0xffff, str); @@ -216,7 +209,7 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) stack = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr); dump_mem("", "Exception stack", stack, - stack + sizeof(struct pt_regs), false); + stack + sizeof(struct pt_regs)); } } } @@ -254,10 +247,9 @@ static int __die(const char *str, int err, struct thread_info *thread, pr_emerg("Process %.*s (pid: %d, stack limit = 0x%p)\n", TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), thread + 1); - if (!user_mode(regs) || in_interrupt()) { + if (!user_mode(regs)) { dump_mem(KERN_EMERG, "Stack: ", regs->sp, - THREAD_SIZE + (unsigned long)task_stack_page(tsk), - compat_user_mode(regs)); + THREAD_SIZE + (unsigned long)task_stack_page(tsk)); dump_backtrace(regs, tsk); dump_instr(KERN_EMERG, regs); } @@ -373,11 +365,59 @@ exit: return fn ? fn(regs, instr) : 1; } -asmlinkage void __exception do_undefinstr(struct pt_regs *regs) +static void force_signal_inject(int signal, int code, struct pt_regs *regs, + unsigned long address) { siginfo_t info; void __user *pc = (void __user *)instruction_pointer(regs); + const char *desc; + switch (signal) { + case SIGILL: + desc = "undefined instruction"; + break; + case SIGSEGV: + desc = "illegal memory access"; + break; + default: + desc = "bad mode"; + break; + } + + if (unhandled_signal(current, signal) && + show_unhandled_signals_ratelimited()) { + pr_info("%s[%d]: %s: pc=%p\n", + current->comm, task_pid_nr(current), desc, pc); + dump_instr(KERN_INFO, regs); + } + + info.si_signo = signal; + info.si_errno = 0; + info.si_code = code; + info.si_addr = pc; + + arm64_notify_die(desc, regs, &info, 0); +} + +/* + * Set up process info to signal segmentation fault - called on access error. + */ +void arm64_notify_segfault(struct pt_regs *regs, unsigned long addr) +{ + int code; + + down_read(¤t->mm->mmap_sem); + if (find_vma(current->mm, addr) == NULL) + code = SEGV_MAPERR; + else + code = SEGV_ACCERR; + up_read(¤t->mm->mmap_sem); + + force_signal_inject(SIGSEGV, code, regs, addr); +} + +asmlinkage void __exception do_undefinstr(struct pt_regs *regs) +{ /* check for AArch32 breakpoint instructions */ if (!aarch32_break_handler(regs)) return; @@ -385,18 +425,66 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs) if (call_undef_hook(regs) == 0) return; - if (unhandled_signal(current, SIGILL) && show_unhandled_signals_ratelimited()) { - pr_info("%s[%d]: undefined instruction: pc=%p\n", - current->comm, task_pid_nr(current), pc); - dump_instr(KERN_INFO, regs); - } + force_signal_inject(SIGILL, ILL_ILLOPC, regs, 0); +} - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_ILLOPC; - info.si_addr = pc; +void cpu_enable_cache_maint_trap(void *__unused) +{ + config_sctlr_el1(SCTLR_EL1_UCI, 0); +} + +#define __user_cache_maint(insn, address, res) \ + asm volatile ( \ + "1: " insn ", %1\n" \ + " mov %w0, #0\n" \ + "2:\n" \ + " .pushsection .fixup,\"ax\"\n" \ + " .align 2\n" \ + "3: mov %w0, %w2\n" \ + " b 2b\n" \ + " .popsection\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r" (res) \ + : "r" (address), "i" (-EFAULT) ) + +asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs) +{ + unsigned long address; + int ret; - arm64_notify_die("Oops - undefined instruction", regs, &info, 0); + /* if this is a write with: Op0=1, Op2=1, Op1=3, CRn=7 */ + if ((esr & 0x01fffc01) == 0x0012dc00) { + int rt = (esr >> 5) & 0x1f; + int crm = (esr >> 1) & 0x0f; + + address = (rt == 31) ? 0 : regs->regs[rt]; + + switch (crm) { + case 11: /* DC CVAU, gets promoted */ + __user_cache_maint("dc civac", address, ret); + break; + case 10: /* DC CVAC, gets promoted */ + __user_cache_maint("dc civac", address, ret); + break; + case 14: /* DC CIVAC */ + __user_cache_maint("dc civac", address, ret); + break; + case 5: /* IC IVAU */ + __user_cache_maint("ic ivau", address, ret); + break; + default: + force_signal_inject(SIGILL, ILL_ILLOPC, regs, 0); + return; + } + } else { + force_signal_inject(SIGILL, ILL_ILLOPC, regs, 0); + return; + } + + if (ret) + arm64_notify_segfault(regs, address); + else + regs->pc += 4; } long compat_arm_syscall(struct pt_regs *regs); @@ -465,7 +553,7 @@ static const char *esr_class_str[] = { const char *esr_get_class_string(u32 esr) { - return esr_class_str[esr >> ESR_ELx_EC_SHIFT]; + return esr_class_str[ESR_ELx_EC(esr)]; } /* diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index 9fefb005812a..076312b17d4f 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -214,10 +214,16 @@ void update_vsyscall(struct timekeeper *tk) vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec; if (!use_syscall) { + /* tkr_mono.cycle_last == tkr_raw.cycle_last */ vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; + vdso_data->raw_time_sec = tk->raw_time.tv_sec; + vdso_data->raw_time_nsec = tk->raw_time.tv_nsec; vdso_data->xtime_clock_sec = tk->xtime_sec; vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec; - vdso_data->cs_mult = tk->tkr_mono.mult; + /* tkr_raw.xtime_nsec == 0 */ + vdso_data->cs_mono_mult = tk->tkr_mono.mult; + vdso_data->cs_raw_mult = tk->tkr_raw.mult; + /* tkr_mono.shift == tkr_raw.shift */ vdso_data->cs_shift = tk->tkr_mono.shift; } diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index b467fd0a384b..62c84f7cb01b 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -23,7 +23,7 @@ GCOV_PROFILE := n ccflags-y += -Wl,-shared obj-y += vdso.o -extra-y += vdso.lds vdso-offsets.h +extra-y += vdso.lds CPPFLAGS_vdso.lds += -P -C -U$(ARCH) # Force dependency (incbin is bad) @@ -42,11 +42,10 @@ $(obj)/%.so: $(obj)/%.so.dbg FORCE gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ define cmd_vdsosym - $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ && \ - cp $@ include/generated/ + $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ endef -$(obj)/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE +include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE $(call if_changed,vdsosym) # Assembly rules for the .S files diff --git a/arch/arm64/kernel/vdso/gettimeofday.S b/arch/arm64/kernel/vdso/gettimeofday.S index efa79e8d4196..e00b4671bd7c 100644 --- a/arch/arm64/kernel/vdso/gettimeofday.S +++ b/arch/arm64/kernel/vdso/gettimeofday.S @@ -26,24 +26,109 @@ #define NSEC_PER_SEC_HI16 0x3b9a vdso_data .req x6 -use_syscall .req w7 -seqcnt .req w8 +seqcnt .req w7 +w_tmp .req w8 +x_tmp .req x8 + +/* + * Conventions for macro arguments: + * - An argument is write-only if its name starts with "res". + * - All other arguments are read-only, unless otherwise specified. + */ .macro seqcnt_acquire 9999: ldr seqcnt, [vdso_data, #VDSO_TB_SEQ_COUNT] tbnz seqcnt, #0, 9999b dmb ishld - ldr use_syscall, [vdso_data, #VDSO_USE_SYSCALL] .endm - .macro seqcnt_read, cnt + .macro seqcnt_check fail dmb ishld - ldr \cnt, [vdso_data, #VDSO_TB_SEQ_COUNT] + ldr w_tmp, [vdso_data, #VDSO_TB_SEQ_COUNT] + cmp w_tmp, seqcnt + b.ne \fail .endm - .macro seqcnt_check, cnt, fail - cmp \cnt, seqcnt - b.ne \fail + .macro syscall_check fail + ldr w_tmp, [vdso_data, #VDSO_USE_SYSCALL] + cbnz w_tmp, \fail + .endm + + .macro get_nsec_per_sec res + mov \res, #NSEC_PER_SEC_LO16 + movk \res, #NSEC_PER_SEC_HI16, lsl #16 + .endm + + /* + * Returns the clock delta, in nanoseconds left-shifted by the clock + * shift. + */ + .macro get_clock_shifted_nsec res, cycle_last, mult + /* Read the virtual counter. */ + isb + mrs x_tmp, cntvct_el0 + /* Calculate cycle delta and convert to ns. */ + sub \res, x_tmp, \cycle_last + /* We can only guarantee 56 bits of precision. */ + movn x_tmp, #0xff00, lsl #48 + and \res, x_tmp, \res + mul \res, \res, \mult + .endm + + /* + * Returns in res_{sec,nsec} the REALTIME timespec, based on the + * "wall time" (xtime) and the clock_mono delta. + */ + .macro get_ts_realtime res_sec, res_nsec, \ + clock_nsec, xtime_sec, xtime_nsec, nsec_to_sec + add \res_nsec, \clock_nsec, \xtime_nsec + udiv x_tmp, \res_nsec, \nsec_to_sec + add \res_sec, \xtime_sec, x_tmp + msub \res_nsec, x_tmp, \nsec_to_sec, \res_nsec + .endm + + /* + * Returns in res_{sec,nsec} the timespec based on the clock_raw delta, + * used for CLOCK_MONOTONIC_RAW. + */ + .macro get_ts_clock_raw res_sec, res_nsec, clock_nsec, nsec_to_sec + udiv \res_sec, \clock_nsec, \nsec_to_sec + msub \res_nsec, \res_sec, \nsec_to_sec, \clock_nsec + .endm + + /* sec and nsec are modified in place. */ + .macro add_ts sec, nsec, ts_sec, ts_nsec, nsec_to_sec + /* Add timespec. */ + add \sec, \sec, \ts_sec + add \nsec, \nsec, \ts_nsec + + /* Normalise the new timespec. */ + cmp \nsec, \nsec_to_sec + b.lt 9999f + sub \nsec, \nsec, \nsec_to_sec + add \sec, \sec, #1 +9999: + cmp \nsec, #0 + b.ge 9998f + add \nsec, \nsec, \nsec_to_sec + sub \sec, \sec, #1 +9998: + .endm + + .macro clock_gettime_return, shift=0 + .if \shift == 1 + lsr x11, x11, x12 + .endif + stp x10, x11, [x1, #TSPEC_TV_SEC] + mov x0, xzr + ret + .endm + + .macro jump_slot jumptable, index, label + .if (. - \jumptable) != 4 * (\index) + .error "Jump slot index mismatch" + .endif + b \label .endm .text @@ -51,18 +136,25 @@ seqcnt .req w8 /* int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz); */ ENTRY(__kernel_gettimeofday) .cfi_startproc - mov x2, x30 - .cfi_register x30, x2 - - /* Acquire the sequence counter and get the timespec. */ adr vdso_data, _vdso_data -1: seqcnt_acquire - cbnz use_syscall, 4f - /* If tv is NULL, skip to the timezone code. */ cbz x0, 2f - bl __do_get_tspec - seqcnt_check w9, 1b + + /* Compute the time of day. */ +1: seqcnt_acquire + syscall_check fail=4f + ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] + /* w11 = cs_mono_mult, w12 = cs_shift */ + ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] + ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] + seqcnt_check fail=1b + + get_nsec_per_sec res=x9 + lsl x9, x9, x12 + + get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 + get_ts_realtime res_sec=x10, res_nsec=x11, \ + clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 /* Convert ns to us. */ mov x13, #1000 @@ -76,95 +168,126 @@ ENTRY(__kernel_gettimeofday) stp w4, w5, [x1, #TZ_MINWEST] 3: mov x0, xzr - ret x2 + ret 4: /* Syscall fallback. */ mov x8, #__NR_gettimeofday svc #0 - ret x2 + ret .cfi_endproc ENDPROC(__kernel_gettimeofday) +#define JUMPSLOT_MAX CLOCK_MONOTONIC_COARSE + /* int __kernel_clock_gettime(clockid_t clock_id, struct timespec *tp); */ ENTRY(__kernel_clock_gettime) .cfi_startproc - cmp w0, #CLOCK_REALTIME - ccmp w0, #CLOCK_MONOTONIC, #0x4, ne - b.ne 2f + cmp w0, #JUMPSLOT_MAX + b.hi syscall + adr vdso_data, _vdso_data + adr x_tmp, jumptable + add x_tmp, x_tmp, w0, uxtw #2 + br x_tmp + + ALIGN +jumptable: + jump_slot jumptable, CLOCK_REALTIME, realtime + jump_slot jumptable, CLOCK_MONOTONIC, monotonic + b syscall + b syscall + jump_slot jumptable, CLOCK_MONOTONIC_RAW, monotonic_raw + jump_slot jumptable, CLOCK_REALTIME_COARSE, realtime_coarse + jump_slot jumptable, CLOCK_MONOTONIC_COARSE, monotonic_coarse + + .if (. - jumptable) != 4 * (JUMPSLOT_MAX + 1) + .error "Wrong jumptable size" + .endif + + ALIGN +realtime: + seqcnt_acquire + syscall_check fail=syscall + ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] + /* w11 = cs_mono_mult, w12 = cs_shift */ + ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] + ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] + seqcnt_check fail=realtime - mov x2, x30 - .cfi_register x30, x2 + /* All computations are done with left-shifted nsecs. */ + get_nsec_per_sec res=x9 + lsl x9, x9, x12 - /* Get kernel timespec. */ - adr vdso_data, _vdso_data -1: seqcnt_acquire - cbnz use_syscall, 7f + get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 + get_ts_realtime res_sec=x10, res_nsec=x11, \ + clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 + clock_gettime_return, shift=1 - bl __do_get_tspec - seqcnt_check w9, 1b + ALIGN +monotonic: + seqcnt_acquire + syscall_check fail=syscall + ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] + /* w11 = cs_mono_mult, w12 = cs_shift */ + ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] + ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] + ldp x3, x4, [vdso_data, #VDSO_WTM_CLK_SEC] + seqcnt_check fail=monotonic - mov x30, x2 + /* All computations are done with left-shifted nsecs. */ + lsl x4, x4, x12 + get_nsec_per_sec res=x9 + lsl x9, x9, x12 - cmp w0, #CLOCK_MONOTONIC - b.ne 6f + get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 + get_ts_realtime res_sec=x10, res_nsec=x11, \ + clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 - /* Get wtm timespec. */ - ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC] + add_ts sec=x10, nsec=x11, ts_sec=x3, ts_nsec=x4, nsec_to_sec=x9 + clock_gettime_return, shift=1 - /* Check the sequence counter. */ - seqcnt_read w9 - seqcnt_check w9, 1b - b 4f -2: - cmp w0, #CLOCK_REALTIME_COARSE - ccmp w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne - b.ne 8f + ALIGN +monotonic_raw: + seqcnt_acquire + syscall_check fail=syscall + ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] + /* w11 = cs_raw_mult, w12 = cs_shift */ + ldp w12, w11, [vdso_data, #VDSO_CS_SHIFT] + ldp x13, x14, [vdso_data, #VDSO_RAW_TIME_SEC] + seqcnt_check fail=monotonic_raw - /* xtime_coarse_nsec is already right-shifted */ - mov x12, #0 + /* All computations are done with left-shifted nsecs. */ + lsl x14, x14, x12 + get_nsec_per_sec res=x9 + lsl x9, x9, x12 - /* Get coarse timespec. */ - adr vdso_data, _vdso_data -3: seqcnt_acquire + get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 + get_ts_clock_raw res_sec=x10, res_nsec=x11, \ + clock_nsec=x15, nsec_to_sec=x9 + + add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9 + clock_gettime_return, shift=1 + + ALIGN +realtime_coarse: + seqcnt_acquire ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC] + seqcnt_check fail=realtime_coarse + clock_gettime_return - /* Get wtm timespec. */ + ALIGN +monotonic_coarse: + seqcnt_acquire + ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC] ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC] + seqcnt_check fail=monotonic_coarse - /* Check the sequence counter. */ - seqcnt_read w9 - seqcnt_check w9, 3b + /* Computations are done in (non-shifted) nsecs. */ + get_nsec_per_sec res=x9 + add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9 + clock_gettime_return - cmp w0, #CLOCK_MONOTONIC_COARSE - b.ne 6f -4: - /* Add on wtm timespec. */ - add x10, x10, x13 - lsl x14, x14, x12 - add x11, x11, x14 - - /* Normalise the new timespec. */ - mov x15, #NSEC_PER_SEC_LO16 - movk x15, #NSEC_PER_SEC_HI16, lsl #16 - lsl x15, x15, x12 - cmp x11, x15 - b.lt 5f - sub x11, x11, x15 - add x10, x10, #1 -5: - cmp x11, #0 - b.ge 6f - add x11, x11, x15 - sub x10, x10, #1 - -6: /* Store to the user timespec. */ - lsr x11, x11, x12 - stp x10, x11, [x1, #TSPEC_TV_SEC] - mov x0, xzr - ret -7: - mov x30, x2 -8: /* Syscall fallback. */ + ALIGN +syscall: /* Syscall fallback. */ mov x8, #__NR_clock_gettime svc #0 ret @@ -176,6 +299,7 @@ ENTRY(__kernel_clock_getres) .cfi_startproc cmp w0, #CLOCK_REALTIME ccmp w0, #CLOCK_MONOTONIC, #0x4, ne + ccmp w0, #CLOCK_MONOTONIC_RAW, #0x4, ne b.ne 1f ldr x2, 5f @@ -203,46 +327,3 @@ ENTRY(__kernel_clock_getres) .quad CLOCK_COARSE_RES .cfi_endproc ENDPROC(__kernel_clock_getres) - -/* - * Read the current time from the architected counter. - * Expects vdso_data to be initialised. - * Clobbers the temporary registers (x9 - x15). - * Returns: - * - w9 = vDSO sequence counter - * - (x10, x11) = (ts->tv_sec, shifted ts->tv_nsec) - * - w12 = cs_shift - */ -ENTRY(__do_get_tspec) - .cfi_startproc - - /* Read from the vDSO data page. */ - ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] - ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] - ldp w11, w12, [vdso_data, #VDSO_CS_MULT] - seqcnt_read w9 - - /* Read the virtual counter. */ - isb - mrs x15, cntvct_el0 - - /* Calculate cycle delta and convert to ns. */ - sub x10, x15, x10 - /* We can only guarantee 56 bits of precision. */ - movn x15, #0xff00, lsl #48 - and x10, x15, x10 - mul x10, x10, x11 - - /* Use the kernel time to calculate the new timespec. */ - mov x11, #NSEC_PER_SEC_LO16 - movk x11, #NSEC_PER_SEC_HI16, lsl #16 - lsl x11, x11, x12 - add x15, x10, x14 - udiv x14, x15, x11 - add x10, x13, x14 - mul x13, x14, x11 - sub x11, x15, x13 - - ret - .cfi_endproc -ENDPROC(__do_get_tspec) diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 435e820e898d..89d6e177ecbd 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -118,9 +118,11 @@ SECTIONS __exception_text_end = .; IRQENTRY_TEXT SOFTIRQENTRY_TEXT + ENTRY_TEXT TEXT_TEXT SCHED_TEXT LOCK_TEXT + KPROBES_TEXT HYPERVISOR_TEXT IDMAP_TEXT HIBERNATE_TEXT @@ -131,12 +133,13 @@ SECTIONS } . = ALIGN(SEGMENT_ALIGN); - RO_DATA(PAGE_SIZE) /* everything from this point to */ - EXCEPTION_TABLE(8) /* _etext will be marked RO NX */ + _etext = .; /* End of text section */ + + RO_DATA(PAGE_SIZE) /* everything from this point to */ + EXCEPTION_TABLE(8) /* __init_begin will be marked RO NX */ NOTES . = ALIGN(SEGMENT_ALIGN); - _etext = .; /* End of text and rodata section */ __init_begin = .; INIT_TEXT_SECTION(8) diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 3246c4aba5b1..fa96fe2bd469 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -106,7 +106,7 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run) run->exit_reason = KVM_EXIT_DEBUG; run->debug.arch.hsr = hsr; - switch (hsr >> ESR_ELx_EC_SHIFT) { + switch (ESR_ELx_EC(hsr)) { case ESR_ELx_EC_WATCHPT_LOW: run->debug.arch.far = vcpu->arch.fault.far_el2; /* fall through */ @@ -149,7 +149,7 @@ static exit_handle_fn arm_exit_handlers[] = { static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) { u32 hsr = kvm_vcpu_get_hsr(vcpu); - u8 hsr_ec = hsr >> ESR_ELx_EC_SHIFT; + u8 hsr_ec = ESR_ELx_EC(hsr); if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) || !arm_exit_handlers[hsr_ec]) { diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile index 778d0effa2af..0c85febcc1eb 100644 --- a/arch/arm64/kvm/hyp/Makefile +++ b/arch/arm64/kvm/hyp/Makefile @@ -17,6 +17,10 @@ obj-$(CONFIG_KVM_ARM_HOST) += tlb.o obj-$(CONFIG_KVM_ARM_HOST) += hyp-entry.o obj-$(CONFIG_KVM_ARM_HOST) += s2-setup.o +# KVM code is run at a different exception code with a different map, so +# compiler instrumentation that inserts callbacks or checks into the code may +# cause crashes. Just disable it. GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := n +KCOV_INSTRUMENT := n diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c index 437cfad5e3d8..4373997d1a70 100644 --- a/arch/arm64/kvm/hyp/switch.c +++ b/arch/arm64/kvm/hyp/switch.c @@ -198,7 +198,7 @@ static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar) static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu) { u64 esr = read_sysreg_el2(esr); - u8 ec = esr >> ESR_ELx_EC_SHIFT; + u8 ec = ESR_ELx_EC(esr); u64 hpfar, far; vcpu->arch.fault.esr_el2 = esr; diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c index 0f7c40eb3f53..934137647837 100644 --- a/arch/arm64/kvm/hyp/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/sysreg-sr.c @@ -27,8 +27,8 @@ static void __hyp_text __sysreg_do_nothing(struct kvm_cpu_context *ctxt) { } /* * Non-VHE: Both host and guest must save everything. * - * VHE: Host must save tpidr*_el[01], actlr_el1, sp0, pc, pstate, and - * guest must save everything. + * VHE: Host must save tpidr*_el[01], actlr_el1, mdscr_el1, sp0, pc, + * pstate, and guest must save everything. */ static void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt) @@ -37,6 +37,7 @@ static void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt) ctxt->sys_regs[TPIDR_EL0] = read_sysreg(tpidr_el0); ctxt->sys_regs[TPIDRRO_EL0] = read_sysreg(tpidrro_el0); ctxt->sys_regs[TPIDR_EL1] = read_sysreg(tpidr_el1); + ctxt->sys_regs[MDSCR_EL1] = read_sysreg(mdscr_el1); ctxt->gp_regs.regs.sp = read_sysreg(sp_el0); ctxt->gp_regs.regs.pc = read_sysreg_el2(elr); ctxt->gp_regs.regs.pstate = read_sysreg_el2(spsr); @@ -61,7 +62,6 @@ static void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt) ctxt->sys_regs[AMAIR_EL1] = read_sysreg_el1(amair); ctxt->sys_regs[CNTKCTL_EL1] = read_sysreg_el1(cntkctl); ctxt->sys_regs[PAR_EL1] = read_sysreg(par_el1); - ctxt->sys_regs[MDSCR_EL1] = read_sysreg(mdscr_el1); ctxt->gp_regs.sp_el1 = read_sysreg(sp_el1); ctxt->gp_regs.elr_el1 = read_sysreg_el1(elr); @@ -90,6 +90,7 @@ static void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctx write_sysreg(ctxt->sys_regs[TPIDR_EL0], tpidr_el0); write_sysreg(ctxt->sys_regs[TPIDRRO_EL0], tpidrro_el0); write_sysreg(ctxt->sys_regs[TPIDR_EL1], tpidr_el1); + write_sysreg(ctxt->sys_regs[MDSCR_EL1], mdscr_el1); write_sysreg(ctxt->gp_regs.regs.sp, sp_el0); write_sysreg_el2(ctxt->gp_regs.regs.pc, elr); write_sysreg_el2(ctxt->gp_regs.regs.pstate, spsr); @@ -114,7 +115,6 @@ static void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt) write_sysreg_el1(ctxt->sys_regs[AMAIR_EL1], amair); write_sysreg_el1(ctxt->sys_regs[CNTKCTL_EL1], cntkctl); write_sysreg(ctxt->sys_regs[PAR_EL1], par_el1); - write_sysreg(ctxt->sys_regs[MDSCR_EL1], mdscr_el1); write_sysreg(ctxt->gp_regs.sp_el1, sp_el1); write_sysreg_el1(ctxt->gp_regs.elr_el1, elr); diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 17e8306dca29..0b90497d4424 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -66,7 +66,7 @@ .endm end .req x5 -ENTRY(__copy_from_user) +ENTRY(__arch_copy_from_user) ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \ CONFIG_ARM64_PAN) add end, x0, x2 @@ -75,7 +75,7 @@ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \ CONFIG_ARM64_PAN) mov x0, #0 // Nothing to copy ret -ENDPROC(__copy_from_user) +ENDPROC(__arch_copy_from_user) .section .fixup,"ax" .align 2 diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index 21faae60f988..7a7efe255034 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -65,7 +65,7 @@ .endm end .req x5 -ENTRY(__copy_to_user) +ENTRY(__arch_copy_to_user) ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \ CONFIG_ARM64_PAN) add end, x0, x2 @@ -74,7 +74,7 @@ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \ CONFIG_ARM64_PAN) mov x0, #0 ret -ENDPROC(__copy_to_user) +ENDPROC(__arch_copy_to_user) .section .fixup,"ax" .align 2 diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 50ff9ba3a236..07d7352d7c38 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -52,7 +52,7 @@ ENTRY(__flush_cache_user_range) sub x3, x2, #1 bic x4, x0, x3 1: -USER(9f, dc cvau, x4 ) // clean D line to PoU +user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE add x4, x4, x2 cmp x4, x1 b.lo 1b diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index b7b397802088..efcf1f7ef1e4 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -179,7 +179,7 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu) &asid_generation); flush_context(cpu); - /* We have at least 1 ASID per CPU, so this will always succeed */ + /* We have more ASIDs than CPUs, so this will always succeed */ asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1); set_asid: @@ -227,8 +227,11 @@ switch_mm_fastpath: static int asids_init(void) { asid_bits = get_cpu_asid_bits(); - /* If we end up with more CPUs than ASIDs, expect things to crash */ - WARN_ON(NUM_USER_ASIDS < num_possible_cpus()); + /* + * Expect allocation after rollover to fail if we don't have at least + * one more ASID than CPUs. ASID #0 is reserved for init_mm. + */ + WARN_ON(NUM_USER_ASIDS - 1 <= num_possible_cpus()); atomic64_set(&asid_generation, ASID_FIRST_VERSION); asid_map = kzalloc(BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(*asid_map), GFP_KERNEL); diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index c566ec83719f..f6c55afab3e2 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,8 @@ #include +static int swiotlb __read_mostly; + static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot, bool coherent) { @@ -341,6 +344,13 @@ static int __swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt, return ret; } +static int __swiotlb_dma_supported(struct device *hwdev, u64 mask) +{ + if (swiotlb) + return swiotlb_dma_supported(hwdev, mask); + return 1; +} + static struct dma_map_ops swiotlb_dma_ops = { .alloc = __dma_alloc, .free = __dma_free, @@ -354,7 +364,7 @@ static struct dma_map_ops swiotlb_dma_ops = { .sync_single_for_device = __swiotlb_sync_single_for_device, .sync_sg_for_cpu = __swiotlb_sync_sg_for_cpu, .sync_sg_for_device = __swiotlb_sync_sg_for_device, - .dma_supported = swiotlb_dma_supported, + .dma_supported = __swiotlb_dma_supported, .mapping_error = swiotlb_dma_mapping_error, }; @@ -513,6 +523,9 @@ EXPORT_SYMBOL(dummy_dma_ops); static int __init arm64_dma_init(void) { + if (swiotlb_force || max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) + swiotlb = 1; + return atomic_pool_init(); } arch_initcall(arm64_dma_init); @@ -848,15 +861,16 @@ static int __iommu_attach_notifier(struct notifier_block *nb, { struct iommu_dma_notifier_data *master, *tmp; - if (action != BUS_NOTIFY_ADD_DEVICE) + if (action != BUS_NOTIFY_BIND_DRIVER) return 0; mutex_lock(&iommu_dma_notifier_lock); list_for_each_entry_safe(master, tmp, &iommu_dma_masters, list) { - if (do_iommu_attach(master->dev, master->ops, - master->dma_base, master->size)) { + if (data == master->dev && do_iommu_attach(master->dev, + master->ops, master->dma_base, master->size)) { list_del(&master->list); kfree(master); + break; } } mutex_unlock(&iommu_dma_notifier_lock); @@ -870,17 +884,8 @@ static int __init register_iommu_dma_ops_notifier(struct bus_type *bus) if (!nb) return -ENOMEM; - /* - * The device must be attached to a domain before the driver probe - * routine gets a chance to start allocating DMA buffers. However, - * the IOMMU driver also needs a chance to configure the iommu_group - * via its add_device callback first, so we need to make the attach - * happen between those two points. Since the IOMMU core uses a bus - * notifier with default priority for add_device, do the same but - * with a lower priority to ensure the appropriate ordering. - */ + nb->notifier_call = __iommu_attach_notifier; - nb->priority = -100; ret = bus_register_notifier(bus, nb); if (ret) { @@ -904,10 +909,6 @@ static int __init __iommu_dma_init(void) if (!ret) ret = register_iommu_dma_ops_notifier(&pci_bus_type); #endif - - /* handle devices queued before this arch_initcall */ - if (!ret) - __iommu_attach_notifier(NULL, BUS_NOTIFY_ADD_DEVICE, NULL); return ret; } arch_initcall(__iommu_dma_init); diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c index ccfde237d6e6..f94b80eb295d 100644 --- a/arch/arm64/mm/dump.c +++ b/arch/arm64/mm/dump.c @@ -27,11 +27,7 @@ #include #include #include - -struct addr_marker { - unsigned long start_address; - const char *name; -}; +#include static const struct addr_marker address_markers[] = { #ifdef CONFIG_KASAN @@ -290,7 +286,8 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } -static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long start) +static void walk_pgd(struct pg_state *st, struct mm_struct *mm, + unsigned long start) { pgd_t *pgd = pgd_offset(mm, 0UL); unsigned i; @@ -309,12 +306,13 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long st static int ptdump_show(struct seq_file *m, void *v) { + struct ptdump_info *info = m->private; struct pg_state st = { .seq = m, - .marker = address_markers, + .marker = info->markers, }; - walk_pgd(&st, &init_mm, VA_START); + walk_pgd(&st, info->mm, info->base_addr); note_page(&st, 0, 0, 0); return 0; @@ -322,7 +320,7 @@ static int ptdump_show(struct seq_file *m, void *v) static int ptdump_open(struct inode *inode, struct file *file) { - return single_open(file, ptdump_show, NULL); + return single_open(file, ptdump_show, inode->i_private); } static const struct file_operations ptdump_fops = { @@ -332,7 +330,7 @@ static const struct file_operations ptdump_fops = { .release = single_release, }; -static int ptdump_init(void) +int ptdump_register(struct ptdump_info *info, const char *name) { struct dentry *pe; unsigned i, j; @@ -342,8 +340,18 @@ static int ptdump_init(void) for (j = 0; j < pg_level[i].num; j++) pg_level[i].mask |= pg_level[i].bits[j].mask; - pe = debugfs_create_file("kernel_page_tables", 0400, NULL, NULL, - &ptdump_fops); + pe = debugfs_create_file(name, 0400, NULL, info, &ptdump_fops); return pe ? 0 : -ENOMEM; } + +static struct ptdump_info kernel_ptdump_info = { + .mm = &init_mm, + .markers = address_markers, + .base_addr = VA_START, +}; + +static int ptdump_init(void) +{ + return ptdump_register(&kernel_ptdump_info, "kernel_page_tables"); +} device_initcall(ptdump_init); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 013e2cbe7924..c8beaa0da7df 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -41,6 +41,28 @@ static const char *fault_name(unsigned int esr); +#ifdef CONFIG_KPROBES +static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) +{ + int ret = 0; + + /* kprobe_running() needs smp_processor_id() */ + if (!user_mode(regs)) { + preempt_disable(); + if (kprobe_running() && kprobe_fault_handler(regs, esr)) + ret = 1; + preempt_enable(); + } + + return ret; +} +#else +static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) +{ + return 0; +} +#endif + /* * Dump out the page tables associated with 'addr' in mm 'mm'. */ @@ -202,8 +224,6 @@ static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *re #define VM_FAULT_BADMAP 0x010000 #define VM_FAULT_BADACCESS 0x020000 -#define ESR_LNX_EXEC (1 << 24) - static int __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int mm_flags, unsigned long vm_flags, struct task_struct *tsk) @@ -233,7 +253,7 @@ good_area: goto out; } - return handle_mm_fault(mm, vma, addr & PAGE_MASK, mm_flags); + return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags); check_stack: if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) @@ -242,14 +262,19 @@ out: return fault; } -static inline int permission_fault(unsigned int esr) +static inline bool is_permission_fault(unsigned int esr) { - unsigned int ec = (esr & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT; + unsigned int ec = ESR_ELx_EC(esr); unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; return (ec == ESR_ELx_EC_DABT_CUR && fsc_type == ESR_ELx_FSC_PERM); } +static bool is_el0_instruction_abort(unsigned int esr) +{ + return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; +} + static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { @@ -259,6 +284,9 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC; unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + if (notify_page_fault(regs, esr)) + return 0; + tsk = current; mm = tsk->mm; @@ -272,15 +300,16 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, if (user_mode(regs)) mm_flags |= FAULT_FLAG_USER; - if (esr & ESR_LNX_EXEC) { + if (is_el0_instruction_abort(esr)) { vm_flags = VM_EXEC; } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { vm_flags = VM_WRITE; mm_flags |= FAULT_FLAG_WRITE; } - if (permission_fault(esr) && (addr < USER_DS)) { - if (get_fs() == KERNEL_DS) + if (is_permission_fault(esr) && (addr < USER_DS)) { + /* regs->orig_addr_limit may be 0 if we entered from EL0 */ + if (regs->orig_addr_limit == KERNEL_DS) die("Accessing user space memory with fs=KERNEL_DS", regs, esr); if (!search_exception_tables(regs->pc)) @@ -629,6 +658,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, return rv; } +NOKPROBE_SYMBOL(do_debug_exception); #ifdef CONFIG_ARM64_PAN void cpu_enable_pan(void *__unused) diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index dbd12ea8ce68..43a76b07eb32 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -71,10 +71,6 @@ void __sync_icache_dcache(pte_t pte, unsigned long addr) { struct page *page = pte_page(pte); - /* no flushing needed for anonymous pages */ - if (!page_mapping(page)) - return; - if (!test_and_set_bit(PG_dcache_clean, &page->flags)) sync_icache_aliases(page_address(page), PAGE_SIZE << compound_order(page)); diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index d45f8627012c..2ade7a6a10a7 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -160,12 +160,10 @@ static void __init arm64_memory_present(void) static void __init arm64_memory_present(void) { struct memblock_region *reg; - int nid = 0; for_each_memblock(memory, reg) { -#ifdef CONFIG_NUMA - nid = reg->nid; -#endif + int nid = memblock_get_region_node(reg); + memory_present(nid, memblock_region_memory_base_pfn(reg), memblock_region_memory_end_pfn(reg)); } @@ -403,7 +401,8 @@ static void __init free_unused_memmap(void) */ void __init mem_init(void) { - swiotlb_init(1); + if (swiotlb_force || max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) + swiotlb_init(1); set_max_mapnr(pfn_to_page(max_pfn) - mem_map); @@ -430,9 +429,9 @@ void __init mem_init(void) pr_cont(" vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n", MLG(VMALLOC_START, VMALLOC_END)); pr_cont(" .text : 0x%p" " - 0x%p" " (%6ld KB)\n", - MLK_ROUNDUP(_text, __start_rodata)); + MLK_ROUNDUP(_text, _etext)); pr_cont(" .rodata : 0x%p" " - 0x%p" " (%6ld KB)\n", - MLK_ROUNDUP(__start_rodata, _etext)); + MLK_ROUNDUP(__start_rodata, __init_begin)); pr_cont(" .init : 0x%p" " - 0x%p" " (%6ld KB)\n", MLK_ROUNDUP(__init_begin, __init_end)); pr_cont(" .data : 0x%p" " - 0x%p" " (%6ld KB)\n", diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 0f85a46c3e18..51a558195bb9 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -77,7 +77,6 @@ static phys_addr_t __init early_pgtable_alloc(void) void *ptr; phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE); - BUG_ON(!phys); /* * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE @@ -97,24 +96,6 @@ static phys_addr_t __init early_pgtable_alloc(void) return phys; } -/* - * remap a PMD into pages - */ -static void split_pmd(pmd_t *pmd, pte_t *pte) -{ - unsigned long pfn = pmd_pfn(*pmd); - int i = 0; - - do { - /* - * Need to have the least restrictive permissions available - * permissions will be fixed up later - */ - set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC)); - pfn++; - } while (pte++, i++, i < PTRS_PER_PTE); -} - static void alloc_init_pte(pmd_t *pmd, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot, @@ -122,15 +103,13 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr, { pte_t *pte; - if (pmd_none(*pmd) || pmd_sect(*pmd)) { + BUG_ON(pmd_sect(*pmd)); + if (pmd_none(*pmd)) { phys_addr_t pte_phys; BUG_ON(!pgtable_alloc); pte_phys = pgtable_alloc(); pte = pte_set_fixmap(pte_phys); - if (pmd_sect(*pmd)) - split_pmd(pmd, pte); __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE); - flush_tlb_all(); pte_clear_fixmap(); } BUG_ON(pmd_bad(*pmd)); @@ -144,41 +123,10 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr, pte_clear_fixmap(); } -static void split_pud(pud_t *old_pud, pmd_t *pmd) -{ - unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; - pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); - int i = 0; - - do { - set_pmd(pmd, __pmd(addr | pgprot_val(prot))); - addr += PMD_SIZE; - } while (pmd++, i++, i < PTRS_PER_PMD); -} - -#ifdef CONFIG_DEBUG_PAGEALLOC -static bool block_mappings_allowed(phys_addr_t (*pgtable_alloc)(void)) -{ - - /* - * If debug_page_alloc is enabled we must map the linear map - * using pages. However, other mappings created by - * create_mapping_noalloc must use sections in some cases. Allow - * sections to be used in those cases, where no pgtable_alloc - * function is provided. - */ - return !pgtable_alloc || !debug_pagealloc_enabled(); -} -#else -static bool block_mappings_allowed(phys_addr_t (*pgtable_alloc)(void)) -{ - return true; -} -#endif - static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, - phys_addr_t (*pgtable_alloc)(void)) + phys_addr_t (*pgtable_alloc)(void), + bool allow_block_mappings) { pmd_t *pmd; unsigned long next; @@ -186,20 +134,13 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, /* * Check for initial section mappings in the pgd/pud and remove them. */ - if (pud_none(*pud) || pud_sect(*pud)) { + BUG_ON(pud_sect(*pud)); + if (pud_none(*pud)) { phys_addr_t pmd_phys; BUG_ON(!pgtable_alloc); pmd_phys = pgtable_alloc(); pmd = pmd_set_fixmap(pmd_phys); - if (pud_sect(*pud)) { - /* - * need to have the 1G of mappings continue to be - * present - */ - split_pud(pud, pmd); - } __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE); - flush_tlb_all(); pmd_clear_fixmap(); } BUG_ON(pud_bad(*pud)); @@ -209,7 +150,7 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, next = pmd_addr_end(addr, end); /* try section mapping first */ if (((addr | next | phys) & ~SECTION_MASK) == 0 && - block_mappings_allowed(pgtable_alloc)) { + allow_block_mappings) { pmd_t old_pmd =*pmd; pmd_set_huge(pmd, phys, prot); /* @@ -248,7 +189,8 @@ static inline bool use_1G_block(unsigned long addr, unsigned long next, static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, - phys_addr_t (*pgtable_alloc)(void)) + phys_addr_t (*pgtable_alloc)(void), + bool allow_block_mappings) { pud_t *pud; unsigned long next; @@ -268,8 +210,7 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, /* * For 4K granule only, attempt to put down a 1GB block */ - if (use_1G_block(addr, next, phys) && - block_mappings_allowed(pgtable_alloc)) { + if (use_1G_block(addr, next, phys) && allow_block_mappings) { pud_t old_pud = *pud; pud_set_huge(pud, phys, prot); @@ -290,7 +231,7 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, } } else { alloc_init_pmd(pud, addr, next, phys, prot, - pgtable_alloc); + pgtable_alloc, allow_block_mappings); } phys += next - addr; } while (pud++, addr = next, addr != end); @@ -298,15 +239,14 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, pud_clear_fixmap(); } -/* - * Create the page directory entries and any necessary page tables for the - * mapping specified by 'md'. - */ -static void init_pgd(pgd_t *pgd, phys_addr_t phys, unsigned long virt, - phys_addr_t size, pgprot_t prot, - phys_addr_t (*pgtable_alloc)(void)) +static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(void), + bool allow_block_mappings) { unsigned long addr, length, end, next; + pgd_t *pgd = pgd_offset_raw(pgdir, virt); /* * If the virtual and physical address don't have the same offset @@ -322,29 +262,23 @@ static void init_pgd(pgd_t *pgd, phys_addr_t phys, unsigned long virt, end = addr + length; do { next = pgd_addr_end(addr, end); - alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc); + alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc, + allow_block_mappings); phys += next - addr; } while (pgd++, addr = next, addr != end); } -static phys_addr_t late_pgtable_alloc(void) +static phys_addr_t pgd_pgtable_alloc(void) { void *ptr = (void *)__get_free_page(PGALLOC_GFP); - BUG_ON(!ptr); + if (!ptr || !pgtable_page_ctor(virt_to_page(ptr))) + BUG(); /* Ensure the zeroed page is visible to the page table walker */ dsb(ishst); return __pa(ptr); } -static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, - unsigned long virt, phys_addr_t size, - pgprot_t prot, - phys_addr_t (*alloc)(void)) -{ - init_pgd(pgd_offset_raw(pgdir, virt), phys, virt, size, prot, alloc); -} - /* * This function can only be used to modify existing table entries, * without allocating new levels of table. Note that this permits the @@ -358,16 +292,17 @@ static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } - __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, - NULL); + __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, true); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, - pgprot_t prot) + pgprot_t prot, bool allow_block_mappings) { + BUG_ON(mm == &init_mm); + __create_pgd_mapping(mm->pgd, phys, virt, size, prot, - late_pgtable_alloc); + pgd_pgtable_alloc, allow_block_mappings); } static void create_mapping_late(phys_addr_t phys, unsigned long virt, @@ -380,51 +315,54 @@ static void create_mapping_late(phys_addr_t phys, unsigned long virt, } __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, - late_pgtable_alloc); + NULL, !debug_pagealloc_enabled()); } static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end) { unsigned long kernel_start = __pa(_text); - unsigned long kernel_end = __pa(_etext); + unsigned long kernel_end = __pa(__init_begin); /* * Take care not to create a writable alias for the * read-only text and rodata sections of the kernel image. */ - /* No overlap with the kernel text */ + /* No overlap with the kernel text/rodata */ if (end < kernel_start || start >= kernel_end) { __create_pgd_mapping(pgd, start, __phys_to_virt(start), end - start, PAGE_KERNEL, - early_pgtable_alloc); + early_pgtable_alloc, + !debug_pagealloc_enabled()); return; } /* - * This block overlaps the kernel text mapping. + * This block overlaps the kernel text/rodata mappings. * Map the portion(s) which don't overlap. */ if (start < kernel_start) __create_pgd_mapping(pgd, start, __phys_to_virt(start), kernel_start - start, PAGE_KERNEL, - early_pgtable_alloc); + early_pgtable_alloc, + !debug_pagealloc_enabled()); if (kernel_end < end) __create_pgd_mapping(pgd, kernel_end, __phys_to_virt(kernel_end), end - kernel_end, PAGE_KERNEL, - early_pgtable_alloc); + early_pgtable_alloc, + !debug_pagealloc_enabled()); /* - * Map the linear alias of the [_text, _etext) interval as + * Map the linear alias of the [_text, __init_begin) interval as * read-only/non-executable. This makes the contents of the * region accessible to subsystems such as hibernate, but * protects it from inadvertent modification or execution. */ __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start), kernel_end - kernel_start, PAGE_KERNEL_RO, - early_pgtable_alloc); + early_pgtable_alloc, !debug_pagealloc_enabled()); } static void __init map_mem(pgd_t *pgd) @@ -449,14 +387,14 @@ void mark_rodata_ro(void) { unsigned long section_size; - section_size = (unsigned long)__start_rodata - (unsigned long)_text; + section_size = (unsigned long)_etext - (unsigned long)_text; create_mapping_late(__pa(_text), (unsigned long)_text, section_size, PAGE_KERNEL_ROX); /* - * mark .rodata as read only. Use _etext rather than __end_rodata to - * cover NOTES and EXCEPTION_TABLE. + * mark .rodata as read only. Use __init_begin rather than __end_rodata + * to cover NOTES and EXCEPTION_TABLE. */ - section_size = (unsigned long)_etext - (unsigned long)__start_rodata; + section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata, section_size, PAGE_KERNEL_RO); } @@ -481,7 +419,7 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, BUG_ON(!PAGE_ALIGNED(size)); __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, - early_pgtable_alloc); + early_pgtable_alloc, !debug_pagealloc_enabled()); vma->addr = va_start; vma->phys_addr = pa_start; @@ -499,8 +437,8 @@ static void __init map_kernel(pgd_t *pgd) { static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data; - map_kernel_segment(pgd, _text, __start_rodata, PAGE_KERNEL_EXEC, &vmlinux_text); - map_kernel_segment(pgd, __start_rodata, _etext, PAGE_KERNEL, &vmlinux_rodata); + map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text); + map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata); map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC, &vmlinux_init); map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data); diff --git a/arch/arm64/mm/numa.c b/arch/arm64/mm/numa.c index 98dc1047f2a2..c7fe3ec70774 100644 --- a/arch/arm64/mm/numa.c +++ b/arch/arm64/mm/numa.c @@ -17,6 +17,7 @@ * along with this program. If not, see . */ +#include #include #include #include @@ -29,7 +30,7 @@ static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE }; static int numa_distance_cnt; static u8 *numa_distance; -static int numa_off; +static bool numa_off; static __init int numa_parse_early_param(char *opt) { @@ -37,7 +38,7 @@ static __init int numa_parse_early_param(char *opt) return -EINVAL; if (!strncmp(opt, "off", 3)) { pr_info("%s\n", "NUMA turned off"); - numa_off = 1; + numa_off = true; } return 0; } @@ -131,25 +132,25 @@ void __init early_map_cpu_to_node(unsigned int cpu, int nid) * numa_add_memblk - Set node id to memblk * @nid: NUMA node ID of the new memblk * @start: Start address of the new memblk - * @size: Size of the new memblk + * @end: End address of the new memblk * * RETURNS: * 0 on success, -errno on failure. */ -int __init numa_add_memblk(int nid, u64 start, u64 size) +int __init numa_add_memblk(int nid, u64 start, u64 end) { int ret; - ret = memblock_set_node(start, size, &memblock.memory, nid); + ret = memblock_set_node(start, (end - start), &memblock.memory, nid); if (ret < 0) { pr_err("NUMA: memblock [0x%llx - 0x%llx] failed to add on node %d\n", - start, (start + size - 1), nid); + start, (end - 1), nid); return ret; } node_set(nid, numa_nodes_parsed); pr_info("NUMA: Adding memblock [0x%llx - 0x%llx] on node %d\n", - start, (start + size - 1), nid); + start, (end - 1), nid); return ret; } @@ -362,12 +363,15 @@ static int __init dummy_numa_init(void) int ret; struct memblock_region *mblk; - pr_info("%s\n", "No NUMA configuration found"); + if (numa_off) + pr_info("NUMA disabled\n"); /* Forced off on command line. */ + else + pr_info("No NUMA configuration found\n"); pr_info("NUMA: Faking a node at [mem %#018Lx-%#018Lx]\n", 0LLU, PFN_PHYS(max_pfn) - 1); for_each_memblock(memory, mblk) { - ret = numa_add_memblk(0, mblk->base, mblk->size); + ret = numa_add_memblk(0, mblk->base, mblk->base + mblk->size); if (!ret) continue; @@ -375,7 +379,7 @@ static int __init dummy_numa_init(void) return ret; } - numa_off = 1; + numa_off = true; return 0; } @@ -388,7 +392,9 @@ static int __init dummy_numa_init(void) void __init arm64_numa_init(void) { if (!numa_off) { - if (!numa_init(of_numa_init)) + if (!acpi_disabled && !numa_init(arm64_acpi_numa_init)) + return; + if (acpi_disabled && !numa_init(of_numa_init)) return; } diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index c4317879b938..5bb61de23201 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -180,6 +180,8 @@ ENTRY(__cpu_setup) msr cpacr_el1, x0 // Enable FP/ASIMD mov x0, #1 << 12 // Reset mdscr_el1 and disable msr mdscr_el1, x0 // access to the DCC from EL0 + isb // Unmask debug exceptions now, + enable_dbg // since this is per-cpu reset_pmuserenr_el0 x0 // Disable PMU access from EL0 /* * Memory region attributes for LPAE: diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index aee5637ea436..7c16e547ccb2 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -1,7 +1,7 @@ /* * BPF JIT compiler for ARM64 * - * Copyright (C) 2014-2015 Zi Shen Lim + * Copyright (C) 2014-2016 Zi Shen Lim * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -55,6 +55,7 @@ #define A64_BL(imm26) A64_BRANCH((imm26) << 2, LINK) /* Unconditional branch (register) */ +#define A64_BR(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_NOLINK) #define A64_BLR(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_LINK) #define A64_RET(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_RETURN) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 49ba37e4bfc0..b2fc97a2c56c 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -18,6 +18,7 @@ #define pr_fmt(fmt) "bpf_jit: " fmt +#include #include #include #include @@ -33,6 +34,7 @@ int bpf_jit_enable __read_mostly; #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) +#define TCALL_CNT (MAX_BPF_JIT_REG + 2) /* Map BPF registers to A64 registers */ static const int bpf2a64[] = { @@ -54,6 +56,8 @@ static const int bpf2a64[] = { /* temporary registers for internal BPF JIT */ [TMP_REG_1] = A64_R(10), [TMP_REG_2] = A64_R(11), + /* tail_call_cnt */ + [TCALL_CNT] = A64_R(26), /* temporary register for blinding constants */ [BPF_REG_AX] = A64_R(9), }; @@ -146,13 +150,18 @@ static inline int epilogue_offset(const struct jit_ctx *ctx) #define STACK_SIZE STACK_ALIGN(_STACK_SIZE) -static void build_prologue(struct jit_ctx *ctx) +#define PROLOGUE_OFFSET 8 + +static int build_prologue(struct jit_ctx *ctx) { const u8 r6 = bpf2a64[BPF_REG_6]; const u8 r7 = bpf2a64[BPF_REG_7]; const u8 r8 = bpf2a64[BPF_REG_8]; const u8 r9 = bpf2a64[BPF_REG_9]; const u8 fp = bpf2a64[BPF_REG_FP]; + const u8 tcc = bpf2a64[TCALL_CNT]; + const int idx0 = ctx->idx; + int cur_offset; /* * BPF prog stack layout @@ -162,8 +171,6 @@ static void build_prologue(struct jit_ctx *ctx) * |FP/LR| * current A64_FP => -16:+-----+ * | ... | callee saved registers - * +-----+ - * | | x25/x26 * BPF fp register => -64:+-----+ <= (BPF_FP) * | | * | ... | BPF prog stack @@ -183,18 +190,90 @@ static void build_prologue(struct jit_ctx *ctx) emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); emit(A64_MOV(1, A64_FP, A64_SP), ctx); - /* Save callee-saved register */ + /* Save callee-saved registers */ emit(A64_PUSH(r6, r7, A64_SP), ctx); emit(A64_PUSH(r8, r9, A64_SP), ctx); + emit(A64_PUSH(fp, tcc, A64_SP), ctx); - /* Save fp (x25) and x26. SP requires 16 bytes alignment */ - emit(A64_PUSH(fp, A64_R(26), A64_SP), ctx); - - /* Set up BPF prog stack base register (x25) */ + /* Set up BPF prog stack base register */ emit(A64_MOV(1, fp, A64_SP), ctx); + /* Initialize tail_call_cnt */ + emit(A64_MOVZ(1, tcc, 0, 0), ctx); + /* Set up function call stack */ emit(A64_SUB_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); + + cur_offset = ctx->idx - idx0; + if (cur_offset != PROLOGUE_OFFSET) { + pr_err_once("PROLOGUE_OFFSET = %d, expected %d!\n", + cur_offset, PROLOGUE_OFFSET); + return -1; + } + return 0; +} + +static int out_offset = -1; /* initialized on the first pass of build_body() */ +static int emit_bpf_tail_call(struct jit_ctx *ctx) +{ + /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ + const u8 r2 = bpf2a64[BPF_REG_2]; + const u8 r3 = bpf2a64[BPF_REG_3]; + + const u8 tmp = bpf2a64[TMP_REG_1]; + const u8 prg = bpf2a64[TMP_REG_2]; + const u8 tcc = bpf2a64[TCALL_CNT]; + const int idx0 = ctx->idx; +#define cur_offset (ctx->idx - idx0) +#define jmp_offset (out_offset - (cur_offset)) + size_t off; + + /* if (index >= array->map.max_entries) + * goto out; + */ + off = offsetof(struct bpf_array, map.max_entries); + emit_a64_mov_i64(tmp, off, ctx); + emit(A64_LDR32(tmp, r2, tmp), ctx); + emit(A64_CMP(0, r3, tmp), ctx); + emit(A64_B_(A64_COND_GE, jmp_offset), ctx); + + /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * goto out; + * tail_call_cnt++; + */ + emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx); + emit(A64_CMP(1, tcc, tmp), ctx); + emit(A64_B_(A64_COND_GT, jmp_offset), ctx); + emit(A64_ADD_I(1, tcc, tcc, 1), ctx); + + /* prog = array->ptrs[index]; + * if (prog == NULL) + * goto out; + */ + off = offsetof(struct bpf_array, ptrs); + emit_a64_mov_i64(tmp, off, ctx); + emit(A64_LDR64(tmp, r2, tmp), ctx); + emit(A64_LDR64(prg, tmp, r3), ctx); + emit(A64_CBZ(1, prg, jmp_offset), ctx); + + /* goto *(prog->bpf_func + prologue_size); */ + off = offsetof(struct bpf_prog, bpf_func); + emit_a64_mov_i64(tmp, off, ctx); + emit(A64_LDR64(tmp, prg, tmp), ctx); + emit(A64_ADD_I(1, tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx); + emit(A64_BR(tmp), ctx); + + /* out: */ + if (out_offset == -1) + out_offset = cur_offset; + if (cur_offset != out_offset) { + pr_err_once("tail_call out_offset = %d, expected %d!\n", + cur_offset, out_offset); + return -1; + } + return 0; +#undef cur_offset +#undef jmp_offset } static void build_epilogue(struct jit_ctx *ctx) @@ -499,13 +578,15 @@ emit_cond_jmp: const u64 func = (u64)__bpf_call_base + imm; emit_a64_mov_i64(tmp, func, ctx); - emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); - emit(A64_MOV(1, A64_FP, A64_SP), ctx); emit(A64_BLR(tmp), ctx); emit(A64_MOV(1, r0, A64_R(0)), ctx); - emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); break; } + /* tail call */ + case BPF_JMP | BPF_CALL | BPF_X: + if (emit_bpf_tail_call(ctx)) + return -EFAULT; + break; /* function return */ case BPF_JMP | BPF_EXIT: /* Optimization: when last instruction is EXIT, @@ -650,11 +731,8 @@ emit_cond_jmp: emit_a64_mov_i64(r3, size, ctx); emit(A64_SUB_I(1, r4, fp, STACK_SIZE), ctx); emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx); - emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); - emit(A64_MOV(1, A64_FP, A64_SP), ctx); emit(A64_BLR(r5), ctx); emit(A64_MOV(1, r0, A64_R(0)), ctx); - emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); jmp_offset = epilogue_offset(ctx); check_imm19(jmp_offset); @@ -780,7 +858,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) goto out_off; } - build_prologue(&ctx); + if (build_prologue(&ctx)) { + prog = orig_prog; + goto out_off; + } ctx.epilogue_offset = ctx.idx; build_epilogue(&ctx); diff --git a/arch/arm64/xen/Makefile b/arch/arm64/xen/Makefile index 74a8d87e542b..8ff8aa9c6228 100644 --- a/arch/arm64/xen/Makefile +++ b/arch/arm64/xen/Makefile @@ -1,2 +1,3 @@ xen-arm-y += $(addprefix ../../arm/xen/, enlighten.o grant-table.o p2m.o mm.o) obj-y := xen-arm.o hypercall.o +obj-$(CONFIG_XEN_EFI) += $(addprefix ../../arm/xen/, efi.o) diff --git a/arch/arm64/xen/hypercall.S b/arch/arm64/xen/hypercall.S index 70df80e8da2c..329c8027b0a9 100644 --- a/arch/arm64/xen/hypercall.S +++ b/arch/arm64/xen/hypercall.S @@ -82,6 +82,7 @@ HYPERCALL3(vcpu_op); HYPERCALL1(tmem_op); HYPERCALL1(platform_op_raw); HYPERCALL2(multicall); +HYPERCALL2(vm_assist); ENTRY(privcmd_call) mov x16, x0 diff --git a/arch/avr32/include/asm/atomic.h b/arch/avr32/include/asm/atomic.h index d74fd8ce980a..3d5ce38a6f0b 100644 --- a/arch/avr32/include/asm/atomic.h +++ b/arch/avr32/include/asm/atomic.h @@ -41,21 +41,49 @@ static inline int __atomic_##op##_return(int i, atomic_t *v) \ return result; \ } +#define ATOMIC_FETCH_OP(op, asm_op, asm_con) \ +static inline int __atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int result, val; \ + \ + asm volatile( \ + "/* atomic_fetch_" #op " */\n" \ + "1: ssrf 5\n" \ + " ld.w %0, %3\n" \ + " mov %1, %0\n" \ + " " #asm_op " %1, %4\n" \ + " stcond %2, %1\n" \ + " brne 1b" \ + : "=&r" (result), "=&r" (val), "=o" (v->counter) \ + : "m" (v->counter), #asm_con (i) \ + : "cc"); \ + \ + return result; \ +} + ATOMIC_OP_RETURN(sub, sub, rKs21) ATOMIC_OP_RETURN(add, add, r) +ATOMIC_FETCH_OP (sub, sub, rKs21) +ATOMIC_FETCH_OP (add, add, r) -#define ATOMIC_OP(op, asm_op) \ +#define ATOMIC_OPS(op, asm_op) \ ATOMIC_OP_RETURN(op, asm_op, r) \ static inline void atomic_##op(int i, atomic_t *v) \ { \ (void)__atomic_##op##_return(i, v); \ +} \ +ATOMIC_FETCH_OP(op, asm_op, r) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + return __atomic_fetch_##op(i, v); \ } -ATOMIC_OP(and, and) -ATOMIC_OP(or, or) -ATOMIC_OP(xor, eor) +ATOMIC_OPS(and, and) +ATOMIC_OPS(or, or) +ATOMIC_OPS(xor, eor) -#undef ATOMIC_OP +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN /* @@ -87,6 +115,14 @@ static inline int atomic_add_return(int i, atomic_t *v) return __atomic_add_return(i, v); } +static inline int atomic_fetch_add(int i, atomic_t *v) +{ + if (IS_21BIT_CONST(i)) + return __atomic_fetch_sub(-i, v); + + return __atomic_fetch_add(i, v); +} + /* * atomic_sub_return - subtract the atomic variable * @i: integer value to subtract @@ -102,6 +138,14 @@ static inline int atomic_sub_return(int i, atomic_t *v) return __atomic_add_return(-i, v); } +static inline int atomic_fetch_sub(int i, atomic_t *v) +{ + if (IS_21BIT_CONST(i)) + return __atomic_fetch_sub(i, v); + + return __atomic_fetch_add(-i, v); +} + /* * __atomic_add_unless - add unless the number is a given value * @v: pointer of type atomic_t diff --git a/arch/avr32/include/asm/pgalloc.h b/arch/avr32/include/asm/pgalloc.h index 1aba19d68c5e..db039cb368be 100644 --- a/arch/avr32/include/asm/pgalloc.h +++ b/arch/avr32/include/asm/pgalloc.h @@ -43,7 +43,7 @@ static inline void pgd_ctor(void *x) */ static inline pgd_t *pgd_alloc(struct mm_struct *mm) { - return quicklist_alloc(QUICK_PGD, GFP_KERNEL | __GFP_REPEAT, pgd_ctor); + return quicklist_alloc(QUICK_PGD, GFP_KERNEL, pgd_ctor); } static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) @@ -54,7 +54,7 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - return quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL); + return quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm, @@ -63,7 +63,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, struct page *page; void *pg; - pg = quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL); + pg = quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL); if (!pg) return NULL; diff --git a/arch/avr32/mm/fault.c b/arch/avr32/mm/fault.c index c03533937a9f..a4b7edac8f10 100644 --- a/arch/avr32/mm/fault.c +++ b/arch/avr32/mm/fault.c @@ -134,7 +134,7 @@ good_area: * sure we exit gracefully rather than endlessly redo the * fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/blackfin/include/asm/atomic.h b/arch/blackfin/include/asm/atomic.h index 1c1c42330c99..63c7deceeeb6 100644 --- a/arch/blackfin/include/asm/atomic.h +++ b/arch/blackfin/include/asm/atomic.h @@ -17,6 +17,7 @@ asmlinkage int __raw_uncached_fetch_asm(const volatile int *ptr); asmlinkage int __raw_atomic_add_asm(volatile int *ptr, int value); +asmlinkage int __raw_atomic_xadd_asm(volatile int *ptr, int value); asmlinkage int __raw_atomic_and_asm(volatile int *ptr, int value); asmlinkage int __raw_atomic_or_asm(volatile int *ptr, int value); @@ -28,10 +29,17 @@ asmlinkage int __raw_atomic_test_asm(const volatile int *ptr, int value); #define atomic_add_return(i, v) __raw_atomic_add_asm(&(v)->counter, i) #define atomic_sub_return(i, v) __raw_atomic_add_asm(&(v)->counter, -(i)) +#define atomic_fetch_add(i, v) __raw_atomic_xadd_asm(&(v)->counter, i) +#define atomic_fetch_sub(i, v) __raw_atomic_xadd_asm(&(v)->counter, -(i)) + #define atomic_or(i, v) (void)__raw_atomic_or_asm(&(v)->counter, i) #define atomic_and(i, v) (void)__raw_atomic_and_asm(&(v)->counter, i) #define atomic_xor(i, v) (void)__raw_atomic_xor_asm(&(v)->counter, i) +#define atomic_fetch_or(i, v) __raw_atomic_or_asm(&(v)->counter, i) +#define atomic_fetch_and(i, v) __raw_atomic_and_asm(&(v)->counter, i) +#define atomic_fetch_xor(i, v) __raw_atomic_xor_asm(&(v)->counter, i) + #endif #include diff --git a/arch/blackfin/include/asm/spinlock.h b/arch/blackfin/include/asm/spinlock.h index 490c7caa02d9..c58f4a83ed6f 100644 --- a/arch/blackfin/include/asm/spinlock.h +++ b/arch/blackfin/include/asm/spinlock.h @@ -12,6 +12,8 @@ #else #include +#include +#include asmlinkage int __raw_spin_is_locked_asm(volatile int *ptr); asmlinkage void __raw_spin_lock_asm(volatile int *ptr); @@ -48,8 +50,7 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) { - while (arch_spin_is_locked(lock)) - cpu_relax(); + smp_cond_load_acquire(&lock->lock, !VAL); } static inline int arch_read_can_lock(arch_rwlock_t *rw) diff --git a/arch/blackfin/kernel/bfin_ksyms.c b/arch/blackfin/kernel/bfin_ksyms.c index a401c27b69b4..68096e8f787f 100644 --- a/arch/blackfin/kernel/bfin_ksyms.c +++ b/arch/blackfin/kernel/bfin_ksyms.c @@ -84,6 +84,7 @@ EXPORT_SYMBOL(insl_16); #ifdef CONFIG_SMP EXPORT_SYMBOL(__raw_atomic_add_asm); +EXPORT_SYMBOL(__raw_atomic_xadd_asm); EXPORT_SYMBOL(__raw_atomic_and_asm); EXPORT_SYMBOL(__raw_atomic_or_asm); EXPORT_SYMBOL(__raw_atomic_xor_asm); diff --git a/arch/blackfin/mach-bf561/atomic.S b/arch/blackfin/mach-bf561/atomic.S index 26fccb5568b9..1e2989c5d6b2 100644 --- a/arch/blackfin/mach-bf561/atomic.S +++ b/arch/blackfin/mach-bf561/atomic.S @@ -605,6 +605,28 @@ ENTRY(___raw_atomic_add_asm) rts; ENDPROC(___raw_atomic_add_asm) +/* + * r0 = ptr + * r1 = value + * + * ADD a signed value to a 32bit word and return the old value atomically. + * Clobbers: r3:0, p1:0 + */ +ENTRY(___raw_atomic_xadd_asm) + p1 = r0; + r3 = r1; + [--sp] = rets; + call _get_core_lock; + r3 = [p1]; + r2 = r3 + r2; + [p1] = r2; + r1 = p1; + call _put_core_lock; + r0 = r3; + rets = [sp++]; + rts; +ENDPROC(___raw_atomic_add_asm) + /* * r0 = ptr * r1 = mask @@ -618,10 +640,9 @@ ENTRY(___raw_atomic_and_asm) r3 = r1; [--sp] = rets; call _get_core_lock; - r2 = [p1]; - r3 = r2 & r3; - [p1] = r3; - r3 = r2; + r3 = [p1]; + r2 = r2 & r3; + [p1] = r2; r1 = p1; call _put_core_lock; r0 = r3; @@ -642,10 +663,9 @@ ENTRY(___raw_atomic_or_asm) r3 = r1; [--sp] = rets; call _get_core_lock; - r2 = [p1]; - r3 = r2 | r3; - [p1] = r3; - r3 = r2; + r3 = [p1]; + r2 = r2 | r3; + [p1] = r2; r1 = p1; call _put_core_lock; r0 = r3; @@ -666,10 +686,9 @@ ENTRY(___raw_atomic_xor_asm) r3 = r1; [--sp] = rets; call _get_core_lock; - r2 = [p1]; - r3 = r2 ^ r3; - [p1] = r3; - r3 = r2; + r3 = [p1]; + r2 = r2 ^ r3; + [p1] = r2; r1 = p1; call _put_core_lock; r0 = r3; diff --git a/arch/blackfin/mach-bf609/boards/ezkit.c b/arch/blackfin/mach-bf609/boards/ezkit.c index aad5d7416886..9231e5a72b93 100644 --- a/arch/blackfin/mach-bf609/boards/ezkit.c +++ b/arch/blackfin/mach-bf609/boards/ezkit.c @@ -1002,14 +1002,12 @@ static struct adv7842_output_format adv7842_opf[] = { { .op_ch_sel = ADV7842_OP_CH_SEL_BRG, .op_format_sel = ADV7842_OP_FORMAT_SEL_SDR_ITU656_8, - .op_656_range = 1, .blank_data = 1, .insert_av_codes = 1, }, { .op_ch_sel = ADV7842_OP_CH_SEL_RGB, .op_format_sel = ADV7842_OP_FORMAT_SEL_SDR_ITU656_16, - .op_656_range = 1, .blank_data = 1, }, }; diff --git a/arch/cris/include/asm/pgalloc.h b/arch/cris/include/asm/pgalloc.h index 235ece437ddd..42f1affb9c2d 100644 --- a/arch/cris/include/asm/pgalloc.h +++ b/arch/cris/include/asm/pgalloc.h @@ -24,14 +24,14 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); return pte; } static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *pte; - pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0); + pte = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0); if (!pte) return NULL; if (!pgtable_page_ctor(pte)) { diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c index 3066d40a6db1..112ef26c7f2e 100644 --- a/arch/cris/mm/fault.c +++ b/arch/cris/mm/fault.c @@ -168,7 +168,7 @@ retry: * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/frv/include/asm/atomic.h b/arch/frv/include/asm/atomic.h index 64f02d451aa8..1c2a5e264fc7 100644 --- a/arch/frv/include/asm/atomic.h +++ b/arch/frv/include/asm/atomic.h @@ -60,16 +60,6 @@ static inline int atomic_add_negative(int i, atomic_t *v) return atomic_add_return(i, v) < 0; } -static inline void atomic_add(int i, atomic_t *v) -{ - atomic_add_return(i, v); -} - -static inline void atomic_sub(int i, atomic_t *v) -{ - atomic_sub_return(i, v); -} - static inline void atomic_inc(atomic_t *v) { atomic_inc_return(v); @@ -136,16 +126,6 @@ static inline long long atomic64_add_negative(long long i, atomic64_t *v) return atomic64_add_return(i, v) < 0; } -static inline void atomic64_add(long long i, atomic64_t *v) -{ - atomic64_add_return(i, v); -} - -static inline void atomic64_sub(long long i, atomic64_t *v) -{ - atomic64_sub_return(i, v); -} - static inline void atomic64_inc(atomic64_t *v) { atomic64_inc_return(v); @@ -182,11 +162,19 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) } #define ATOMIC_OP(op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + return __atomic32_fetch_##op(i, &v->counter); \ +} \ static inline void atomic_##op(int i, atomic_t *v) \ { \ (void)__atomic32_fetch_##op(i, &v->counter); \ } \ \ +static inline long long atomic64_fetch_##op(long long i, atomic64_t *v) \ +{ \ + return __atomic64_fetch_##op(i, &v->counter); \ +} \ static inline void atomic64_##op(long long i, atomic64_t *v) \ { \ (void)__atomic64_fetch_##op(i, &v->counter); \ @@ -195,6 +183,8 @@ static inline void atomic64_##op(long long i, atomic64_t *v) \ ATOMIC_OP(or) ATOMIC_OP(and) ATOMIC_OP(xor) +ATOMIC_OP(add) +ATOMIC_OP(sub) #undef ATOMIC_OP diff --git a/arch/frv/include/asm/atomic_defs.h b/arch/frv/include/asm/atomic_defs.h index 36e126d2f801..d4912c88b829 100644 --- a/arch/frv/include/asm/atomic_defs.h +++ b/arch/frv/include/asm/atomic_defs.h @@ -162,6 +162,8 @@ ATOMIC_EXPORT(__atomic64_fetch_##op); ATOMIC_FETCH_OP(or) ATOMIC_FETCH_OP(and) ATOMIC_FETCH_OP(xor) +ATOMIC_FETCH_OP(add) +ATOMIC_FETCH_OP(sub) ATOMIC_OP_RETURN(add) ATOMIC_OP_RETURN(sub) diff --git a/arch/frv/include/asm/serial.h b/arch/frv/include/asm/serial.h index bce0d0d07e60..614c6d76789a 100644 --- a/arch/frv/include/asm/serial.h +++ b/arch/frv/include/asm/serial.h @@ -12,7 +12,3 @@ * the base baud is derived from the clock speed and so is variable */ #define BASE_BAUD 0 - -#define STD_COM_FLAGS UPF_BOOT_AUTOCONF - -#define SERIAL_PORT_DFNS diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c index 61d99767fe16..614a46c413d2 100644 --- a/arch/frv/mm/fault.c +++ b/arch/frv/mm/fault.c @@ -164,7 +164,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, ear0, flags); + fault = handle_mm_fault(vma, ear0, flags); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/frv/mm/pgalloc.c b/arch/frv/mm/pgalloc.c index 41907d25ed38..c9ed14f6c67d 100644 --- a/arch/frv/mm/pgalloc.c +++ b/arch/frv/mm/pgalloc.c @@ -22,7 +22,7 @@ pgd_t swapper_pg_dir[PTRS_PER_PGD] __attribute__((aligned(PAGE_SIZE))); pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); + pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL); if (pte) clear_page(pte); return pte; @@ -33,9 +33,9 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) struct page *page; #ifdef CONFIG_HIGHPTE - page = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM|__GFP_REPEAT, 0); + page = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM, 0); #else - page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0); + page = alloc_pages(GFP_KERNEL, 0); #endif if (!page) return NULL; diff --git a/arch/h8300/include/asm/atomic.h b/arch/h8300/include/asm/atomic.h index 4435a445ae7e..349a47a918db 100644 --- a/arch/h8300/include/asm/atomic.h +++ b/arch/h8300/include/asm/atomic.h @@ -28,6 +28,19 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return ret; \ } +#define ATOMIC_FETCH_OP(op, c_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + h8300flags flags; \ + int ret; \ + \ + flags = arch_local_irq_save(); \ + ret = v->counter; \ + v->counter c_op i; \ + arch_local_irq_restore(flags); \ + return ret; \ +} + #define ATOMIC_OP(op, c_op) \ static inline void atomic_##op(int i, atomic_t *v) \ { \ @@ -41,17 +54,21 @@ static inline void atomic_##op(int i, atomic_t *v) \ ATOMIC_OP_RETURN(add, +=) ATOMIC_OP_RETURN(sub, -=) -ATOMIC_OP(and, &=) -ATOMIC_OP(or, |=) -ATOMIC_OP(xor, ^=) +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) + +ATOMIC_OPS(and, &=) +ATOMIC_OPS(or, |=) +ATOMIC_OPS(xor, ^=) +ATOMIC_OPS(add, +=) +ATOMIC_OPS(sub, -=) +#undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic_add(i, v) (void)atomic_add_return(i, v) #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) - -#define atomic_sub(i, v) (void)atomic_sub_return(i, v) #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) #define atomic_inc_return(v) atomic_add_return(1, v) diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index 57298e7b4867..1941e4baaee6 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -8,8 +8,7 @@ config HEXAGON # select HAVE_REGS_AND_STACK_ACCESS_API # select HAVE_HW_BREAKPOINT if PERF_EVENTS # select ARCH_HAS_CPU_IDLE_WAIT - # select ARCH_WANT_OPTIONAL_GPIOLIB - # select ARCH_REQUIRE_GPIOLIB + # select GPIOLIB # select HAVE_CLK # select GENERIC_PENDING_IRQ if SMP select GENERIC_ATOMIC64 diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h index 55696c4100d4..a62ba368b27d 100644 --- a/arch/hexagon/include/asm/atomic.h +++ b/arch/hexagon/include/asm/atomic.h @@ -110,7 +110,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ ); \ } \ -#define ATOMIC_OP_RETURN(op) \ +#define ATOMIC_OP_RETURN(op) \ static inline int atomic_##op##_return(int i, atomic_t *v) \ { \ int output; \ @@ -127,16 +127,37 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return output; \ } -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int output, val; \ + \ + __asm__ __volatile__ ( \ + "1: %0 = memw_locked(%2);\n" \ + " %1 = "#op "(%0,%3);\n" \ + " memw_locked(%2,P3)=%1;\n" \ + " if !P3 jump 1b;\n" \ + : "=&r" (output), "=&r" (val) \ + : "r" (&v->counter), "r" (i) \ + : "memory", "p3" \ + ); \ + return output; \ +} + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/hexagon/include/asm/pgalloc.h b/arch/hexagon/include/asm/pgalloc.h index 77da3b0ae3c2..eeebf862c46c 100644 --- a/arch/hexagon/include/asm/pgalloc.h +++ b/arch/hexagon/include/asm/pgalloc.h @@ -64,7 +64,7 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, { struct page *pte; - pte = alloc_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); + pte = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!pte) return NULL; if (!pgtable_page_ctor(pte)) { @@ -78,7 +78,7 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - gfp_t flags = GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO; + gfp_t flags = GFP_KERNEL | __GFP_ZERO; return (pte_t *) __get_free_page(flags); } diff --git a/arch/hexagon/include/asm/spinlock.h b/arch/hexagon/include/asm/spinlock.h index 12ca4ebc0338..a1c55788c5d6 100644 --- a/arch/hexagon/include/asm/spinlock.h +++ b/arch/hexagon/include/asm/spinlock.h @@ -23,6 +23,8 @@ #define _ASM_SPINLOCK_H #include +#include +#include /* * This file is pulled in for SMP builds. @@ -176,8 +178,12 @@ static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock) * SMP spinlocks are intended to allow only a single CPU at the lock */ #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) -#define arch_spin_unlock_wait(lock) \ - do {while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->lock, !VAL); +} + #define arch_spin_is_locked(x) ((x)->lock != 0) #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) diff --git a/arch/hexagon/mm/vm_fault.c b/arch/hexagon/mm/vm_fault.c index 8704c9320032..bd7c251e2bce 100644 --- a/arch/hexagon/mm/vm_fault.c +++ b/arch/hexagon/mm/vm_fault.c @@ -101,7 +101,7 @@ good_area: break; } - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index f80758cb7157..6a15083cc366 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -39,13 +39,12 @@ config IA64 select GENERIC_PENDING_IRQ if SMP select GENERIC_IRQ_SHOW select GENERIC_IRQ_LEGACY - select ARCH_WANT_OPTIONAL_GPIOLIB select ARCH_HAVE_NMI_SAFE_CMPXCHG select GENERIC_IOMAP select GENERIC_SMP_IDLE_THREAD select ARCH_INIT_TASK select ARCH_TASK_STRUCT_ALLOCATOR - select ARCH_THREAD_INFO_ALLOCATOR + select ARCH_THREAD_STACK_ALLOCATOR select ARCH_CLOCKSOURCE_DATA select GENERIC_TIME_VSYSCALL_OLD select SYSCTL_ARCH_UNALIGN_NO_WARN diff --git a/arch/ia64/include/asm/acpi.h b/arch/ia64/include/asm/acpi.h index aa0fdf125aba..a3d0211970e9 100644 --- a/arch/ia64/include/asm/acpi.h +++ b/arch/ia64/include/asm/acpi.h @@ -140,6 +140,9 @@ static inline void per_cpu_scan_finalize(int min_cpus, int reserve_cpus) } } } + +extern void acpi_numa_fixup(void); + #endif /* CONFIG_ACPI_NUMA */ #endif /*__KERNEL__*/ diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h index 8dfb5f6f6c35..f565ad376142 100644 --- a/arch/ia64/include/asm/atomic.h +++ b/arch/ia64/include/asm/atomic.h @@ -42,8 +42,27 @@ ia64_atomic_##op (int i, atomic_t *v) \ return new; \ } -ATOMIC_OP(add, +) -ATOMIC_OP(sub, -) +#define ATOMIC_FETCH_OP(op, c_op) \ +static __inline__ int \ +ia64_atomic_fetch_##op (int i, atomic_t *v) \ +{ \ + __s32 old, new; \ + CMPXCHG_BUGCHECK_DECL \ + \ + do { \ + CMPXCHG_BUGCHECK(v); \ + old = atomic_read(v); \ + new = old c_op i; \ + } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic_t)) != old); \ + return old; \ +} + +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) + +ATOMIC_OPS(add, +) +ATOMIC_OPS(sub, -) #define atomic_add_return(i,v) \ ({ \ @@ -69,14 +88,44 @@ ATOMIC_OP(sub, -) : ia64_atomic_sub(__ia64_asr_i, v); \ }) -ATOMIC_OP(and, &) -ATOMIC_OP(or, |) -ATOMIC_OP(xor, ^) +#define atomic_fetch_add(i,v) \ +({ \ + int __ia64_aar_i = (i); \ + (__builtin_constant_p(i) \ + && ( (__ia64_aar_i == 1) || (__ia64_aar_i == 4) \ + || (__ia64_aar_i == 8) || (__ia64_aar_i == 16) \ + || (__ia64_aar_i == -1) || (__ia64_aar_i == -4) \ + || (__ia64_aar_i == -8) || (__ia64_aar_i == -16))) \ + ? ia64_fetchadd(__ia64_aar_i, &(v)->counter, acq) \ + : ia64_atomic_fetch_add(__ia64_aar_i, v); \ +}) + +#define atomic_fetch_sub(i,v) \ +({ \ + int __ia64_asr_i = (i); \ + (__builtin_constant_p(i) \ + && ( (__ia64_asr_i == 1) || (__ia64_asr_i == 4) \ + || (__ia64_asr_i == 8) || (__ia64_asr_i == 16) \ + || (__ia64_asr_i == -1) || (__ia64_asr_i == -4) \ + || (__ia64_asr_i == -8) || (__ia64_asr_i == -16))) \ + ? ia64_fetchadd(-__ia64_asr_i, &(v)->counter, acq) \ + : ia64_atomic_fetch_sub(__ia64_asr_i, v); \ +}) + +ATOMIC_FETCH_OP(and, &) +ATOMIC_FETCH_OP(or, |) +ATOMIC_FETCH_OP(xor, ^) + +#define atomic_and(i,v) (void)ia64_atomic_fetch_and(i,v) +#define atomic_or(i,v) (void)ia64_atomic_fetch_or(i,v) +#define atomic_xor(i,v) (void)ia64_atomic_fetch_xor(i,v) -#define atomic_and(i,v) (void)ia64_atomic_and(i,v) -#define atomic_or(i,v) (void)ia64_atomic_or(i,v) -#define atomic_xor(i,v) (void)ia64_atomic_xor(i,v) +#define atomic_fetch_and(i,v) ia64_atomic_fetch_and(i,v) +#define atomic_fetch_or(i,v) ia64_atomic_fetch_or(i,v) +#define atomic_fetch_xor(i,v) ia64_atomic_fetch_xor(i,v) +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP #define ATOMIC64_OP(op, c_op) \ @@ -94,8 +143,27 @@ ia64_atomic64_##op (__s64 i, atomic64_t *v) \ return new; \ } -ATOMIC64_OP(add, +) -ATOMIC64_OP(sub, -) +#define ATOMIC64_FETCH_OP(op, c_op) \ +static __inline__ long \ +ia64_atomic64_fetch_##op (__s64 i, atomic64_t *v) \ +{ \ + __s64 old, new; \ + CMPXCHG_BUGCHECK_DECL \ + \ + do { \ + CMPXCHG_BUGCHECK(v); \ + old = atomic64_read(v); \ + new = old c_op i; \ + } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic64_t)) != old); \ + return old; \ +} + +#define ATOMIC64_OPS(op, c_op) \ + ATOMIC64_OP(op, c_op) \ + ATOMIC64_FETCH_OP(op, c_op) + +ATOMIC64_OPS(add, +) +ATOMIC64_OPS(sub, -) #define atomic64_add_return(i,v) \ ({ \ @@ -121,14 +189,44 @@ ATOMIC64_OP(sub, -) : ia64_atomic64_sub(__ia64_asr_i, v); \ }) -ATOMIC64_OP(and, &) -ATOMIC64_OP(or, |) -ATOMIC64_OP(xor, ^) +#define atomic64_fetch_add(i,v) \ +({ \ + long __ia64_aar_i = (i); \ + (__builtin_constant_p(i) \ + && ( (__ia64_aar_i == 1) || (__ia64_aar_i == 4) \ + || (__ia64_aar_i == 8) || (__ia64_aar_i == 16) \ + || (__ia64_aar_i == -1) || (__ia64_aar_i == -4) \ + || (__ia64_aar_i == -8) || (__ia64_aar_i == -16))) \ + ? ia64_fetchadd(__ia64_aar_i, &(v)->counter, acq) \ + : ia64_atomic64_fetch_add(__ia64_aar_i, v); \ +}) + +#define atomic64_fetch_sub(i,v) \ +({ \ + long __ia64_asr_i = (i); \ + (__builtin_constant_p(i) \ + && ( (__ia64_asr_i == 1) || (__ia64_asr_i == 4) \ + || (__ia64_asr_i == 8) || (__ia64_asr_i == 16) \ + || (__ia64_asr_i == -1) || (__ia64_asr_i == -4) \ + || (__ia64_asr_i == -8) || (__ia64_asr_i == -16))) \ + ? ia64_fetchadd(-__ia64_asr_i, &(v)->counter, acq) \ + : ia64_atomic64_fetch_sub(__ia64_asr_i, v); \ +}) + +ATOMIC64_FETCH_OP(and, &) +ATOMIC64_FETCH_OP(or, |) +ATOMIC64_FETCH_OP(xor, ^) + +#define atomic64_and(i,v) (void)ia64_atomic64_fetch_and(i,v) +#define atomic64_or(i,v) (void)ia64_atomic64_fetch_or(i,v) +#define atomic64_xor(i,v) (void)ia64_atomic64_fetch_xor(i,v) -#define atomic64_and(i,v) (void)ia64_atomic64_and(i,v) -#define atomic64_or(i,v) (void)ia64_atomic64_or(i,v) -#define atomic64_xor(i,v) (void)ia64_atomic64_xor(i,v) +#define atomic64_fetch_and(i,v) ia64_atomic64_fetch_and(i,v) +#define atomic64_fetch_or(i,v) ia64_atomic64_fetch_or(i,v) +#define atomic64_fetch_xor(i,v) ia64_atomic64_fetch_xor(i,v) +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) diff --git a/arch/ia64/include/asm/mutex.h b/arch/ia64/include/asm/mutex.h index f41e66d65e31..28cb819e0ff9 100644 --- a/arch/ia64/include/asm/mutex.h +++ b/arch/ia64/include/asm/mutex.h @@ -82,7 +82,7 @@ __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) static inline int __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) { - if (cmpxchg_acq(count, 1, 0) == 1) + if (atomic_read(count) == 1 && cmpxchg_acq(count, 1, 0) == 1) return 1; return 0; } diff --git a/arch/ia64/include/asm/rwsem.h b/arch/ia64/include/asm/rwsem.h index 8b23e070b844..8fa98dd303b4 100644 --- a/arch/ia64/include/asm/rwsem.h +++ b/arch/ia64/include/asm/rwsem.h @@ -40,7 +40,7 @@ static inline void __down_read (struct rw_semaphore *sem) { - long result = ia64_fetchadd8_acq((unsigned long *)&sem->count, 1); + long result = ia64_fetchadd8_acq((unsigned long *)&sem->count.counter, 1); if (result < 0) rwsem_down_read_failed(sem); @@ -55,9 +55,9 @@ ___down_write (struct rw_semaphore *sem) long old, new; do { - old = sem->count; + old = atomic_long_read(&sem->count); new = old + RWSEM_ACTIVE_WRITE_BIAS; - } while (cmpxchg_acq(&sem->count, old, new) != old); + } while (atomic_long_cmpxchg_acquire(&sem->count, old, new) != old); return old; } @@ -85,7 +85,7 @@ __down_write_killable (struct rw_semaphore *sem) static inline void __up_read (struct rw_semaphore *sem) { - long result = ia64_fetchadd8_rel((unsigned long *)&sem->count, -1); + long result = ia64_fetchadd8_rel((unsigned long *)&sem->count.counter, -1); if (result < 0 && (--result & RWSEM_ACTIVE_MASK) == 0) rwsem_wake(sem); @@ -100,9 +100,9 @@ __up_write (struct rw_semaphore *sem) long old, new; do { - old = sem->count; + old = atomic_long_read(&sem->count); new = old - RWSEM_ACTIVE_WRITE_BIAS; - } while (cmpxchg_rel(&sem->count, old, new) != old); + } while (atomic_long_cmpxchg_release(&sem->count, old, new) != old); if (new < 0 && (new & RWSEM_ACTIVE_MASK) == 0) rwsem_wake(sem); @@ -115,8 +115,8 @@ static inline int __down_read_trylock (struct rw_semaphore *sem) { long tmp; - while ((tmp = sem->count) >= 0) { - if (tmp == cmpxchg_acq(&sem->count, tmp, tmp+1)) { + while ((tmp = atomic_long_read(&sem->count)) >= 0) { + if (tmp == atomic_long_cmpxchg_acquire(&sem->count, tmp, tmp+1)) { return 1; } } @@ -129,8 +129,8 @@ __down_read_trylock (struct rw_semaphore *sem) static inline int __down_write_trylock (struct rw_semaphore *sem) { - long tmp = cmpxchg_acq(&sem->count, RWSEM_UNLOCKED_VALUE, - RWSEM_ACTIVE_WRITE_BIAS); + long tmp = atomic_long_cmpxchg_acquire(&sem->count, + RWSEM_UNLOCKED_VALUE, RWSEM_ACTIVE_WRITE_BIAS); return tmp == RWSEM_UNLOCKED_VALUE; } @@ -143,19 +143,12 @@ __downgrade_write (struct rw_semaphore *sem) long old, new; do { - old = sem->count; + old = atomic_long_read(&sem->count); new = old - RWSEM_WAITING_BIAS; - } while (cmpxchg_rel(&sem->count, old, new) != old); + } while (atomic_long_cmpxchg_release(&sem->count, old, new) != old); if (old < 0) rwsem_downgrade_wake(sem); } -/* - * Implement atomic add functionality. These used to be "inline" functions, but GCC v3.1 - * doesn't quite optimize this stuff right and ends up with bad calls to fetchandadd. - */ -#define rwsem_atomic_add(delta, sem) atomic64_add(delta, (atomic64_t *)(&(sem)->count)) -#define rwsem_atomic_update(delta, sem) atomic64_add_return(delta, (atomic64_t *)(&(sem)->count)) - #endif /* _ASM_IA64_RWSEM_H */ diff --git a/arch/ia64/include/asm/spinlock.h b/arch/ia64/include/asm/spinlock.h index 45698cd15b7b..ca9e76149a4a 100644 --- a/arch/ia64/include/asm/spinlock.h +++ b/arch/ia64/include/asm/spinlock.h @@ -15,6 +15,8 @@ #include #include +#include +#include #define arch_spin_lock_init(x) ((x)->lock = 0) @@ -86,6 +88,8 @@ static __always_inline void __ticket_spin_unlock_wait(arch_spinlock_t *lock) return; cpu_relax(); } + + smp_acquire__after_ctrl_dep(); } static inline int __ticket_spin_is_locked(arch_spinlock_t *lock) diff --git a/arch/ia64/include/asm/thread_info.h b/arch/ia64/include/asm/thread_info.h index aa995b67c3f5..d1212b84fb83 100644 --- a/arch/ia64/include/asm/thread_info.h +++ b/arch/ia64/include/asm/thread_info.h @@ -48,15 +48,15 @@ struct thread_info { #ifndef ASM_OFFSETS_C /* how to get the thread information struct from C */ #define current_thread_info() ((struct thread_info *) ((char *) current + IA64_TASK_SIZE)) -#define alloc_thread_info_node(tsk, node) \ - ((struct thread_info *) ((char *) (tsk) + IA64_TASK_SIZE)) +#define alloc_thread_stack_node(tsk, node) \ + ((unsigned long *) ((char *) (tsk) + IA64_TASK_SIZE)) #define task_thread_info(tsk) ((struct thread_info *) ((char *) (tsk) + IA64_TASK_SIZE)) #else #define current_thread_info() ((struct thread_info *) 0) -#define alloc_thread_info_node(tsk, node) ((struct thread_info *) 0) +#define alloc_thread_stack_node(tsk, node) ((unsigned long *) 0) #define task_thread_info(tsk) ((struct thread_info *) 0) #endif -#define free_thread_info(ti) /* nothing */ +#define free_thread_stack(ti) /* nothing */ #define task_stack_page(tsk) ((void *)(tsk)) #define __HAVE_THREAD_FUNCTIONS diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h index 39d64e0df1de..77e541cf0e5d 100644 --- a/arch/ia64/include/asm/tlb.h +++ b/arch/ia64/include/asm/tlb.h @@ -205,17 +205,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) * must be delayed until after the TLB has been flushed (see comments at the beginning of * this file). */ -static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) +static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page) { + if (tlb->nr == tlb->max) + return true; + tlb->need_flush = 1; if (!tlb->nr && tlb->pages == tlb->local) __tlb_alloc_page(tlb); tlb->pages[tlb->nr++] = page; - VM_BUG_ON(tlb->nr > tlb->max); - - return tlb->max - tlb->nr; + return false; } static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) @@ -235,8 +236,28 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) { - if (!__tlb_remove_page(tlb, page)) + if (__tlb_remove_page(tlb, page)) { tlb_flush_mmu(tlb); + __tlb_remove_page(tlb, page); + } +} + +static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return __tlb_remove_page(tlb, page); +} + +static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, + struct page *page) +{ + return __tlb_remove_page(tlb, page); +} + +static inline void tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return tlb_remove_page(tlb, page); } /* diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c index b1698bc042c8..92b7bc956795 100644 --- a/arch/ia64/kernel/acpi.c +++ b/arch/ia64/kernel/acpi.c @@ -524,7 +524,7 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) return 0; } -void __init acpi_numa_arch_fixup(void) +void __init acpi_numa_fixup(void) { int i, j, node_from, node_to; diff --git a/arch/ia64/kernel/init_task.c b/arch/ia64/kernel/init_task.c index f9efe9739d3f..0eaa89f3defd 100644 --- a/arch/ia64/kernel/init_task.c +++ b/arch/ia64/kernel/init_task.c @@ -26,6 +26,7 @@ static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand); * handled. This is done by having a special ".data..init_task" section... */ #define init_thread_info init_task_mem.s.thread_info +#define init_stack init_task_mem.stack union { struct { diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index 2029a38a72ae..afddb3e80a29 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -552,6 +552,7 @@ setup_arch (char **cmdline_p) early_acpi_boot_init(); # ifdef CONFIG_ACPI_NUMA acpi_numa_init(); + acpi_numa_fixup(); # ifdef CONFIG_ACPI_HOTPLUG_CPU prefill_possible_map(); # endif diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 70b40d1205a6..fa6ad95e992e 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -159,7 +159,7 @@ retry: * sure we exit gracefully rather than endlessly redo the * fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/m32r/boot/compressed/m32r_sio.c b/arch/m32r/boot/compressed/m32r_sio.c index 01d877c6868f..cf3023dced49 100644 --- a/arch/m32r/boot/compressed/m32r_sio.c +++ b/arch/m32r/boot/compressed/m32r_sio.c @@ -8,12 +8,13 @@ #include -static void putc(char c); +static void m32r_putc(char c); static int puts(const char *s) { char c; - while ((c = *s++)) putc(c); + while ((c = *s++)) + m32r_putc(c); return 0; } @@ -41,7 +42,7 @@ static int puts(const char *s) #define BOOT_SIO0TXB PLD_ESIO0TXB #endif -static void putc(char c) +static void m32r_putc(char c) { while ((*BOOT_SIO0STS & 0x3) != 0x3) cpu_relax(); @@ -61,7 +62,7 @@ static void putc(char c) #define SIO0TXB (volatile unsigned short *)(0x00efd000 + 30) #endif -static void putc(char c) +static void m32r_putc(char c) { while ((*SIO0STS & 0x1) == 0) cpu_relax(); diff --git a/arch/m32r/include/asm/atomic.h b/arch/m32r/include/asm/atomic.h index ea35160d632b..640cc1c7099f 100644 --- a/arch/m32r/include/asm/atomic.h +++ b/arch/m32r/include/asm/atomic.h @@ -89,16 +89,44 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ return result; \ } -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +static __inline__ int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned long flags; \ + int result, val; \ + \ + local_irq_save(flags); \ + __asm__ __volatile__ ( \ + "# atomic_fetch_" #op " \n\t" \ + DCACHE_CLEAR("%0", "r4", "%2") \ + M32R_LOCK" %1, @%2; \n\t" \ + "mv %0, %1 \n\t" \ + #op " %1, %3; \n\t" \ + M32R_UNLOCK" %1, @%2; \n\t" \ + : "=&r" (result), "=&r" (val) \ + : "r" (&v->counter), "r" (i) \ + : "memory" \ + __ATOMIC_CLOBBER \ + ); \ + local_irq_restore(flags); \ + \ + return result; \ +} + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/m32r/include/asm/spinlock.h b/arch/m32r/include/asm/spinlock.h index fa13694eaae3..323c7fc953cd 100644 --- a/arch/m32r/include/asm/spinlock.h +++ b/arch/m32r/include/asm/spinlock.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include /* * Your basic SMP spinlocks, allowing only a single CPU anywhere @@ -27,8 +29,11 @@ #define arch_spin_is_locked(x) (*(volatile int *)(&(x)->slock) <= 0) #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) -#define arch_spin_unlock_wait(x) \ - do { cpu_relax(); } while (arch_spin_is_locked(x)) + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->slock, VAL > 0); +} /** * arch_spin_trylock - Try spin lock and return a result diff --git a/arch/m32r/kernel/m32r_ksyms.c b/arch/m32r/kernel/m32r_ksyms.c index b727e693c805..23f26f4adfff 100644 --- a/arch/m32r/kernel/m32r_ksyms.c +++ b/arch/m32r/kernel/m32r_ksyms.c @@ -41,6 +41,9 @@ EXPORT_SYMBOL(cpu_data); EXPORT_SYMBOL(smp_flush_tlb_page); #endif +extern int __ucmpdi2(unsigned long long a, unsigned long long b); +EXPORT_SYMBOL(__ucmpdi2); + /* compiler generated symbol */ extern void __ashldi3(void); extern void __ashrdi3(void); diff --git a/arch/m32r/lib/Makefile b/arch/m32r/lib/Makefile index d16b4e40d1ae..5889eb9610b5 100644 --- a/arch/m32r/lib/Makefile +++ b/arch/m32r/lib/Makefile @@ -3,5 +3,5 @@ # lib-y := checksum.o ashxdi3.o memset.o memcpy.o \ - delay.o strlen.o usercopy.o csum_partial_copy.o - + delay.o strlen.o usercopy.o csum_partial_copy.o \ + ucmpdi2.o diff --git a/arch/m32r/lib/libgcc.h b/arch/m32r/lib/libgcc.h new file mode 100644 index 000000000000..267aa435bc35 --- /dev/null +++ b/arch/m32r/lib/libgcc.h @@ -0,0 +1,23 @@ +#ifndef __ASM_LIBGCC_H +#define __ASM_LIBGCC_H + +#include + +#ifdef __BIG_ENDIAN +struct DWstruct { + int high, low; +}; +#elif defined(__LITTLE_ENDIAN) +struct DWstruct { + int low, high; +}; +#else +#error I feel sick. +#endif + +typedef union { + struct DWstruct s; + long long ll; +} DWunion; + +#endif /* __ASM_LIBGCC_H */ diff --git a/arch/m32r/lib/ucmpdi2.c b/arch/m32r/lib/ucmpdi2.c new file mode 100644 index 000000000000..9d3c682c89b5 --- /dev/null +++ b/arch/m32r/lib/ucmpdi2.c @@ -0,0 +1,17 @@ +#include "libgcc.h" + +int __ucmpdi2(unsigned long long a, unsigned long long b) +{ + const DWunion au = {.ll = a}; + const DWunion bu = {.ll = b}; + + if ((unsigned int)au.s.high < (unsigned int)bu.s.high) + return 0; + else if ((unsigned int)au.s.high > (unsigned int)bu.s.high) + return 2; + if ((unsigned int)au.s.low < (unsigned int)bu.s.low) + return 0; + else if ((unsigned int)au.s.low > (unsigned int)bu.s.low) + return 2; + return 1; +} diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c index 8f9875b7933d..a3785d3644c2 100644 --- a/arch/m32r/mm/fault.c +++ b/arch/m32r/mm/fault.c @@ -196,7 +196,7 @@ good_area: */ addr = (address & PAGE_MASK); set_thread_fault_code(error_code); - fault = handle_mm_fault(mm, vma, addr, flags); + fault = handle_mm_fault(vma, addr, flags); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/m68k/coldfire/head.S b/arch/m68k/coldfire/head.S index fa31be297b85..73d92ea0ce65 100644 --- a/arch/m68k/coldfire/head.S +++ b/arch/m68k/coldfire/head.S @@ -288,7 +288,7 @@ _clear_bss: #endif /* - * Assember start up done, start code proper. + * Assembler start up done, start code proper. */ jsr start_kernel /* start Linux kernel */ diff --git a/arch/m68k/coldfire/m5272.c b/arch/m68k/coldfire/m5272.c index c525e4c08f84..9abb1a441da0 100644 --- a/arch/m68k/coldfire/m5272.c +++ b/arch/m68k/coldfire/m5272.c @@ -111,7 +111,7 @@ void __init config_BSP(char *commandp, int size) /***************************************************************************/ /* - * Some 5272 based boards have the FEC ethernet diectly connected to + * Some 5272 based boards have the FEC ethernet directly connected to * an ethernet switch. In this case we need to use the fixed phy type, * and we need to declare it early in boot. */ diff --git a/arch/m68k/coldfire/pci.c b/arch/m68k/coldfire/pci.c index 821de928dc3f..6a640be48568 100644 --- a/arch/m68k/coldfire/pci.c +++ b/arch/m68k/coldfire/pci.c @@ -42,7 +42,7 @@ static unsigned long iospace; /* * We need to be carefull probing on bus 0 (directly connected to host - * bridge). We should only acccess the well defined possible devices in + * bridge). We should only access the well defined possible devices in * use, ignore aliases and the like. */ static unsigned char mcf_host_slot2sid[32] = { diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 3ee6976f6088..8f5b6f7dd136 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -359,6 +360,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -553,7 +555,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index e96787ffcbce..31bded9c83d4 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -341,6 +342,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -512,7 +514,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 083fe6beac14..0d7739e04ae2 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -350,6 +351,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -533,7 +535,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 475130c06dcb..2cbb5c465fec 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -340,6 +341,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -504,7 +506,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 4339658c200f..96102a42c156 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -341,6 +342,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -514,7 +516,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 831cc8c3a2e2..97d88f7dc5a7 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -357,6 +358,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -536,7 +538,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 6377afeb522b..be25ef208f0f 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -390,6 +391,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -616,7 +618,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 4304b3d56262..a008344360c9 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -339,6 +340,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -504,7 +506,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 074bda4094ff..6735a25f36d4 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -340,6 +341,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -504,7 +506,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 07b9fa8d7f2e..780c6e9f6cf9 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -346,6 +347,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -527,7 +529,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 36e6fae02d45..44693cf361e5 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -337,6 +338,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -506,7 +508,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 903acf929511..ef0071d61158 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -9,6 +9,7 @@ CONFIG_LOG_BUF_SHIFT=16 # CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_USERFAULTFD=y CONFIG_SLAB=y CONFIG_MODULES=y @@ -337,6 +338,7 @@ CONFIG_MACVTAP=m CONFIG_IPVLAN=m CONFIG_VXLAN=m CONFIG_GENEVE=m +CONFIG_GTP=m CONFIG_MACSEC=m CONFIG_NETCONSOLE=m CONFIG_NETCONSOLE_DYNAMIC=y @@ -506,7 +508,9 @@ CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m +CONFIG_TEST_UUID=m CONFIG_TEST_RHASHTABLE=m +CONFIG_TEST_HASH=m CONFIG_TEST_LKM=m CONFIG_TEST_USER_COPY=m CONFIG_TEST_BPF=m diff --git a/arch/m68k/ifpsp060/src/fpsp.S b/arch/m68k/ifpsp060/src/fpsp.S index 78cb60f5bb4d..9bbffebe3eb5 100644 --- a/arch/m68k/ifpsp060/src/fpsp.S +++ b/arch/m68k/ifpsp060/src/fpsp.S @@ -10191,7 +10191,7 @@ xdnrm_con: xdnrm_sd: mov.l %a1,-(%sp) tst.b LOCAL_EX(%a0) # is denorm pos or neg? - smi.b %d1 # set d0 accodingly + smi.b %d1 # set d0 accordingly bsr.l unf_sub mov.l (%sp)+,%a1 xdnrm_exit: @@ -10990,7 +10990,7 @@ src_qnan_m: # routines where an instruction is selected by an index into # a large jump table corresponding to a given instruction which # has been decoded. Flow continues here where we now decode -# further accoding to the source operand type. +# further according to the source operand type. # global fsinh @@ -23196,14 +23196,14 @@ m_sign: # # 1. Branch on the sign of the adjusted exponent. # 2p.(positive exp) -# 2. Check M16 and the digits in lwords 2 and 3 in decending order. +# 2. Check M16 and the digits in lwords 2 and 3 in descending order. # 3. Add one for each zero encountered until a non-zero digit. # 4. Subtract the count from the exp. # 5. Check if the exp has crossed zero in #3 above; make the exp abs # and set SE. # 6. Multiply the mantissa by 10**count. # 2n.(negative exp) -# 2. Check the digits in lwords 3 and 2 in decending order. +# 2. Check the digits in lwords 3 and 2 in descending order. # 3. Add one for each zero encountered until a non-zero digit. # 4. Add the count to the exp. # 5. Check if the exp has crossed zero in #3 above; clear SE. diff --git a/arch/m68k/ifpsp060/src/pfpsp.S b/arch/m68k/ifpsp060/src/pfpsp.S index 4aedef973cf6..3535e6c87eec 100644 --- a/arch/m68k/ifpsp060/src/pfpsp.S +++ b/arch/m68k/ifpsp060/src/pfpsp.S @@ -13156,14 +13156,14 @@ m_sign: # # 1. Branch on the sign of the adjusted exponent. # 2p.(positive exp) -# 2. Check M16 and the digits in lwords 2 and 3 in decending order. +# 2. Check M16 and the digits in lwords 2 and 3 in descending order. # 3. Add one for each zero encountered until a non-zero digit. # 4. Subtract the count from the exp. # 5. Check if the exp has crossed zero in #3 above; make the exp abs # and set SE. # 6. Multiply the mantissa by 10**count. # 2n.(negative exp) -# 2. Check the digits in lwords 3 and 2 in decending order. +# 2. Check the digits in lwords 3 and 2 in descending order. # 3. Add one for each zero encountered until a non-zero digit. # 4. Add the count to the exp. # 5. Check if the exp has crossed zero in #3 above; clear SE. diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index 4858178260f9..cf4c3a7b1a45 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -53,6 +53,21 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return t; \ } +#define ATOMIC_FETCH_OP(op, c_op, asm_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int t, tmp; \ + \ + __asm__ __volatile__( \ + "1: movel %2,%1\n" \ + " " #asm_op "l %3,%1\n" \ + " casl %2,%1,%0\n" \ + " jne 1b" \ + : "+m" (*v), "=&d" (t), "=&d" (tmp) \ + : "g" (i), "2" (atomic_read(v))); \ + return tmp; \ +} + #else #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ @@ -68,20 +83,41 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ return t; \ } +#define ATOMIC_FETCH_OP(op, c_op, asm_op) \ +static inline int atomic_fetch_##op(int i, atomic_t * v) \ +{ \ + unsigned long flags; \ + int t; \ + \ + local_irq_save(flags); \ + t = v->counter; \ + v->counter c_op i; \ + local_irq_restore(flags); \ + \ + return t; \ +} + #endif /* CONFIG_RMW_INSNS */ #define ATOMIC_OPS(op, c_op, asm_op) \ ATOMIC_OP(op, c_op, asm_op) \ - ATOMIC_OP_RETURN(op, c_op, asm_op) + ATOMIC_OP_RETURN(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -ATOMIC_OP(and, &=, and) -ATOMIC_OP(or, |=, or) -ATOMIC_OP(xor, ^=, eor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op, asm_op) \ + ATOMIC_OP(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) + +ATOMIC_OPS(and, &=, and) +ATOMIC_OPS(or, |=, or) +ATOMIC_OPS(xor, ^=, eor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/m68k/include/asm/dma.h b/arch/m68k/include/asm/dma.h index 429fe26e320c..208b4daa14b3 100644 --- a/arch/m68k/include/asm/dma.h +++ b/arch/m68k/include/asm/dma.h @@ -18,7 +18,7 @@ * AUG/22/2000 : added support for 32-bit Dual-Address-Mode (K) 2000 * Oliver Kamphenkel (O.Kamphenkel@tu-bs.de) * - * AUG/25/2000 : addad support for 8, 16 and 32-bit Single-Address-Mode (K)2000 + * AUG/25/2000 : added support for 8, 16 and 32-bit Single-Address-Mode (K)2000 * Oliver Kamphenkel (O.Kamphenkel@tu-bs.de) * * APR/18/2002 : added proper support for MCF5272 DMA controller. diff --git a/arch/m68k/include/asm/m525xsim.h b/arch/m68k/include/asm/m525xsim.h index f186459072e9..699f20c8a0fe 100644 --- a/arch/m68k/include/asm/m525xsim.h +++ b/arch/m68k/include/asm/m525xsim.h @@ -123,10 +123,10 @@ /* * I2C module. */ -#define MCFI2C_BASE0 (MCF_MBAR + 0x280) /* Base addreess I2C0 */ +#define MCFI2C_BASE0 (MCF_MBAR + 0x280) /* Base address I2C0 */ #define MCFI2C_SIZE0 0x20 /* Register set size */ -#define MCFI2C_BASE1 (MCF_MBAR2 + 0x440) /* Base addreess I2C1 */ +#define MCFI2C_BASE1 (MCF_MBAR2 + 0x440) /* Base address I2C1 */ #define MCFI2C_SIZE1 0x20 /* Register set size */ /* diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h index f9924fbcfe42..fb95aed5f428 100644 --- a/arch/m68k/include/asm/mcf_pgalloc.h +++ b/arch/m68k/include/asm/mcf_pgalloc.h @@ -14,7 +14,7 @@ extern const char bad_pmd_string[]; extern inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - unsigned long page = __get_free_page(GFP_DMA|__GFP_REPEAT); + unsigned long page = __get_free_page(GFP_DMA); if (!page) return NULL; @@ -51,7 +51,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page, static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) { - struct page *page = alloc_pages(GFP_DMA|__GFP_REPEAT, 0); + struct page *page = alloc_pages(GFP_DMA, 0); pte_t *pte; if (!page) diff --git a/arch/m68k/include/asm/mcfmmu.h b/arch/m68k/include/asm/mcfmmu.h index 26cc3d5a63f8..8824236e303f 100644 --- a/arch/m68k/include/asm/mcfmmu.h +++ b/arch/m68k/include/asm/mcfmmu.h @@ -38,7 +38,7 @@ /* * MMU Operation register. */ -#define MMUOR_UAA 0x00000001 /* Update allocatiom address */ +#define MMUOR_UAA 0x00000001 /* Update allocation address */ #define MMUOR_ACC 0x00000002 /* TLB access */ #define MMUOR_RD 0x00000004 /* TLB access read */ #define MMUOR_WR 0x00000000 /* TLB access write */ diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h index 24bcba496c75..c895b987202c 100644 --- a/arch/m68k/include/asm/motorola_pgalloc.h +++ b/arch/m68k/include/asm/motorola_pgalloc.h @@ -11,7 +11,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long ad { pte_t *pte; - pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); if (pte) { __flush_page_to_ram(pte); flush_tlb_kernel_page(pte); @@ -32,7 +32,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres struct page *page; pte_t *pte; - page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0); + page = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0); if(!page) return NULL; if (!pgtable_page_ctor(page)) { diff --git a/arch/m68k/include/asm/q40_master.h b/arch/m68k/include/asm/q40_master.h index fc5b36278d04..c48d21b68f04 100644 --- a/arch/m68k/include/asm/q40_master.h +++ b/arch/m68k/include/asm/q40_master.h @@ -1,6 +1,6 @@ /* * Q40 master Chip Control - * RTC stuff merged for compactnes.. + * RTC stuff merged for compactness. */ #ifndef _Q40_MASTER_H diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h index 0931388de47f..1901f61f926f 100644 --- a/arch/m68k/include/asm/sun3_pgalloc.h +++ b/arch/m68k/include/asm/sun3_pgalloc.h @@ -37,7 +37,7 @@ do { \ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - unsigned long page = __get_free_page(GFP_KERNEL|__GFP_REPEAT); + unsigned long page = __get_free_page(GFP_KERNEL); if (!page) return NULL; @@ -49,7 +49,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { - struct page *page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0); + struct page *page = alloc_pages(GFP_KERNEL, 0); if (page == NULL) return NULL; diff --git a/arch/m68k/mac/iop.c b/arch/m68k/mac/iop.c index 4d2adfb32a2a..7990b6f50105 100644 --- a/arch/m68k/mac/iop.c +++ b/arch/m68k/mac/iop.c @@ -60,7 +60,7 @@ * * The host talks to the IOPs using a rather simple message-passing scheme via * a shared memory area in the IOP RAM. Each IOP has seven "channels"; each - * channel is conneced to a specific software driver on the IOP. For example + * channel is connected to a specific software driver on the IOP. For example * on the SCC IOP there is one channel for each serial port. Each channel has * an incoming and and outgoing message queue with a depth of one. * diff --git a/arch/m68k/math-emu/fp_decode.h b/arch/m68k/math-emu/fp_decode.h index 759679d9ab96..6d1e760e2a0e 100644 --- a/arch/m68k/math-emu/fp_decode.h +++ b/arch/m68k/math-emu/fp_decode.h @@ -130,7 +130,7 @@ do_fscc=0 bfextu %d2{#13,#3},%d0 .endm -| decode the 8bit diplacement from the brief extension word +| decode the 8bit displacement from the brief extension word .macro fp_decode_disp8 move.b %d2,%d0 ext.w %d0 diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index 6a94cdd0c830..bd66a0b20c6b 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -136,7 +136,7 @@ good_area: * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); pr_debug("handle_mm_fault returns %d\n", fault); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) diff --git a/arch/metag/include/asm/atomic_lnkget.h b/arch/metag/include/asm/atomic_lnkget.h index 88fa25fae8bd..def2c642f053 100644 --- a/arch/metag/include/asm/atomic_lnkget.h +++ b/arch/metag/include/asm/atomic_lnkget.h @@ -69,16 +69,44 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return result; \ } -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int result, temp; \ + \ + smp_mb(); \ + \ + asm volatile ( \ + "1: LNKGETD %1, [%2]\n" \ + " " #op " %0, %1, %3\n" \ + " LNKSETD [%2], %0\n" \ + " DEFR %0, TXSTAT\n" \ + " ANDT %0, %0, #HI(0x3f000000)\n" \ + " CMPT %0, #HI(0x02000000)\n" \ + " BNZ 1b\n" \ + : "=&d" (temp), "=&d" (result) \ + : "da" (&v->counter), "bd" (i) \ + : "cc"); \ + \ + smp_mb(); \ + \ + return result; \ +} + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/metag/include/asm/atomic_lock1.h b/arch/metag/include/asm/atomic_lock1.h index 0295d9b8d5bf..6c1380a8a0d4 100644 --- a/arch/metag/include/asm/atomic_lock1.h +++ b/arch/metag/include/asm/atomic_lock1.h @@ -64,15 +64,40 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return result; \ } -#define ATOMIC_OPS(op, c_op) ATOMIC_OP(op, c_op) ATOMIC_OP_RETURN(op, c_op) +#define ATOMIC_FETCH_OP(op, c_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned long result; \ + unsigned long flags; \ + \ + __global_lock1(flags); \ + result = v->counter; \ + fence(); \ + v->counter c_op i; \ + __global_unlock1(flags); \ + \ + return result; \ +} + +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_OP_RETURN(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) -ATOMIC_OP(and, &=) -ATOMIC_OP(or, |=) -ATOMIC_OP(xor, ^=) #undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) + +ATOMIC_OPS(and, &=) +ATOMIC_OPS(or, |=) +ATOMIC_OPS(xor, ^=) + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/metag/include/asm/pgalloc.h b/arch/metag/include/asm/pgalloc.h index 3104df0a4822..c2caa1ee4360 100644 --- a/arch/metag/include/asm/pgalloc.h +++ b/arch/metag/include/asm/pgalloc.h @@ -42,8 +42,7 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | - __GFP_ZERO); + pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); return pte; } @@ -51,7 +50,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *pte; - pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO, 0); + pte = alloc_pages(GFP_KERNEL | __GFP_ZERO, 0); if (!pte) return NULL; if (!pgtable_page_ctor(pte)) { diff --git a/arch/metag/include/asm/spinlock.h b/arch/metag/include/asm/spinlock.h index 86a7cf3d1386..c0c7a22be1ae 100644 --- a/arch/metag/include/asm/spinlock.h +++ b/arch/metag/include/asm/spinlock.h @@ -1,14 +1,24 @@ #ifndef __ASM_SPINLOCK_H #define __ASM_SPINLOCK_H +#include +#include + #ifdef CONFIG_METAG_ATOMICITY_LOCK1 #include #else #include #endif -#define arch_spin_unlock_wait(lock) \ - do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) +/* + * both lock1 and lnkget are test-and-set spinlocks with 0 unlocked and 1 + * locked. + */ + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->lock, !VAL); +} #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) diff --git a/arch/metag/mm/fault.c b/arch/metag/mm/fault.c index f57edca63609..372783a67dda 100644 --- a/arch/metag/mm/fault.c +++ b/arch/metag/mm/fault.c @@ -133,7 +133,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return 0; diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 636e0720fb20..86f65721e629 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -3,7 +3,6 @@ config MICROBLAZE select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_WANT_IPC_PARSE_VERSION - select ARCH_WANT_OPTIONAL_GPIOLIB select BUILDTIME_EXTABLE_SORT select CLKSRC_OF select CLONE_BACKWARDS3 diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h index 61436d69775c..7c89390c0c13 100644 --- a/arch/microblaze/include/asm/pgalloc.h +++ b/arch/microblaze/include/asm/pgalloc.h @@ -116,9 +116,9 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, struct page *ptepage; #ifdef CONFIG_HIGHPTE - int flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT; + int flags = GFP_KERNEL | __GFP_HIGHMEM; #else - int flags = GFP_KERNEL | __GFP_REPEAT; + int flags = GFP_KERNEL; #endif ptepage = alloc_pages(flags, 0); diff --git a/arch/microblaze/kernel/timer.c b/arch/microblaze/kernel/timer.c index 67e2ef48d2d0..5bbf38b916ef 100644 --- a/arch/microblaze/kernel/timer.c +++ b/arch/microblaze/kernel/timer.c @@ -170,7 +170,7 @@ static struct irqaction timer_irqaction = { .dev_id = &clockevent_xilinx_timer, }; -static __init void xilinx_clockevent_init(void) +static __init int xilinx_clockevent_init(void) { clockevent_xilinx_timer.mult = div_sc(timer_clock_freq, NSEC_PER_SEC, @@ -181,6 +181,8 @@ static __init void xilinx_clockevent_init(void) clockevent_delta2ns(1, &clockevent_xilinx_timer); clockevent_xilinx_timer.cpumask = cpumask_of(0); clockevents_register_device(&clockevent_xilinx_timer); + + return 0; } static u64 xilinx_clock_read(void) @@ -229,8 +231,14 @@ static struct clocksource clocksource_microblaze = { static int __init xilinx_clocksource_init(void) { - if (clocksource_register_hz(&clocksource_microblaze, timer_clock_freq)) - panic("failed to register clocksource"); + int ret; + + ret = clocksource_register_hz(&clocksource_microblaze, + timer_clock_freq); + if (ret) { + pr_err("failed to register clocksource"); + return ret; + } /* stop timer1 */ write_fn(read_fn(timer_baseaddr + TCSR1) & ~TCSR_ENT, @@ -239,16 +247,16 @@ static int __init xilinx_clocksource_init(void) write_fn(TCSR_TINT|TCSR_ENT|TCSR_ARHT, timer_baseaddr + TCSR1); /* register timecounter - for ftrace support */ - init_xilinx_timecounter(); - return 0; + return init_xilinx_timecounter(); } -static void __init xilinx_timer_init(struct device_node *timer) +static int __init xilinx_timer_init(struct device_node *timer) { struct clk *clk; static int initialized; u32 irq; u32 timer_num = 1; + int ret; if (initialized) return; @@ -258,7 +266,7 @@ static void __init xilinx_timer_init(struct device_node *timer) timer_baseaddr = of_iomap(timer, 0); if (!timer_baseaddr) { pr_err("ERROR: invalid timer base address\n"); - BUG(); + return -ENXIO; } write_fn = timer_write32; @@ -271,11 +279,15 @@ static void __init xilinx_timer_init(struct device_node *timer) } irq = irq_of_parse_and_map(timer, 0); + if (irq <= 0) { + pr_err("Failed to parse and map irq"); + return -EINVAL; + } of_property_read_u32(timer, "xlnx,one-timer-only", &timer_num); if (timer_num) { - pr_emerg("Please enable two timers in HW\n"); - BUG(); + pr_err("Please enable two timers in HW\n"); + return -EINVAL; } pr_info("%s: irq=%d\n", timer->full_name, irq); @@ -297,14 +309,27 @@ static void __init xilinx_timer_init(struct device_node *timer) freq_div_hz = timer_clock_freq / HZ; - setup_irq(irq, &timer_irqaction); + ret = setup_irq(irq, &timer_irqaction); + if (ret) { + pr_err("Failed to setup IRQ"); + return ret; + } + #ifdef CONFIG_HEART_BEAT microblaze_setup_heartbeat(); #endif - xilinx_clocksource_init(); - xilinx_clockevent_init(); + + ret = xilinx_clocksource_init(); + if (ret) + return ret; + + ret = xilinx_clockevent_init(); + if (ret) + return ret; sched_clock_register(xilinx_clock_read, 32, timer_clock_freq); + + return 0; } CLOCKSOURCE_OF_DECLARE(xilinx_timer, "xlnx,xps-timer-1.00.a", diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index 177dfc003643..abb678ccde6f 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -216,7 +216,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/microblaze/mm/pgtable.c b/arch/microblaze/mm/pgtable.c index 4f4520e779a5..eb99fcc76088 100644 --- a/arch/microblaze/mm/pgtable.c +++ b/arch/microblaze/mm/pgtable.c @@ -239,8 +239,7 @@ __init_refok pte_t *pte_alloc_one_kernel(struct mm_struct *mm, { pte_t *pte; if (mem_init_done) { - pte = (pte_t *)__get_free_page(GFP_KERNEL | - __GFP_REPEAT | __GFP_ZERO); + pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); } else { pte = (pte_t *)early_get_page(); if (pte) diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index 835b402e4574..0ab176bdb8e8 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -66,7 +66,7 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \ " " #asm_op " %0, %2 \n" \ " sc %0, %1 \n" \ " .set mips0 \n" \ - : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ + : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ : "Ir" (i)); \ } while (unlikely(!temp)); \ } else { \ @@ -79,12 +79,10 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \ } #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static __inline__ int atomic_##op##_return(int i, atomic_t * v) \ +static __inline__ int atomic_##op##_return_relaxed(int i, atomic_t * v) \ { \ int result; \ \ - smp_mb__before_llsc(); \ - \ if (kernel_uses_llsc && R10000_LLSC_WAR) { \ int temp; \ \ @@ -125,23 +123,84 @@ static __inline__ int atomic_##op##_return(int i, atomic_t * v) \ raw_local_irq_restore(flags); \ } \ \ - smp_llsc_mb(); \ + return result; \ +} + +#define ATOMIC_FETCH_OP(op, c_op, asm_op) \ +static __inline__ int atomic_fetch_##op##_relaxed(int i, atomic_t * v) \ +{ \ + int result; \ + \ + if (kernel_uses_llsc && R10000_LLSC_WAR) { \ + int temp; \ + \ + __asm__ __volatile__( \ + " .set arch=r4000 \n" \ + "1: ll %1, %2 # atomic_fetch_" #op " \n" \ + " " #asm_op " %0, %1, %3 \n" \ + " sc %0, %2 \n" \ + " beqzl %0, 1b \n" \ + " move %0, %1 \n" \ + " .set mips0 \n" \ + : "=&r" (result), "=&r" (temp), \ + "+" GCC_OFF_SMALL_ASM() (v->counter) \ + : "Ir" (i)); \ + } else if (kernel_uses_llsc) { \ + int temp; \ + \ + do { \ + __asm__ __volatile__( \ + " .set "MIPS_ISA_LEVEL" \n" \ + " ll %1, %2 # atomic_fetch_" #op " \n" \ + " " #asm_op " %0, %1, %3 \n" \ + " sc %0, %2 \n" \ + " .set mips0 \n" \ + : "=&r" (result), "=&r" (temp), \ + "+" GCC_OFF_SMALL_ASM() (v->counter) \ + : "Ir" (i)); \ + } while (unlikely(!result)); \ + \ + result = temp; \ + } else { \ + unsigned long flags; \ + \ + raw_local_irq_save(flags); \ + result = v->counter; \ + v->counter c_op i; \ + raw_local_irq_restore(flags); \ + } \ \ return result; \ } #define ATOMIC_OPS(op, c_op, asm_op) \ ATOMIC_OP(op, c_op, asm_op) \ - ATOMIC_OP_RETURN(op, c_op, asm_op) + ATOMIC_OP_RETURN(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) ATOMIC_OPS(add, +=, addu) ATOMIC_OPS(sub, -=, subu) -ATOMIC_OP(and, &=, and) -ATOMIC_OP(or, |=, or) -ATOMIC_OP(xor, ^=, xor) +#define atomic_add_return_relaxed atomic_add_return_relaxed +#define atomic_sub_return_relaxed atomic_sub_return_relaxed +#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed +#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed + +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op, asm_op) \ + ATOMIC_OP(op, c_op, asm_op) \ + ATOMIC_FETCH_OP(op, c_op, asm_op) + +ATOMIC_OPS(and, &=, and) +ATOMIC_OPS(or, |=, or) +ATOMIC_OPS(xor, ^=, xor) + +#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed +#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed +#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -362,12 +421,10 @@ static __inline__ void atomic64_##op(long i, atomic64_t * v) \ } #define ATOMIC64_OP_RETURN(op, c_op, asm_op) \ -static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ +static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v) \ { \ long result; \ \ - smp_mb__before_llsc(); \ - \ if (kernel_uses_llsc && R10000_LLSC_WAR) { \ long temp; \ \ @@ -409,22 +466,85 @@ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ raw_local_irq_restore(flags); \ } \ \ - smp_llsc_mb(); \ + return result; \ +} + +#define ATOMIC64_FETCH_OP(op, c_op, asm_op) \ +static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \ +{ \ + long result; \ + \ + if (kernel_uses_llsc && R10000_LLSC_WAR) { \ + long temp; \ + \ + __asm__ __volatile__( \ + " .set arch=r4000 \n" \ + "1: lld %1, %2 # atomic64_fetch_" #op "\n" \ + " " #asm_op " %0, %1, %3 \n" \ + " scd %0, %2 \n" \ + " beqzl %0, 1b \n" \ + " move %0, %1 \n" \ + " .set mips0 \n" \ + : "=&r" (result), "=&r" (temp), \ + "+" GCC_OFF_SMALL_ASM() (v->counter) \ + : "Ir" (i)); \ + } else if (kernel_uses_llsc) { \ + long temp; \ + \ + do { \ + __asm__ __volatile__( \ + " .set "MIPS_ISA_LEVEL" \n" \ + " lld %1, %2 # atomic64_fetch_" #op "\n" \ + " " #asm_op " %0, %1, %3 \n" \ + " scd %0, %2 \n" \ + " .set mips0 \n" \ + : "=&r" (result), "=&r" (temp), \ + "=" GCC_OFF_SMALL_ASM() (v->counter) \ + : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) \ + : "memory"); \ + } while (unlikely(!result)); \ + \ + result = temp; \ + } else { \ + unsigned long flags; \ + \ + raw_local_irq_save(flags); \ + result = v->counter; \ + v->counter c_op i; \ + raw_local_irq_restore(flags); \ + } \ \ return result; \ } #define ATOMIC64_OPS(op, c_op, asm_op) \ ATOMIC64_OP(op, c_op, asm_op) \ - ATOMIC64_OP_RETURN(op, c_op, asm_op) + ATOMIC64_OP_RETURN(op, c_op, asm_op) \ + ATOMIC64_FETCH_OP(op, c_op, asm_op) ATOMIC64_OPS(add, +=, daddu) ATOMIC64_OPS(sub, -=, dsubu) -ATOMIC64_OP(and, &=, and) -ATOMIC64_OP(or, |=, or) -ATOMIC64_OP(xor, ^=, xor) + +#define atomic64_add_return_relaxed atomic64_add_return_relaxed +#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed +#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed +#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed + +#undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, c_op, asm_op) \ + ATOMIC64_OP(op, c_op, asm_op) \ + ATOMIC64_FETCH_OP(op, c_op, asm_op) + +ATOMIC64_OPS(and, &=, and) +ATOMIC64_OPS(or, |=, or) +ATOMIC64_OPS(xor, ^=, xor) + +#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed +#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed +#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed #undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h index b336037e8768..93c079a1cfc8 100644 --- a/arch/mips/include/asm/pgalloc.h +++ b/arch/mips/include/asm/pgalloc.h @@ -69,7 +69,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, { pte_t *pte; - pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, PTE_ORDER); + pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_ZERO, PTE_ORDER); return pte; } @@ -79,7 +79,7 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, { struct page *pte; - pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER); + pte = alloc_pages(GFP_KERNEL, PTE_ORDER); if (!pte) return NULL; clear_highpage(pte); @@ -113,7 +113,7 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) { pmd_t *pmd; - pmd = (pmd_t *) __get_free_pages(GFP_KERNEL|__GFP_REPEAT, PMD_ORDER); + pmd = (pmd_t *) __get_free_pages(GFP_KERNEL, PMD_ORDER); if (pmd) pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table); return pmd; diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index a6b611f1da43..7d44e888134f 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -24,7 +24,7 @@ struct mm_struct; struct vm_area_struct; #define PAGE_NONE __pgprot(_PAGE_PRESENT | _PAGE_NO_READ | \ - _CACHE_CACHABLE_NONCOHERENT) + _page_cachable_default) #define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_WRITE | \ _page_cachable_default) #define PAGE_COPY __pgprot(_PAGE_PRESENT | _PAGE_NO_EXEC | \ @@ -476,7 +476,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) pte.pte_low &= (_PAGE_MODIFIED | _PAGE_ACCESSED | _PFNX_MASK); pte.pte_high &= (_PFN_MASK | _CACHE_MASK); pte.pte_low |= pgprot_val(newprot) & ~_PFNX_MASK; - pte.pte_high |= pgprot_val(newprot) & ~_PFN_MASK; + pte.pte_high |= pgprot_val(newprot) & ~(_PFN_MASK | _CACHE_MASK); return pte; } #elif defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32) @@ -491,7 +491,8 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #else static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { - return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | + (pgprot_val(newprot) & ~_PAGE_CHG_MASK)); } #endif @@ -632,7 +633,8 @@ static inline struct page *pmd_page(pmd_t pmd) static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { - pmd_val(pmd) = (pmd_val(pmd) & _PAGE_CHG_MASK) | pgprot_val(newprot); + pmd_val(pmd) = (pmd_val(pmd) & (_PAGE_CHG_MASK | _PAGE_HUGE)) | + (pgprot_val(newprot) & ~_PAGE_CHG_MASK); return pmd; } diff --git a/arch/mips/include/asm/spinlock.h b/arch/mips/include/asm/spinlock.h index 40196bebe849..f485afe51514 100644 --- a/arch/mips/include/asm/spinlock.h +++ b/arch/mips/include/asm/spinlock.h @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -48,8 +49,22 @@ static inline int arch_spin_value_unlocked(arch_spinlock_t lock) } #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) -#define arch_spin_unlock_wait(x) \ - while (arch_spin_is_locked(x)) { cpu_relax(); } + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + u16 owner = READ_ONCE(lock->h.serving_now); + smp_rmb(); + for (;;) { + arch_spinlock_t tmp = READ_ONCE(*lock); + + if (tmp.h.serving_now == tmp.h.ticket || + tmp.h.serving_now != owner) + break; + + cpu_relax(); + } + smp_acquire__after_ctrl_dep(); +} static inline int arch_spin_is_contended(arch_spinlock_t *lock) { diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 4b88fa031891..9560ad731120 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -153,7 +153,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/mips/ralink/cevt-rt3352.c b/arch/mips/ralink/cevt-rt3352.c index 3ad0b0794f7d..f24eee04e16a 100644 --- a/arch/mips/ralink/cevt-rt3352.c +++ b/arch/mips/ralink/cevt-rt3352.c @@ -117,11 +117,13 @@ static int systick_set_oneshot(struct clock_event_device *evt) return 0; } -static void __init ralink_systick_init(struct device_node *np) +static int __init ralink_systick_init(struct device_node *np) { + int ret; + systick.membase = of_iomap(np, 0); if (!systick.membase) - return; + return -ENXIO; systick_irqaction.name = np->name; systick.dev.name = np->name; @@ -131,16 +133,21 @@ static void __init ralink_systick_init(struct device_node *np) systick.dev.irq = irq_of_parse_and_map(np, 0); if (!systick.dev.irq) { pr_err("%s: request_irq failed", np->name); - return; + return -EINVAL; } - clocksource_mmio_init(systick.membase + SYSTICK_COUNT, np->name, - SYSTICK_FREQ, 301, 16, clocksource_mmio_readl_up); + ret = clocksource_mmio_init(systick.membase + SYSTICK_COUNT, np->name, + SYSTICK_FREQ, 301, 16, + clocksource_mmio_readl_up); + if (ret) + return ret; clockevents_register_device(&systick.dev); pr_info("%s: running - mult: %d, shift: %d\n", np->name, systick.dev.mult, systick.dev.shift); + + return 0; } CLOCKSOURCE_OF_DECLARE(systick, "ralink,cevt-systick", ralink_systick_init); diff --git a/arch/mn10300/include/asm/atomic.h b/arch/mn10300/include/asm/atomic.h index ce318d5ab23b..36389efd45e8 100644 --- a/arch/mn10300/include/asm/atomic.h +++ b/arch/mn10300/include/asm/atomic.h @@ -84,16 +84,41 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return retval; \ } -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int retval, status; \ + \ + asm volatile( \ + "1: mov %4,(_AAR,%3) \n" \ + " mov (_ADR,%3),%1 \n" \ + " mov %1,%0 \n" \ + " " #op " %5,%0 \n" \ + " mov %0,(_ADR,%3) \n" \ + " mov (_ADR,%3),%0 \n" /* flush */ \ + " mov (_ASR,%3),%0 \n" \ + " or %0,%0 \n" \ + " bne 1b \n" \ + : "=&r"(status), "=&r"(retval), "=m"(v->counter) \ + : "a"(ATOMIC_OPS_BASE_ADDR), "r"(&v->counter), "r"(i) \ + : "memory", "cc"); \ + return retval; \ +} + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/mn10300/include/asm/spinlock.h b/arch/mn10300/include/asm/spinlock.h index 1ae580f38933..9c7b8f7942d8 100644 --- a/arch/mn10300/include/asm/spinlock.h +++ b/arch/mn10300/include/asm/spinlock.h @@ -12,6 +12,8 @@ #define _ASM_SPINLOCK_H #include +#include +#include #include #include @@ -23,7 +25,11 @@ */ #define arch_spin_is_locked(x) (*(volatile signed char *)(&(x)->slock) != 0) -#define arch_spin_unlock_wait(x) do { barrier(); } while (arch_spin_is_locked(x)) + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->slock, !VAL); +} static inline void arch_spin_unlock(arch_spinlock_t *lock) { diff --git a/arch/mn10300/include/asm/thread_info.h b/arch/mn10300/include/asm/thread_info.h index 4861a78c7160..f5f90bbf019d 100644 --- a/arch/mn10300/include/asm/thread_info.h +++ b/arch/mn10300/include/asm/thread_info.h @@ -115,7 +115,7 @@ static inline unsigned long current_stack_pointer(void) } #ifndef CONFIG_KGDB -void arch_release_thread_info(struct thread_info *ti); +void arch_release_thread_stack(unsigned long *stack); #endif #define get_thread_info(ti) get_task_struct((ti)->task) #define put_thread_info(ti) put_task_struct((ti)->task) diff --git a/arch/mn10300/kernel/kgdb.c b/arch/mn10300/kernel/kgdb.c index 99770823451a..2d7986c386fe 100644 --- a/arch/mn10300/kernel/kgdb.c +++ b/arch/mn10300/kernel/kgdb.c @@ -397,8 +397,9 @@ static bool kgdb_arch_undo_singlestep(struct pt_regs *regs) * single-step state is cleared. At this point the breakpoints should have * been removed by __switch_to(). */ -void arch_release_thread_info(struct thread_info *ti) +void arch_release_thread_stack(unsigned long *stack) { + struct thread_info *ti = (void *)stack; if (kgdb_sstep_thread == ti) { kgdb_sstep_thread = NULL; diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c index 4a1d181ed32f..f23781d6bbb3 100644 --- a/arch/mn10300/mm/fault.c +++ b/arch/mn10300/mm/fault.c @@ -254,7 +254,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/mn10300/mm/pgtable.c b/arch/mn10300/mm/pgtable.c index e77a7c728081..9577cf768875 100644 --- a/arch/mn10300/mm/pgtable.c +++ b/arch/mn10300/mm/pgtable.c @@ -63,7 +63,7 @@ void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags) pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); + pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL); if (pte) clear_page(pte); return pte; @@ -74,9 +74,9 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) struct page *pte; #ifdef CONFIG_HIGHPTE - pte = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM|__GFP_REPEAT, 0); + pte = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM, 0); #else - pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0); + pte = alloc_pages(GFP_KERNEL, 0); #endif if (!pte) return NULL; diff --git a/arch/nios2/include/asm/pgalloc.h b/arch/nios2/include/asm/pgalloc.h index 6e2985e0a7b9..bb47d08c8ef7 100644 --- a/arch/nios2/include/asm/pgalloc.h +++ b/arch/nios2/include/asm/pgalloc.h @@ -42,8 +42,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, { pte_t *pte; - pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, - PTE_ORDER); + pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_ZERO, PTE_ORDER); return pte; } @@ -53,7 +52,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, { struct page *pte; - pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER); + pte = alloc_pages(GFP_KERNEL, PTE_ORDER); if (pte) { if (!pgtable_page_ctor(pte)) { __free_page(pte); diff --git a/arch/nios2/kernel/time.c b/arch/nios2/kernel/time.c index e835dda2bfe2..d9563ddb337e 100644 --- a/arch/nios2/kernel/time.c +++ b/arch/nios2/kernel/time.c @@ -206,15 +206,21 @@ irqreturn_t timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static void __init nios2_timer_get_base_and_freq(struct device_node *np, +static int __init nios2_timer_get_base_and_freq(struct device_node *np, void __iomem **base, u32 *freq) { *base = of_iomap(np, 0); - if (!*base) - panic("Unable to map reg for %s\n", np->name); + if (!*base) { + pr_crit("Unable to map reg for %s\n", np->name); + return -ENXIO; + } + + if (of_property_read_u32(np, "clock-frequency", freq)) { + pr_crit("Unable to get %s clock frequency\n", np->name); + return -EINVAL; + } - if (of_property_read_u32(np, "clock-frequency", freq)) - panic("Unable to get %s clock frequency\n", np->name); + return 0; } static struct nios2_clockevent_dev nios2_ce = { @@ -231,17 +237,21 @@ static struct nios2_clockevent_dev nios2_ce = { }, }; -static __init void nios2_clockevent_init(struct device_node *timer) +static __init int nios2_clockevent_init(struct device_node *timer) { void __iomem *iobase; u32 freq; - int irq; + int irq, ret; - nios2_timer_get_base_and_freq(timer, &iobase, &freq); + ret = nios2_timer_get_base_and_freq(timer, &iobase, &freq); + if (ret) + return ret; irq = irq_of_parse_and_map(timer, 0); - if (!irq) - panic("Unable to parse timer irq\n"); + if (!irq) { + pr_crit("Unable to parse timer irq\n"); + return -EINVAL; + } nios2_ce.timer.base = iobase; nios2_ce.timer.freq = freq; @@ -253,25 +263,35 @@ static __init void nios2_clockevent_init(struct device_node *timer) /* clear pending interrupt */ timer_writew(&nios2_ce.timer, 0, ALTERA_TIMER_STATUS_REG); - if (request_irq(irq, timer_interrupt, IRQF_TIMER, timer->name, - &nios2_ce.ced)) - panic("Unable to setup timer irq\n"); + ret = request_irq(irq, timer_interrupt, IRQF_TIMER, timer->name, + &nios2_ce.ced); + if (ret) { + pr_crit("Unable to setup timer irq\n"); + return ret; + } clockevents_config_and_register(&nios2_ce.ced, freq, 1, ULONG_MAX); + + return 0; } -static __init void nios2_clocksource_init(struct device_node *timer) +static __init int nios2_clocksource_init(struct device_node *timer) { unsigned int ctrl; void __iomem *iobase; u32 freq; + int ret; - nios2_timer_get_base_and_freq(timer, &iobase, &freq); + ret = nios2_timer_get_base_and_freq(timer, &iobase, &freq); + if (ret) + return ret; nios2_cs.timer.base = iobase; nios2_cs.timer.freq = freq; - clocksource_register_hz(&nios2_cs.cs, freq); + ret = clocksource_register_hz(&nios2_cs.cs, freq); + if (ret) + return ret; timer_writew(&nios2_cs.timer, USHRT_MAX, ALTERA_TIMER_PERIODL_REG); timer_writew(&nios2_cs.timer, USHRT_MAX, ALTERA_TIMER_PERIODH_REG); @@ -282,6 +302,8 @@ static __init void nios2_clocksource_init(struct device_node *timer) /* Calibrate the delay loop directly */ lpj_fine = freq / HZ; + + return 0; } /* @@ -289,22 +311,25 @@ static __init void nios2_clocksource_init(struct device_node *timer) * more instances, the second one gets used as clocksource and all * others are unused. */ -static void __init nios2_time_init(struct device_node *timer) +static int __init nios2_time_init(struct device_node *timer) { static int num_called; + int ret; switch (num_called) { case 0: - nios2_clockevent_init(timer); + ret = nios2_clockevent_init(timer); break; case 1: - nios2_clocksource_init(timer); + ret = nios2_clocksource_init(timer); break; default: break; } num_called++; + + return ret; } void read_persistent_clock(struct timespec *ts) diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c index b51878b0c6b8..affc4eb3f89e 100644 --- a/arch/nios2/mm/fault.c +++ b/arch/nios2/mm/fault.c @@ -131,7 +131,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index 142cb057c41b..489e7f909286 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -10,7 +10,7 @@ config OPENRISC select IRQ_DOMAIN select HANDLE_DOMAIN_IRQ select HAVE_MEMBLOCK - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select HAVE_ARCH_TRACEHOOK select GENERIC_IRQ_CHIP select GENERIC_IRQ_PROBE diff --git a/arch/openrisc/include/asm/pgalloc.h b/arch/openrisc/include/asm/pgalloc.h index 21484e5b9e9a..87eebd185089 100644 --- a/arch/openrisc/include/asm/pgalloc.h +++ b/arch/openrisc/include/asm/pgalloc.h @@ -77,7 +77,7 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *pte; - pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0); + pte = alloc_pages(GFP_KERNEL, 0); if (!pte) return NULL; clear_page(page_address(pte)); diff --git a/arch/openrisc/mm/fault.c b/arch/openrisc/mm/fault.c index 230ac20ae794..e94cd225e816 100644 --- a/arch/openrisc/mm/fault.c +++ b/arch/openrisc/mm/fault.c @@ -163,7 +163,7 @@ good_area: * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/openrisc/mm/ioremap.c b/arch/openrisc/mm/ioremap.c index 62b08ef392be..5b2a95116e8f 100644 --- a/arch/openrisc/mm/ioremap.c +++ b/arch/openrisc/mm/ioremap.c @@ -122,7 +122,7 @@ pte_t __init_refok *pte_alloc_one_kernel(struct mm_struct *mm, pte_t *pte; if (likely(mem_init_done)) { - pte = (pte_t *) __get_free_page(GFP_KERNEL | __GFP_REPEAT); + pte = (pte_t *) __get_free_page(GFP_KERNEL); } else { pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE); #if 0 diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index 1d109990a022..5394b9c5f914 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -121,16 +121,39 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ return ret; \ } -#define ATOMIC_OPS(op, c_op) ATOMIC_OP(op, c_op) ATOMIC_OP_RETURN(op, c_op) +#define ATOMIC_FETCH_OP(op, c_op) \ +static __inline__ int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned long flags; \ + int ret; \ + \ + _atomic_spin_lock_irqsave(v, flags); \ + ret = v->counter; \ + v->counter c_op i; \ + _atomic_spin_unlock_irqrestore(v, flags); \ + \ + return ret; \ +} + +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_OP_RETURN(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) -ATOMIC_OP(and, &=) -ATOMIC_OP(or, |=) -ATOMIC_OP(xor, ^=) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) + +ATOMIC_OPS(and, &=) +ATOMIC_OPS(or, |=) +ATOMIC_OPS(xor, ^=) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -185,15 +208,39 @@ static __inline__ s64 atomic64_##op##_return(s64 i, atomic64_t *v) \ return ret; \ } -#define ATOMIC64_OPS(op, c_op) ATOMIC64_OP(op, c_op) ATOMIC64_OP_RETURN(op, c_op) +#define ATOMIC64_FETCH_OP(op, c_op) \ +static __inline__ s64 atomic64_fetch_##op(s64 i, atomic64_t *v) \ +{ \ + unsigned long flags; \ + s64 ret; \ + \ + _atomic_spin_lock_irqsave(v, flags); \ + ret = v->counter; \ + v->counter c_op i; \ + _atomic_spin_unlock_irqrestore(v, flags); \ + \ + return ret; \ +} + +#define ATOMIC64_OPS(op, c_op) \ + ATOMIC64_OP(op, c_op) \ + ATOMIC64_OP_RETURN(op, c_op) \ + ATOMIC64_FETCH_OP(op, c_op) ATOMIC64_OPS(add, +=) ATOMIC64_OPS(sub, -=) -ATOMIC64_OP(and, &=) -ATOMIC64_OP(or, |=) -ATOMIC64_OP(xor, ^=) #undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, c_op) \ + ATOMIC64_OP(op, c_op) \ + ATOMIC64_FETCH_OP(op, c_op) + +ATOMIC64_OPS(and, &=) +ATOMIC64_OPS(or, |=) +ATOMIC64_OPS(xor, ^=) + +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h index f2fd327dce2e..f08dda3f0995 100644 --- a/arch/parisc/include/asm/pgalloc.h +++ b/arch/parisc/include/asm/pgalloc.h @@ -63,8 +63,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) { - pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, - PMD_ORDER); + pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL, PMD_ORDER); if (pmd) memset(pmd, 0, PAGE_SIZE<, IBM + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ +#include +#include + + .section .rodata +.balign 16 + +.byteswap_constant: + /* byte reverse permute constant */ + .octa 0x0F0E0D0C0B0A09080706050403020100 + +#define MAX_SIZE 32768 +.constants: + + /* Reduce 262144 kbits to 1024 bits */ + /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ + .octa 0x00000000b6ca9e20000000009c37c408 + + /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ + .octa 0x00000000350249a800000001b51df26c + + /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ + .octa 0x00000001862dac54000000000724b9d0 + + /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ + .octa 0x00000001d87fb48c00000001c00532fe + + /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ + .octa 0x00000001f39b699e00000000f05a9362 + + /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ + .octa 0x0000000101da11b400000001e1007970 + + /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ + .octa 0x00000001cab571e000000000a57366ee + + /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ + .octa 0x00000000c7020cfe0000000192011284 + + /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ + .octa 0x00000000cdaed1ae0000000162716d9a + + /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ + .octa 0x00000001e804effc00000000cd97ecde + + /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ + .octa 0x0000000077c3ea3a0000000058812bc0 + + /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ + .octa 0x0000000068df31b40000000088b8c12e + + /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ + .octa 0x00000000b059b6c200000001230b234c + + /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ + .octa 0x0000000145fb8ed800000001120b416e + + /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ + .octa 0x00000000cbc0916800000001974aecb0 + + /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ + .octa 0x000000005ceeedc2000000008ee3f226 + + /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ + .octa 0x0000000047d74e8600000001089aba9a + + /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ + .octa 0x00000001407e9e220000000065113872 + + /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ + .octa 0x00000001da967bda000000005c07ec10 + + /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ + .octa 0x000000006c8983680000000187590924 + + /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ + .octa 0x00000000f2d14c9800000000e35da7c6 + + /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ + .octa 0x00000001993c6ad4000000000415855a + + /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ + .octa 0x000000014683d1ac0000000073617758 + + /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ + .octa 0x00000001a7c93e6c0000000176021d28 + + /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ + .octa 0x000000010211e90a00000001c358fd0a + + /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ + .octa 0x000000001119403e00000001ff7a2c18 + + /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ + .octa 0x000000001c3261aa00000000f2d9f7e4 + + /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ + .octa 0x000000014e37a634000000016cf1f9c8 + + /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ + .octa 0x0000000073786c0c000000010af9279a + + /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ + .octa 0x000000011dc037f80000000004f101e8 + + /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ + .octa 0x0000000031433dfc0000000070bcf184 + + /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ + .octa 0x000000009cde8348000000000a8de642 + + /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ + .octa 0x0000000038d3c2a60000000062ea130c + + /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ + .octa 0x000000011b25f26000000001eb31cbb2 + + /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ + .octa 0x000000001629e6f00000000170783448 + + /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ + .octa 0x0000000160838b4c00000001a684b4c6 + + /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ + .octa 0x000000007a44011c00000000253ca5b4 + + /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ + .octa 0x00000000226f417a0000000057b4b1e2 + + /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ + .octa 0x0000000045eb2eb400000000b6bd084c + + /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ + .octa 0x000000014459d70c0000000123c2d592 + + /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ + .octa 0x00000001d406ed8200000000159dafce + + /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ + .octa 0x0000000160c8e1a80000000127e1a64e + + /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ + .octa 0x0000000027ba80980000000056860754 + + /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ + .octa 0x000000006d92d01800000001e661aae8 + + /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ + .octa 0x000000012ed7e3f200000000f82c6166 + + /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ + .octa 0x000000002dc8778800000000c4f9c7ae + + /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ + .octa 0x0000000018240bb80000000074203d20 + + /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ + .octa 0x000000001ad381580000000198173052 + + /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ + .octa 0x00000001396b78f200000001ce8aba54 + + /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ + .octa 0x000000011a68133400000001850d5d94 + + /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ + .octa 0x000000012104732e00000001d609239c + + /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ + .octa 0x00000000a140d90c000000001595f048 + + /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ + .octa 0x00000001b7215eda0000000042ccee08 + + /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ + .octa 0x00000001aaf1df3c000000010a389d74 + + /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ + .octa 0x0000000029d15b8a000000012a840da6 + + /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ + .octa 0x00000000f1a96922000000001d181c0c + + /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ + .octa 0x00000001ac80d03c0000000068b7d1f6 + + /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ + .octa 0x000000000f11d56a000000005b0f14fc + + /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ + .octa 0x00000001f1c022a20000000179e9e730 + + /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ + .octa 0x0000000173d00ae200000001ce1368d6 + + /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ + .octa 0x00000001d4ffe4ac0000000112c3a84c + + /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ + .octa 0x000000016edc5ae400000000de940fee + + /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ + .octa 0x00000001f1a0214000000000fe896b7e + + /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ + .octa 0x00000000ca0b28a000000001f797431c + + /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ + .octa 0x00000001928e30a20000000053e989ba + + /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ + .octa 0x0000000097b1b002000000003920cd16 + + /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ + .octa 0x00000000b15bf90600000001e6f579b8 + + /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ + .octa 0x00000000411c5d52000000007493cb0a + + /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ + .octa 0x00000001c36f330000000001bdd376d8 + + /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ + .octa 0x00000001119227e0000000016badfee6 + + /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ + .octa 0x00000000114d47020000000071de5c58 + + /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ + .octa 0x00000000458b5b9800000000453f317c + + /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ + .octa 0x000000012e31fb8e0000000121675cce + + /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ + .octa 0x000000005cf619d800000001f409ee92 + + /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ + .octa 0x0000000063f4d8b200000000f36b9c88 + + /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ + .octa 0x000000004138dc8a0000000036b398f4 + + /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ + .octa 0x00000001d29ee8e000000001748f9adc + + /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ + .octa 0x000000006a08ace800000001be94ec00 + + /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ + .octa 0x0000000127d4201000000000b74370d6 + + /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ + .octa 0x0000000019d76b6200000001174d0b98 + + /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ + .octa 0x00000001b1471f6e00000000befc06a4 + + /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ + .octa 0x00000001f64c19cc00000001ae125288 + + /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ + .octa 0x00000000003c0ea00000000095c19b34 + + /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ + .octa 0x000000014d73abf600000001a78496f2 + + /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ + .octa 0x00000001620eb84400000001ac5390a0 + + /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ + .octa 0x0000000147655048000000002a80ed6e + + /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ + .octa 0x0000000067b5077e00000001fa9b0128 + + /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ + .octa 0x0000000010ffe20600000001ea94929e + + /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ + .octa 0x000000000fee8f1e0000000125f4305c + + /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ + .octa 0x00000001da26fbae00000001471e2002 + + /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ + .octa 0x00000001b3a8bd880000000132d2253a + + /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ + .octa 0x00000000e8f3898e00000000f26b3592 + + /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ + .octa 0x00000000b0d0d28c00000000bc8b67b0 + + /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ + .octa 0x0000000030f2a798000000013a826ef2 + + /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ + .octa 0x000000000fba10020000000081482c84 + + /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ + .octa 0x00000000bdb9bd7200000000e77307c2 + + /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ + .octa 0x0000000075d3bf5a00000000d4a07ec8 + + /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ + .octa 0x00000000ef1f98a00000000017102100 + + /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ + .octa 0x00000000689c760200000000db406486 + + /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ + .octa 0x000000016d5fa5fe0000000192db7f88 + + /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ + .octa 0x00000001d0d2b9ca000000018bf67b1e + + /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ + .octa 0x0000000041e7b470000000007c09163e + + /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ + .octa 0x00000001cbb6495e000000000adac060 + + /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ + .octa 0x000000010052a0b000000000bd8316ae + + /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ + .octa 0x00000001d8effb5c000000019f09ab54 + + /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ + .octa 0x00000001d969853c0000000125155542 + + /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ + .octa 0x00000000523ccce2000000018fdb5882 + + /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ + .octa 0x000000001e2436bc00000000e794b3f4 + + /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ + .octa 0x00000000ddd1c3a2000000016f9bb022 + + /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ + .octa 0x0000000019fcfe3800000000290c9978 + + /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ + .octa 0x00000001ce95db640000000083c0f350 + + /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ + .octa 0x00000000af5828060000000173ea6628 + + /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ + .octa 0x00000001006388f600000001c8b4e00a + + /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ + .octa 0x0000000179eca00a00000000de95d6aa + + /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ + .octa 0x0000000122410a6a000000010b7f7248 + + /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ + .octa 0x000000004288e87c00000001326e3a06 + + /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ + .octa 0x000000016c5490da00000000bb62c2e6 + + /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ + .octa 0x00000000d1c71f6e0000000156a4b2c2 + + /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ + .octa 0x00000001b4ce08a6000000011dfe763a + + /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ + .octa 0x00000001466ba60c000000007bcca8e2 + + /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ + .octa 0x00000001f6c488a40000000186118faa + + /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ + .octa 0x000000013bfb06820000000111a65a88 + + /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ + .octa 0x00000000690e9e54000000003565e1c4 + + /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ + .octa 0x00000000281346b6000000012ed02a82 + + /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ + .octa 0x000000015646402400000000c486ecfc + + /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ + .octa 0x000000016063a8dc0000000001b951b2 + + /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ + .octa 0x0000000116a663620000000048143916 + + /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ + .octa 0x000000017e8aa4d200000001dc2ae124 + + /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ + .octa 0x00000001728eb10c00000001416c58d6 + + /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ + .octa 0x00000001b08fd7fa00000000a479744a + + /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ + .octa 0x00000001092a16e80000000096ca3a26 + + /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ + .octa 0x00000000a505637c00000000ff223d4e + + /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ + .octa 0x00000000d94869b2000000010e84da42 + + /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ + .octa 0x00000001c8b203ae00000001b61ba3d0 + + /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ + .octa 0x000000005704aea000000000680f2de8 + + /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ + .octa 0x000000012e295fa2000000008772a9a8 + + /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ + .octa 0x000000011d0908bc0000000155f295bc + + /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ + .octa 0x0000000193ed97ea00000000595f9282 + + /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ + .octa 0x000000013a0f1c520000000164b1c25a + + /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ + .octa 0x000000010c2c40c000000000fbd67c50 + + /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ + .octa 0x00000000ff6fac3e0000000096076268 + + /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ + .octa 0x000000017b3609c000000001d288e4cc + + /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ + .octa 0x0000000088c8c92200000001eaac1bdc + + /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ + .octa 0x00000001751baae600000001f1ea39e2 + + /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ + .octa 0x000000010795297200000001eb6506fc + + /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ + .octa 0x0000000162b00abe000000010f806ffe + + /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ + .octa 0x000000000d7b404c000000010408481e + + /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ + .octa 0x00000000763b13d40000000188260534 + + /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ + .octa 0x00000000f6dc22d80000000058fc73e0 + + /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ + .octa 0x000000007daae06000000000391c59b8 + + /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ + .octa 0x000000013359ab7c000000018b638400 + + /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ + .octa 0x000000008add438a000000011738f5c4 + + /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ + .octa 0x00000001edbefdea000000008cf7c6da + + /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ + .octa 0x000000004104e0f800000001ef97fb16 + + /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ + .octa 0x00000000b48a82220000000102130e20 + + /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ + .octa 0x00000001bcb4684400000000db968898 + + /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ + .octa 0x000000013293ce0a00000000b5047b5e + + /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ + .octa 0x00000001710d0844000000010b90fdb2 + + /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ + .octa 0x0000000117907f6e000000004834a32e + + /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ + .octa 0x0000000087ddf93e0000000059c8f2b0 + + /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ + .octa 0x000000005970e9b00000000122cec508 + + /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ + .octa 0x0000000185b2b7d0000000000a330cda + + /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ + .octa 0x00000001dcee0efc000000014a47148c + + /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ + .octa 0x0000000030da27220000000042c61cb8 + + /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ + .octa 0x000000012f925a180000000012fe6960 + + /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ + .octa 0x00000000dd2e357c00000000dbda2c20 + + /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ + .octa 0x00000000071c80de000000011122410c + + /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ + .octa 0x000000011513140a00000000977b2070 + + /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ + .octa 0x00000001df876e8e000000014050438e + + /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ + .octa 0x000000015f81d6ce0000000147c840e8 + + /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ + .octa 0x000000019dd94dbe00000001cc7c88ce + + /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ + .octa 0x00000001373d206e00000001476b35a4 + + /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ + .octa 0x00000000668ccade000000013d52d508 + + /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ + .octa 0x00000001b192d268000000008e4be32e + + /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ + .octa 0x00000000e30f3a7800000000024120fe + + /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ + .octa 0x000000010ef1f7bc00000000ddecddb4 + + /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ + .octa 0x00000001f5ac738000000000d4d403bc + + /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ + .octa 0x000000011822ea7000000001734b89aa + + /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ + .octa 0x00000000c3a33848000000010e7a58d6 + + /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ + .octa 0x00000001bd151c2400000001f9f04e9c + + /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ + .octa 0x0000000056002d7600000000b692225e + + /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ + .octa 0x000000014657c4f4000000019b8d3f3e + + /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ + .octa 0x0000000113742d7c00000001a874f11e + + /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ + .octa 0x000000019c5920ba000000010d5a4254 + + /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ + .octa 0x000000005216d2d600000000bbb2f5d6 + + /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ + .octa 0x0000000136f5ad8a0000000179cc0e36 + + /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ + .octa 0x000000018b07beb600000001dca1da4a + + /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ + .octa 0x00000000db1e93b000000000feb1a192 + + /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ + .octa 0x000000000b96fa3a00000000d1eeedd6 + + /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ + .octa 0x00000001d9968af0000000008fad9bb4 + + /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ + .octa 0x000000000e4a77a200000001884938e4 + + /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ + .octa 0x00000000508c2ac800000001bc2e9bc0 + + /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ + .octa 0x0000000021572a8000000001f9658a68 + + /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ + .octa 0x00000001b859daf2000000001b9224fc + + /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ + .octa 0x000000016f7884740000000055b2fb84 + + /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ + .octa 0x00000001b438810e000000018b090348 + + /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ + .octa 0x0000000095ddc6f2000000011ccbd5ea + + /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ + .octa 0x00000001d977c20c0000000007ae47f8 + + /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ + .octa 0x00000000ebedb99a0000000172acbec0 + + /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ + .octa 0x00000001df9e9e9200000001c6e3ff20 + + /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ + .octa 0x00000001a4a3f95200000000e1b38744 + + /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ + .octa 0x00000000e2f5122000000000791585b2 + + /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ + .octa 0x000000004aa01f3e00000000ac53b894 + + /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ + .octa 0x00000000b3e90a5800000001ed5f2cf4 + + /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ + .octa 0x000000000c9ca2aa00000001df48b2e0 + + /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ + .octa 0x000000015168231600000000049c1c62 + + /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ + .octa 0x0000000036fce78c000000017c460c12 + + /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ + .octa 0x000000009037dc10000000015be4da7e + + /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ + .octa 0x00000000d3298582000000010f38f668 + + /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ + .octa 0x00000001b42e8ad60000000039f40a00 + + /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ + .octa 0x00000000142a983800000000bd4c10c4 + + /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ + .octa 0x0000000109c7f1900000000042db1d98 + + /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ + .octa 0x0000000056ff931000000001c905bae6 + + /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ + .octa 0x00000001594513aa00000000069d40ea + + /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ + .octa 0x00000001e3b5b1e8000000008e4fbad0 + + /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ + .octa 0x000000011dd5fc080000000047bedd46 + + /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ + .octa 0x00000001675f0cc20000000026396bf8 + + /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ + .octa 0x00000000d1c8dd4400000000379beb92 + + /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ + .octa 0x0000000115ebd3d8000000000abae54a + + /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ + .octa 0x00000001ecbd0dac0000000007e6a128 + + /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ + .octa 0x00000000cdf67af2000000000ade29d2 + + /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ + .octa 0x000000004c01ff4c00000000f974c45c + + /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ + .octa 0x00000000f2d8657e00000000e77ac60a + + /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ + .octa 0x000000006bae74c40000000145895816 + + /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ + .octa 0x0000000152af8aa00000000038e362be + + /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ + .octa 0x0000000004663802000000007f991a64 + + /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ + .octa 0x00000001ab2f5afc00000000fa366d3a + + /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ + .octa 0x0000000074a4ebd400000001a2bb34f0 + + /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ + .octa 0x00000001d7ab3a4c0000000028a9981e + + /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ + .octa 0x00000001a8da60c600000001dbc672be + + /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ + .octa 0x000000013cf6382000000000b04d77f6 + + /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ + .octa 0x00000000bec12e1e0000000124400d96 + + /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ + .octa 0x00000001c6368010000000014ca4b414 + + /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ + .octa 0x00000001e6e78758000000012fe2c938 + + /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ + .octa 0x000000008d7f2b3c00000001faed01e6 + + /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ + .octa 0x000000016b4a156e000000007e80ecfe + + /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ + .octa 0x00000001c63cfeb60000000098daee94 + + /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ + .octa 0x000000015f902670000000010a04edea + + /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ + .octa 0x00000001cd5de11e00000001c00b4524 + + /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ + .octa 0x000000001acaec540000000170296550 + + /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ + .octa 0x000000002bd0ca780000000181afaa48 + + /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ + .octa 0x0000000032d63d5c0000000185a31ffa + + /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ + .octa 0x000000001c6d4e4c000000002469f608 + + /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ + .octa 0x0000000106a60b92000000006980102a + + /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ + .octa 0x00000000d3855e120000000111ea9ca8 + + /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ + .octa 0x00000000e312563600000001bd1d29ce + + /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ + .octa 0x000000009e8f7ea400000001b34b9580 + + /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ + .octa 0x00000001c82e562c000000003076054e + + /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ + .octa 0x00000000ca9f09ce000000012a608ea4 + + /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ + .octa 0x00000000c63764e600000000784d05fe + + /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ + .octa 0x0000000168d2e49e000000016ef0d82a + + /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ + .octa 0x00000000e986c1480000000075bda454 + + /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ + .octa 0x00000000cfb65894000000003dc0a1c4 + + /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ + .octa 0x0000000111cadee400000000e9a5d8be + + /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ + .octa 0x0000000171fb63ce00000001609bc4b4 + +.short_constants: + + /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ + /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, x^2048 mod p(x)` */ + .octa 0x7fec2963e5bf80485cf015c388e56f72 + + /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, x^1920 mod p(x)` */ + .octa 0x38e888d4844752a9963a18920246e2e6 + + /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, x^1792 mod p(x)` */ + .octa 0x42316c00730206ad419a441956993a31 + + /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, x^1664 mod p(x)` */ + .octa 0x543d5c543e65ddf9924752ba2b830011 + + /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, x^1536 mod p(x)` */ + .octa 0x78e87aaf56767c9255bd7f9518e4a304 + + /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, x^1408 mod p(x)` */ + .octa 0x8f68fcec1903da7f6d76739fe0553f1e + + /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, x^1280 mod p(x)` */ + .octa 0x3f4840246791d588c133722b1fe0b5c3 + + /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, x^1152 mod p(x)` */ + .octa 0x34c96751b04de25a64b67ee0e55ef1f3 + + /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, x^1024 mod p(x)` */ + .octa 0x156c8e180b4a395b069db049b8fdb1e7 + + /* x^800 mod p(x)`, x^832 mod p(x)`, x^864 mod p(x)`, x^896 mod p(x)` */ + .octa 0xe0b99ccbe661f7bea11bfaf3c9e90b9e + + /* x^672 mod p(x)`, x^704 mod p(x)`, x^736 mod p(x)`, x^768 mod p(x)` */ + .octa 0x041d37768cd75659817cdc5119b29a35 + + /* x^544 mod p(x)`, x^576 mod p(x)`, x^608 mod p(x)`, x^640 mod p(x)` */ + .octa 0x3a0777818cfaa9651ce9d94b36c41f1c + + /* x^416 mod p(x)`, x^448 mod p(x)`, x^480 mod p(x)`, x^512 mod p(x)` */ + .octa 0x0e148e8252377a554f256efcb82be955 + + /* x^288 mod p(x)`, x^320 mod p(x)`, x^352 mod p(x)`, x^384 mod p(x)` */ + .octa 0x9c25531d19e65ddeec1631edb2dea967 + + /* x^160 mod p(x)`, x^192 mod p(x)`, x^224 mod p(x)`, x^256 mod p(x)` */ + .octa 0x790606ff9957c0a65d27e147510ac59a + + /* x^32 mod p(x)`, x^64 mod p(x)`, x^96 mod p(x)`, x^128 mod p(x)` */ + .octa 0x82f63b786ea2d55ca66805eb18b8ea18 + + +.barrett_constants: + /* 33 bit reflected Barrett constant m - (4^32)/n */ + .octa 0x000000000000000000000000dea713f1 /* x^64 div p(x)` */ + /* 33 bit reflected Barrett constant n */ + .octa 0x00000000000000000000000105ec76f1 + + .text + +#if defined(__BIG_ENDIAN__) +#define BYTESWAP_DATA +#else +#undef BYTESWAP_DATA +#endif + +#define off16 r25 +#define off32 r26 +#define off48 r27 +#define off64 r28 +#define off80 r29 +#define off96 r30 +#define off112 r31 + +#define const1 v24 +#define const2 v25 + +#define byteswap v26 +#define mask_32bit v27 +#define mask_64bit v28 +#define zeroes v29 + +#ifdef BYTESWAP_DATA +#define VPERM(A, B, C, D) vperm A, B, C, D +#else +#define VPERM(A, B, C, D) +#endif + +/* unsigned int __crc32c_vpmsum(unsigned int crc, void *p, unsigned long len) */ +FUNC_START(__crc32c_vpmsum) + std r31,-8(r1) + std r30,-16(r1) + std r29,-24(r1) + std r28,-32(r1) + std r27,-40(r1) + std r26,-48(r1) + std r25,-56(r1) + + li off16,16 + li off32,32 + li off48,48 + li off64,64 + li off80,80 + li off96,96 + li off112,112 + li r0,0 + + /* Enough room for saving 10 non volatile VMX registers */ + subi r6,r1,56+10*16 + subi r7,r1,56+2*16 + + stvx v20,0,r6 + stvx v21,off16,r6 + stvx v22,off32,r6 + stvx v23,off48,r6 + stvx v24,off64,r6 + stvx v25,off80,r6 + stvx v26,off96,r6 + stvx v27,off112,r6 + stvx v28,0,r7 + stvx v29,off16,r7 + + mr r10,r3 + + vxor zeroes,zeroes,zeroes + vspltisw v0,-1 + + vsldoi mask_32bit,zeroes,v0,4 + vsldoi mask_64bit,zeroes,v0,8 + + /* Get the initial value into v8 */ + vxor v8,v8,v8 + MTVRD(v8, R3) + vsldoi v8,zeroes,v8,8 /* shift into bottom 32 bits */ + +#ifdef BYTESWAP_DATA + addis r3,r2,.byteswap_constant@toc@ha + addi r3,r3,.byteswap_constant@toc@l + + lvx byteswap,0,r3 + addi r3,r3,16 +#endif + + cmpdi r5,256 + blt .Lshort + + rldicr r6,r5,0,56 + + /* Checksum in blocks of MAX_SIZE */ +1: lis r7,MAX_SIZE@h + ori r7,r7,MAX_SIZE@l + mr r9,r7 + cmpd r6,r7 + bgt 2f + mr r7,r6 +2: subf r6,r7,r6 + + /* our main loop does 128 bytes at a time */ + srdi r7,r7,7 + + /* + * Work out the offset into the constants table to start at. Each + * constant is 16 bytes, and it is used against 128 bytes of input + * data - 128 / 16 = 8 + */ + sldi r8,r7,4 + srdi r9,r9,3 + subf r8,r8,r9 + + /* We reduce our final 128 bytes in a separate step */ + addi r7,r7,-1 + mtctr r7 + + addis r3,r2,.constants@toc@ha + addi r3,r3,.constants@toc@l + + /* Find the start of our constants */ + add r3,r3,r8 + + /* zero v0-v7 which will contain our checksums */ + vxor v0,v0,v0 + vxor v1,v1,v1 + vxor v2,v2,v2 + vxor v3,v3,v3 + vxor v4,v4,v4 + vxor v5,v5,v5 + vxor v6,v6,v6 + vxor v7,v7,v7 + + lvx const1,0,r3 + + /* + * If we are looping back to consume more data we use the values + * already in v16-v23. + */ + cmpdi r0,1 + beq 2f + + /* First warm up pass */ + lvx v16,0,r4 + lvx v17,off16,r4 + VPERM(v16,v16,v16,byteswap) + VPERM(v17,v17,v17,byteswap) + lvx v18,off32,r4 + lvx v19,off48,r4 + VPERM(v18,v18,v18,byteswap) + VPERM(v19,v19,v19,byteswap) + lvx v20,off64,r4 + lvx v21,off80,r4 + VPERM(v20,v20,v20,byteswap) + VPERM(v21,v21,v21,byteswap) + lvx v22,off96,r4 + lvx v23,off112,r4 + VPERM(v22,v22,v22,byteswap) + VPERM(v23,v23,v23,byteswap) + addi r4,r4,8*16 + + /* xor in initial value */ + vxor v16,v16,v8 + +2: bdz .Lfirst_warm_up_done + + addi r3,r3,16 + lvx const2,0,r3 + + /* Second warm up pass */ + VPMSUMD(v8,v16,const1) + lvx v16,0,r4 + VPERM(v16,v16,v16,byteswap) + ori r2,r2,0 + + VPMSUMD(v9,v17,const1) + lvx v17,off16,r4 + VPERM(v17,v17,v17,byteswap) + ori r2,r2,0 + + VPMSUMD(v10,v18,const1) + lvx v18,off32,r4 + VPERM(v18,v18,v18,byteswap) + ori r2,r2,0 + + VPMSUMD(v11,v19,const1) + lvx v19,off48,r4 + VPERM(v19,v19,v19,byteswap) + ori r2,r2,0 + + VPMSUMD(v12,v20,const1) + lvx v20,off64,r4 + VPERM(v20,v20,v20,byteswap) + ori r2,r2,0 + + VPMSUMD(v13,v21,const1) + lvx v21,off80,r4 + VPERM(v21,v21,v21,byteswap) + ori r2,r2,0 + + VPMSUMD(v14,v22,const1) + lvx v22,off96,r4 + VPERM(v22,v22,v22,byteswap) + ori r2,r2,0 + + VPMSUMD(v15,v23,const1) + lvx v23,off112,r4 + VPERM(v23,v23,v23,byteswap) + + addi r4,r4,8*16 + + bdz .Lfirst_cool_down + + /* + * main loop. We modulo schedule it such that it takes three iterations + * to complete - first iteration load, second iteration vpmsum, third + * iteration xor. + */ + .balign 16 +4: lvx const1,0,r3 + addi r3,r3,16 + ori r2,r2,0 + + vxor v0,v0,v8 + VPMSUMD(v8,v16,const2) + lvx v16,0,r4 + VPERM(v16,v16,v16,byteswap) + ori r2,r2,0 + + vxor v1,v1,v9 + VPMSUMD(v9,v17,const2) + lvx v17,off16,r4 + VPERM(v17,v17,v17,byteswap) + ori r2,r2,0 + + vxor v2,v2,v10 + VPMSUMD(v10,v18,const2) + lvx v18,off32,r4 + VPERM(v18,v18,v18,byteswap) + ori r2,r2,0 + + vxor v3,v3,v11 + VPMSUMD(v11,v19,const2) + lvx v19,off48,r4 + VPERM(v19,v19,v19,byteswap) + lvx const2,0,r3 + ori r2,r2,0 + + vxor v4,v4,v12 + VPMSUMD(v12,v20,const1) + lvx v20,off64,r4 + VPERM(v20,v20,v20,byteswap) + ori r2,r2,0 + + vxor v5,v5,v13 + VPMSUMD(v13,v21,const1) + lvx v21,off80,r4 + VPERM(v21,v21,v21,byteswap) + ori r2,r2,0 + + vxor v6,v6,v14 + VPMSUMD(v14,v22,const1) + lvx v22,off96,r4 + VPERM(v22,v22,v22,byteswap) + ori r2,r2,0 + + vxor v7,v7,v15 + VPMSUMD(v15,v23,const1) + lvx v23,off112,r4 + VPERM(v23,v23,v23,byteswap) + + addi r4,r4,8*16 + + bdnz 4b + +.Lfirst_cool_down: + /* First cool down pass */ + lvx const1,0,r3 + addi r3,r3,16 + + vxor v0,v0,v8 + VPMSUMD(v8,v16,const1) + ori r2,r2,0 + + vxor v1,v1,v9 + VPMSUMD(v9,v17,const1) + ori r2,r2,0 + + vxor v2,v2,v10 + VPMSUMD(v10,v18,const1) + ori r2,r2,0 + + vxor v3,v3,v11 + VPMSUMD(v11,v19,const1) + ori r2,r2,0 + + vxor v4,v4,v12 + VPMSUMD(v12,v20,const1) + ori r2,r2,0 + + vxor v5,v5,v13 + VPMSUMD(v13,v21,const1) + ori r2,r2,0 + + vxor v6,v6,v14 + VPMSUMD(v14,v22,const1) + ori r2,r2,0 + + vxor v7,v7,v15 + VPMSUMD(v15,v23,const1) + ori r2,r2,0 + +.Lsecond_cool_down: + /* Second cool down pass */ + vxor v0,v0,v8 + vxor v1,v1,v9 + vxor v2,v2,v10 + vxor v3,v3,v11 + vxor v4,v4,v12 + vxor v5,v5,v13 + vxor v6,v6,v14 + vxor v7,v7,v15 + + /* + * vpmsumd produces a 96 bit result in the least significant bits + * of the register. Since we are bit reflected we have to shift it + * left 32 bits so it occupies the least significant bits in the + * bit reflected domain. + */ + vsldoi v0,v0,zeroes,4 + vsldoi v1,v1,zeroes,4 + vsldoi v2,v2,zeroes,4 + vsldoi v3,v3,zeroes,4 + vsldoi v4,v4,zeroes,4 + vsldoi v5,v5,zeroes,4 + vsldoi v6,v6,zeroes,4 + vsldoi v7,v7,zeroes,4 + + /* xor with last 1024 bits */ + lvx v8,0,r4 + lvx v9,off16,r4 + VPERM(v8,v8,v8,byteswap) + VPERM(v9,v9,v9,byteswap) + lvx v10,off32,r4 + lvx v11,off48,r4 + VPERM(v10,v10,v10,byteswap) + VPERM(v11,v11,v11,byteswap) + lvx v12,off64,r4 + lvx v13,off80,r4 + VPERM(v12,v12,v12,byteswap) + VPERM(v13,v13,v13,byteswap) + lvx v14,off96,r4 + lvx v15,off112,r4 + VPERM(v14,v14,v14,byteswap) + VPERM(v15,v15,v15,byteswap) + + addi r4,r4,8*16 + + vxor v16,v0,v8 + vxor v17,v1,v9 + vxor v18,v2,v10 + vxor v19,v3,v11 + vxor v20,v4,v12 + vxor v21,v5,v13 + vxor v22,v6,v14 + vxor v23,v7,v15 + + li r0,1 + cmpdi r6,0 + addi r6,r6,128 + bne 1b + + /* Work out how many bytes we have left */ + andi. r5,r5,127 + + /* Calculate where in the constant table we need to start */ + subfic r6,r5,128 + add r3,r3,r6 + + /* How many 16 byte chunks are in the tail */ + srdi r7,r5,4 + mtctr r7 + + /* + * Reduce the previously calculated 1024 bits to 64 bits, shifting + * 32 bits to include the trailing 32 bits of zeros + */ + lvx v0,0,r3 + lvx v1,off16,r3 + lvx v2,off32,r3 + lvx v3,off48,r3 + lvx v4,off64,r3 + lvx v5,off80,r3 + lvx v6,off96,r3 + lvx v7,off112,r3 + addi r3,r3,8*16 + + VPMSUMW(v0,v16,v0) + VPMSUMW(v1,v17,v1) + VPMSUMW(v2,v18,v2) + VPMSUMW(v3,v19,v3) + VPMSUMW(v4,v20,v4) + VPMSUMW(v5,v21,v5) + VPMSUMW(v6,v22,v6) + VPMSUMW(v7,v23,v7) + + /* Now reduce the tail (0 - 112 bytes) */ + cmpdi r7,0 + beq 1f + + lvx v16,0,r4 + lvx v17,0,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off16,r4 + lvx v17,off16,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off32,r4 + lvx v17,off32,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off48,r4 + lvx v17,off48,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off64,r4 + lvx v17,off64,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off80,r4 + lvx v17,off80,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off96,r4 + lvx v17,off96,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + + /* Now xor all the parallel chunks together */ +1: vxor v0,v0,v1 + vxor v2,v2,v3 + vxor v4,v4,v5 + vxor v6,v6,v7 + + vxor v0,v0,v2 + vxor v4,v4,v6 + + vxor v0,v0,v4 + +.Lbarrett_reduction: + /* Barrett constants */ + addis r3,r2,.barrett_constants@toc@ha + addi r3,r3,.barrett_constants@toc@l + + lvx const1,0,r3 + lvx const2,off16,r3 + + vsldoi v1,v0,v0,8 + vxor v0,v0,v1 /* xor two 64 bit results together */ + + /* shift left one bit */ + vspltisb v1,1 + vsl v0,v0,v1 + + vand v0,v0,mask_64bit + + /* + * The reflected version of Barrett reduction. Instead of bit + * reflecting our data (which is expensive to do), we bit reflect our + * constants and our algorithm, which means the intermediate data in + * our vector registers goes from 0-63 instead of 63-0. We can reflect + * the algorithm because we don't carry in mod 2 arithmetic. + */ + vand v1,v0,mask_32bit /* bottom 32 bits of a */ + VPMSUMD(v1,v1,const1) /* ma */ + vand v1,v1,mask_32bit /* bottom 32bits of ma */ + VPMSUMD(v1,v1,const2) /* qn */ + vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */ + + /* + * Since we are bit reflected, the result (ie the low 32 bits) is in + * the high 32 bits. We just need to shift it left 4 bytes + * V0 [ 0 1 X 3 ] + * V0 [ 0 X 2 3 ] + */ + vsldoi v0,v0,zeroes,4 /* shift result into top 64 bits of */ + + /* Get it into r3 */ + MFVRD(R3, v0) + +.Lout: + subi r6,r1,56+10*16 + subi r7,r1,56+2*16 + + lvx v20,0,r6 + lvx v21,off16,r6 + lvx v22,off32,r6 + lvx v23,off48,r6 + lvx v24,off64,r6 + lvx v25,off80,r6 + lvx v26,off96,r6 + lvx v27,off112,r6 + lvx v28,0,r7 + lvx v29,off16,r7 + + ld r31,-8(r1) + ld r30,-16(r1) + ld r29,-24(r1) + ld r28,-32(r1) + ld r27,-40(r1) + ld r26,-48(r1) + ld r25,-56(r1) + + blr + +.Lfirst_warm_up_done: + lvx const1,0,r3 + addi r3,r3,16 + + VPMSUMD(v8,v16,const1) + VPMSUMD(v9,v17,const1) + VPMSUMD(v10,v18,const1) + VPMSUMD(v11,v19,const1) + VPMSUMD(v12,v20,const1) + VPMSUMD(v13,v21,const1) + VPMSUMD(v14,v22,const1) + VPMSUMD(v15,v23,const1) + + b .Lsecond_cool_down + +.Lshort: + cmpdi r5,0 + beq .Lzero + + addis r3,r2,.short_constants@toc@ha + addi r3,r3,.short_constants@toc@l + + /* Calculate where in the constant table we need to start */ + subfic r6,r5,256 + add r3,r3,r6 + + /* How many 16 byte chunks? */ + srdi r7,r5,4 + mtctr r7 + + vxor v19,v19,v19 + vxor v20,v20,v20 + + lvx v0,0,r4 + lvx v16,0,r3 + VPERM(v0,v0,v16,byteswap) + vxor v0,v0,v8 /* xor in initial value */ + VPMSUMW(v0,v0,v16) + bdz .Lv0 + + lvx v1,off16,r4 + lvx v17,off16,r3 + VPERM(v1,v1,v17,byteswap) + VPMSUMW(v1,v1,v17) + bdz .Lv1 + + lvx v2,off32,r4 + lvx v16,off32,r3 + VPERM(v2,v2,v16,byteswap) + VPMSUMW(v2,v2,v16) + bdz .Lv2 + + lvx v3,off48,r4 + lvx v17,off48,r3 + VPERM(v3,v3,v17,byteswap) + VPMSUMW(v3,v3,v17) + bdz .Lv3 + + lvx v4,off64,r4 + lvx v16,off64,r3 + VPERM(v4,v4,v16,byteswap) + VPMSUMW(v4,v4,v16) + bdz .Lv4 + + lvx v5,off80,r4 + lvx v17,off80,r3 + VPERM(v5,v5,v17,byteswap) + VPMSUMW(v5,v5,v17) + bdz .Lv5 + + lvx v6,off96,r4 + lvx v16,off96,r3 + VPERM(v6,v6,v16,byteswap) + VPMSUMW(v6,v6,v16) + bdz .Lv6 + + lvx v7,off112,r4 + lvx v17,off112,r3 + VPERM(v7,v7,v17,byteswap) + VPMSUMW(v7,v7,v17) + bdz .Lv7 + + addi r3,r3,128 + addi r4,r4,128 + + lvx v8,0,r4 + lvx v16,0,r3 + VPERM(v8,v8,v16,byteswap) + VPMSUMW(v8,v8,v16) + bdz .Lv8 + + lvx v9,off16,r4 + lvx v17,off16,r3 + VPERM(v9,v9,v17,byteswap) + VPMSUMW(v9,v9,v17) + bdz .Lv9 + + lvx v10,off32,r4 + lvx v16,off32,r3 + VPERM(v10,v10,v16,byteswap) + VPMSUMW(v10,v10,v16) + bdz .Lv10 + + lvx v11,off48,r4 + lvx v17,off48,r3 + VPERM(v11,v11,v17,byteswap) + VPMSUMW(v11,v11,v17) + bdz .Lv11 + + lvx v12,off64,r4 + lvx v16,off64,r3 + VPERM(v12,v12,v16,byteswap) + VPMSUMW(v12,v12,v16) + bdz .Lv12 + + lvx v13,off80,r4 + lvx v17,off80,r3 + VPERM(v13,v13,v17,byteswap) + VPMSUMW(v13,v13,v17) + bdz .Lv13 + + lvx v14,off96,r4 + lvx v16,off96,r3 + VPERM(v14,v14,v16,byteswap) + VPMSUMW(v14,v14,v16) + bdz .Lv14 + + lvx v15,off112,r4 + lvx v17,off112,r3 + VPERM(v15,v15,v17,byteswap) + VPMSUMW(v15,v15,v17) + +.Lv15: vxor v19,v19,v15 +.Lv14: vxor v20,v20,v14 +.Lv13: vxor v19,v19,v13 +.Lv12: vxor v20,v20,v12 +.Lv11: vxor v19,v19,v11 +.Lv10: vxor v20,v20,v10 +.Lv9: vxor v19,v19,v9 +.Lv8: vxor v20,v20,v8 +.Lv7: vxor v19,v19,v7 +.Lv6: vxor v20,v20,v6 +.Lv5: vxor v19,v19,v5 +.Lv4: vxor v20,v20,v4 +.Lv3: vxor v19,v19,v3 +.Lv2: vxor v20,v20,v2 +.Lv1: vxor v19,v19,v1 +.Lv0: vxor v20,v20,v0 + + vxor v0,v19,v20 + + b .Lbarrett_reduction + +.Lzero: + mr r3,r10 + b .Lout + +FUNC_END(__crc32_vpmsum) diff --git a/arch/powerpc/crypto/crc32c-vpmsum_glue.c b/arch/powerpc/crypto/crc32c-vpmsum_glue.c new file mode 100644 index 000000000000..bfe3d37a24ef --- /dev/null +++ b/arch/powerpc/crypto/crc32c-vpmsum_glue.c @@ -0,0 +1,167 @@ +#include +#include +#include +#include +#include +#include +#include + +#define CHKSUM_BLOCK_SIZE 1 +#define CHKSUM_DIGEST_SIZE 4 + +#define VMX_ALIGN 16 +#define VMX_ALIGN_MASK (VMX_ALIGN-1) + +#define VECTOR_BREAKPOINT 512 + +u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len); + +static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len) +{ + unsigned int prealign; + unsigned int tail; + + if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || in_interrupt()) + return __crc32c_le(crc, p, len); + + if ((unsigned long)p & VMX_ALIGN_MASK) { + prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); + crc = __crc32c_le(crc, p, prealign); + len -= prealign; + p += prealign; + } + + if (len & ~VMX_ALIGN_MASK) { + pagefault_disable(); + enable_kernel_altivec(); + crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); + pagefault_enable(); + } + + tail = len & VMX_ALIGN_MASK; + if (tail) { + p += len & ~VMX_ALIGN_MASK; + crc = __crc32c_le(crc, p, tail); + } + + return crc; +} + +static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm) +{ + u32 *key = crypto_tfm_ctx(tfm); + + *key = 0; + + return 0; +} + +/* + * Setting the seed allows arbitrary accumulators and flexible XOR policy + * If your algorithm starts with ~0, then XOR with ~0 before you set + * the seed. + */ +static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key, + unsigned int keylen) +{ + u32 *mctx = crypto_shash_ctx(hash); + + if (keylen != sizeof(u32)) { + crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + *mctx = le32_to_cpup((__le32 *)key); + return 0; +} + +static int crc32c_vpmsum_init(struct shash_desc *desc) +{ + u32 *mctx = crypto_shash_ctx(desc->tfm); + u32 *crcp = shash_desc_ctx(desc); + + *crcp = *mctx; + + return 0; +} + +static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + u32 *crcp = shash_desc_ctx(desc); + + *crcp = crc32c_vpmsum(*crcp, data, len); + + return 0; +} + +static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len, + u8 *out) +{ + *(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len)); + + return 0; +} + +static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out); +} + +static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out) +{ + u32 *crcp = shash_desc_ctx(desc); + + *(__le32 *)out = ~cpu_to_le32p(crcp); + + return 0; +} + +static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len, + out); +} + +static struct shash_alg alg = { + .setkey = crc32c_vpmsum_setkey, + .init = crc32c_vpmsum_init, + .update = crc32c_vpmsum_update, + .final = crc32c_vpmsum_final, + .finup = crc32c_vpmsum_finup, + .digest = crc32c_vpmsum_digest, + .descsize = sizeof(u32), + .digestsize = CHKSUM_DIGEST_SIZE, + .base = { + .cra_name = "crc32c", + .cra_driver_name = "crc32c-vpmsum", + .cra_priority = 200, + .cra_blocksize = CHKSUM_BLOCK_SIZE, + .cra_ctxsize = sizeof(u32), + .cra_module = THIS_MODULE, + .cra_init = crc32c_vpmsum_cra_init, + } +}; + +static int __init crc32c_vpmsum_mod_init(void) +{ + if (!cpu_has_feature(CPU_FTR_ARCH_207S)) + return -ENODEV; + + return crypto_register_shash(&alg); +} + +static void __exit crc32c_vpmsum_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(crc32c_vpmsum_mod_init); +module_exit(crc32c_vpmsum_mod_fini); + +MODULE_AUTHOR("Anton Blanchard "); +MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_CRYPTO("crc32c"); +MODULE_ALIAS_CRYPTO("crc32c-vpmsum"); diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index ae0751ef8788..f08d567e0ca4 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -78,21 +78,53 @@ static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ return t; \ } +#define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ +static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ +{ \ + int res, t; \ + \ + __asm__ __volatile__( \ +"1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ + #asm_op " %1,%3,%0\n" \ + PPC405_ERR77(0, %4) \ +" stwcx. %1,0,%4\n" \ +" bne- 1b\n" \ + : "=&r" (res), "=&r" (t), "+m" (v->counter) \ + : "r" (a), "r" (&v->counter) \ + : "cc"); \ + \ + return res; \ +} + #define ATOMIC_OPS(op, asm_op) \ ATOMIC_OP(op, asm_op) \ - ATOMIC_OP_RETURN_RELAXED(op, asm_op) + ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ + ATOMIC_FETCH_OP_RELAXED(op, asm_op) ATOMIC_OPS(add, add) ATOMIC_OPS(sub, subf) -ATOMIC_OP(and, and) -ATOMIC_OP(or, or) -ATOMIC_OP(xor, xor) - #define atomic_add_return_relaxed atomic_add_return_relaxed #define atomic_sub_return_relaxed atomic_sub_return_relaxed +#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed +#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed + +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, asm_op) \ + ATOMIC_OP(op, asm_op) \ + ATOMIC_FETCH_OP_RELAXED(op, asm_op) + +ATOMIC_OPS(and, and) +ATOMIC_OPS(or, or) +ATOMIC_OPS(xor, xor) + +#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed +#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed +#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed + #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP_RELAXED #undef ATOMIC_OP_RETURN_RELAXED #undef ATOMIC_OP @@ -329,20 +361,53 @@ atomic64_##op##_return_relaxed(long a, atomic64_t *v) \ return t; \ } +#define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ +static inline long \ +atomic64_fetch_##op##_relaxed(long a, atomic64_t *v) \ +{ \ + long res, t; \ + \ + __asm__ __volatile__( \ +"1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ + #asm_op " %1,%3,%0\n" \ +" stdcx. %1,0,%4\n" \ +" bne- 1b\n" \ + : "=&r" (res), "=&r" (t), "+m" (v->counter) \ + : "r" (a), "r" (&v->counter) \ + : "cc"); \ + \ + return res; \ +} + #define ATOMIC64_OPS(op, asm_op) \ ATOMIC64_OP(op, asm_op) \ - ATOMIC64_OP_RETURN_RELAXED(op, asm_op) + ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ + ATOMIC64_FETCH_OP_RELAXED(op, asm_op) ATOMIC64_OPS(add, add) ATOMIC64_OPS(sub, subf) -ATOMIC64_OP(and, and) -ATOMIC64_OP(or, or) -ATOMIC64_OP(xor, xor) #define atomic64_add_return_relaxed atomic64_add_return_relaxed #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed +#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed +#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed + +#undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, asm_op) \ + ATOMIC64_OP(op, asm_op) \ + ATOMIC64_FETCH_OP_RELAXED(op, asm_op) + +ATOMIC64_OPS(and, and) +ATOMIC64_OPS(or, or) +ATOMIC64_OPS(xor, xor) + +#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed +#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed +#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed + #undef ATOPIC64_OPS +#undef ATOMIC64_FETCH_OP_RELAXED #undef ATOMIC64_OP_RETURN_RELAXED #undef ATOMIC64_OP diff --git a/arch/powerpc/include/asm/book3s/32/pgalloc.h b/arch/powerpc/include/asm/book3s/32/pgalloc.h index a2350194fc76..8e21bb492dca 100644 --- a/arch/powerpc/include/asm/book3s/32/pgalloc.h +++ b/arch/powerpc/include/asm/book3s/32/pgalloc.h @@ -102,7 +102,6 @@ static inline void pgtable_free_tlb(struct mmu_gather *tlb, static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, unsigned long address) { - tlb_flush_pgtable(tlb, address); pgtable_page_dtor(table); pgtable_free_tlb(tlb, page_address(table), 0); } diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index 290157e8d5b2..74839f24f412 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -88,6 +88,7 @@ #define HPTE_R_RPN_SHIFT 12 #define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) #define HPTE_R_PP ASM_CONST(0x0000000000000003) +#define HPTE_R_PPP ASM_CONST(0x8000000000000003) #define HPTE_R_N ASM_CONST(0x0000000000000004) #define HPTE_R_G ASM_CONST(0x0000000000000008) #define HPTE_R_M ASM_CONST(0x0000000000000010) diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h index 488279edb1f0..cd5e7aa8cc34 100644 --- a/arch/powerpc/include/asm/book3s/64/pgalloc.h +++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h @@ -41,7 +41,7 @@ extern struct kmem_cache *pgtable_cache[]; pgtable_cache[(shift) - 1]; \ }) -#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO +#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int); extern void pte_fragment_free(unsigned long *, int); @@ -56,7 +56,7 @@ static inline pgd_t *radix__pgd_alloc(struct mm_struct *mm) return (pgd_t *)__get_free_page(PGALLOC_GFP); #else struct page *page; - page = alloc_pages(PGALLOC_GFP, 4); + page = alloc_pages(PGALLOC_GFP | __GFP_REPEAT, 4); if (!page) return NULL; return (pgd_t *) page_address(page); @@ -93,8 +93,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE), - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE), GFP_KERNEL); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) @@ -110,13 +109,17 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, unsigned long address) { + /* + * By now all the pud entries should be none entries. So go + * ahead and flush the page walk cache + */ + flush_tlb_pgtable(tlb, address); pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE); } static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), GFP_KERNEL); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) @@ -127,6 +130,11 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, unsigned long address) { + /* + * By now all the pud entries should be none entries. So go + * ahead and flush the page walk cache + */ + flush_tlb_pgtable(tlb, address); return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX); } @@ -151,7 +159,7 @@ static inline pgtable_t pmd_pgtable(pmd_t pmd) static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); + return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm, @@ -198,7 +206,11 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, unsigned long address) { - tlb_flush_pgtable(tlb, address); + /* + * By now all the pud entries should be none entries. So go + * ahead and flush the page walk cache + */ + flush_tlb_pgtable(tlb, address); pgtable_free_tlb(tlb, table, 0); } diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 88a5ecaa157b..ab84c89c9e98 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -230,6 +230,7 @@ extern unsigned long __kernel_virt_size; #define KERN_VIRT_SIZE __kernel_virt_size extern struct page *vmemmap; extern unsigned long ioremap_bot; +extern unsigned long pci_io_base; #endif /* __ASSEMBLY__ */ #include diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h index 937d4e247ac3..df294224e280 100644 --- a/arch/powerpc/include/asm/book3s/64/radix.h +++ b/arch/powerpc/include/asm/book3s/64/radix.h @@ -228,5 +228,20 @@ extern void radix__vmemmap_remove_mapping(unsigned long start, extern int radix__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t flags, unsigned int psz); + +static inline unsigned long radix__get_tree_size(void) +{ + unsigned long rts_field; + /* + * we support 52 bits, hence 52-31 = 21, 0b10101 + * RTS encoding details + * bits 0 - 3 of rts -> bits 6 - 8 unsigned long + * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long + */ + rts_field = (0x5UL << 5); /* 6 - 8 bits */ + rts_field |= (0x2UL << 61); + + return rts_field; +} #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index 13ef38828dfe..3fa94fcac628 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -18,16 +18,19 @@ extern void radix__local_flush_tlb_mm(struct mm_struct *mm); extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); extern void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, unsigned long ap, int nid); +extern void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); extern void radix__tlb_flush(struct mmu_gather *tlb); #ifdef CONFIG_SMP extern void radix__flush_tlb_mm(struct mm_struct *mm); extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); extern void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, unsigned long ap, int nid); +extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); #else #define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm) #define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr) #define radix___flush_tlb_page(mm,addr,p,i) radix___local_flush_tlb_page(mm,addr,p,i) +#define radix__flush_tlb_pwc(tlb, addr) radix__local_flush_tlb_pwc(tlb, addr) #endif #endif diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index d98424ae356c..96e5769b18b0 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -72,5 +72,19 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) #define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) #endif /* CONFIG_SMP */ +/* + * flush the page walk cache for the address + */ +static inline void flush_tlb_pgtable(struct mmu_gather *tlb, unsigned long address) +{ + /* + * Flush the page table walk cache on freeing a page table. We already + * have marked the upper/higher level page table entry none by now. + * So it is safe to flush PWC here. + */ + if (!radix_enabled()) + return; + radix__flush_tlb_pwc(tlb, address); +} #endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/book3s/pgalloc.h b/arch/powerpc/include/asm/book3s/pgalloc.h index 54f591e9572e..c0a69ae92256 100644 --- a/arch/powerpc/include/asm/book3s/pgalloc.h +++ b/arch/powerpc/include/asm/book3s/pgalloc.h @@ -4,11 +4,6 @@ #include extern void tlb_remove_table(struct mmu_gather *tlb, void *table); -static inline void tlb_flush_pgtable(struct mmu_gather *tlb, - unsigned long address) -{ - -} #ifdef CONFIG_PPC64 #include diff --git a/arch/powerpc/include/asm/mutex.h b/arch/powerpc/include/asm/mutex.h index 127ab23e1f6c..078155fa1189 100644 --- a/arch/powerpc/include/asm/mutex.h +++ b/arch/powerpc/include/asm/mutex.h @@ -124,7 +124,7 @@ __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) static inline int __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) { - if (likely(__mutex_cmpxchg_lock(count, 1, 0) == 1)) + if (likely(atomic_read(count) == 1 && __mutex_cmpxchg_lock(count, 1, 0) == 1)) return 1; return 0; } diff --git a/arch/powerpc/include/asm/nohash/64/pgalloc.h b/arch/powerpc/include/asm/nohash/64/pgalloc.h index 069369f6414b..897d2e1c8a9b 100644 --- a/arch/powerpc/include/asm/nohash/64/pgalloc.h +++ b/arch/powerpc/include/asm/nohash/64/pgalloc.h @@ -57,8 +57,7 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE), - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE), GFP_KERNEL); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) @@ -88,7 +87,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); + return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm, @@ -190,8 +189,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), GFP_KERNEL); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index ee09e99097f0..9bd87f269d6d 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -71,10 +71,8 @@ pte_t *__find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, static inline pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, bool *is_thp, unsigned *shift) { - if (!arch_irqs_disabled()) { - pr_info("%s called with irq enabled\n", __func__); - dump_stack(); - } + VM_WARN(!arch_irqs_disabled(), + "%s called with irq enabled\n", __func__); return __find_linux_pte_or_hugepte(pgdir, ea, is_thp, shift); } diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 1d035c1cc889..49cd8760aa7c 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -174,6 +174,8 @@ #define PPC_INST_MFSPR_DSCR_USER_MASK 0xfc1fffff #define PPC_INST_MTSPR_DSCR_USER 0x7c0303a6 #define PPC_INST_MTSPR_DSCR_USER_MASK 0xfc1fffff +#define PPC_INST_MFVSRD 0x7c000066 +#define PPC_INST_MTVSRD 0x7c000166 #define PPC_INST_SLBFEE 0x7c0007a7 #define PPC_INST_STRING 0x7c00042a @@ -188,6 +190,8 @@ #define PPC_INST_WAIT 0x7c00007c #define PPC_INST_TLBIVAX 0x7c000624 #define PPC_INST_TLBSRX_DOT 0x7c0006a5 +#define PPC_INST_VPMSUMW 0x10000488 +#define PPC_INST_VPMSUMD 0x100004c8 #define PPC_INST_XXLOR 0xf0000510 #define PPC_INST_XXSWAPD 0xf0000250 #define PPC_INST_XVCPSGNDP 0xf0000780 @@ -359,6 +363,14 @@ VSX_XX1((s), a, b)) #define LXVD2X(s, a, b) stringify_in_c(.long PPC_INST_LXVD2X | \ VSX_XX1((s), a, b)) +#define MFVRD(a, t) stringify_in_c(.long PPC_INST_MFVSRD | \ + VSX_XX1((t)+32, a, R0)) +#define MTVRD(t, a) stringify_in_c(.long PPC_INST_MTVSRD | \ + VSX_XX1((t)+32, a, R0)) +#define VPMSUMW(t, a, b) stringify_in_c(.long PPC_INST_VPMSUMW | \ + VSX_XX3((t), a, b)) +#define VPMSUMD(t, a, b) stringify_in_c(.long PPC_INST_VPMSUMD | \ + VSX_XX3((t), a, b)) #define XXLOR(t, a, b) stringify_in_c(.long PPC_INST_XXLOR | \ VSX_XX3((t), a, b)) #define XXSWAPD(t, a) stringify_in_c(.long PPC_INST_XXSWAPD | \ diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index 2b31632376a5..051af612a7e1 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -286,6 +286,9 @@ n: #endif +#define FUNC_START(name) _GLOBAL(name) +#define FUNC_END(name) + /* * LOAD_REG_IMMEDIATE(rn, expr) * Loads the value of the constant expression 'expr' into register 'rn' diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 2714a3b81d24..d70101e1e25c 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -642,13 +642,12 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, if (pe->type & EEH_PE_VF) { eeh_pe_dev_traverse(pe, eeh_rmv_device, NULL); } else { - eeh_pe_state_clear(pe, EEH_PE_PRI_BUS); pci_lock_rescan_remove(); pci_hp_remove_devices(bus); pci_unlock_rescan_remove(); } } else if (frozen_bus) { - eeh_pe_dev_traverse(pe, eeh_rmv_device, &rmv_data); + eeh_pe_dev_traverse(pe, eeh_rmv_device, rmv_data); } /* @@ -692,10 +691,12 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, */ edev = list_first_entry(&pe->edevs, struct eeh_dev, list); eeh_pe_traverse(pe, eeh_pe_detach_dev, NULL); - if (pe->type & EEH_PE_VF) + if (pe->type & EEH_PE_VF) { eeh_add_virt_device(edev, NULL); - else + } else { + eeh_pe_state_clear(pe, EEH_PE_PRI_BUS); pci_hp_add_devices(bus); + } } else if (frozen_bus && rmv_data->removed) { pr_info("EEH: Sleep 5s ahead of partial hotplug\n"); ssleep(5); diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 4c9440629128..8bcc1b457115 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1399,11 +1399,12 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_RADIX) lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */ mtlr r10 -BEGIN_MMU_FTR_SECTION - b 2f -END_MMU_FTR_SECTION_IFSET(MMU_FTR_RADIX) andi. r10,r12,MSR_RI /* check for unrecoverable exception */ +BEGIN_MMU_FTR_SECTION beq- 2f +FTR_SECTION_ELSE + b 2f +ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_RADIX) .machine push .machine "power4" diff --git a/arch/powerpc/kernel/iomap.c b/arch/powerpc/kernel/iomap.c index 12e48d56f771..3963f0b68d52 100644 --- a/arch/powerpc/kernel/iomap.c +++ b/arch/powerpc/kernel/iomap.c @@ -38,6 +38,18 @@ EXPORT_SYMBOL(ioread16); EXPORT_SYMBOL(ioread16be); EXPORT_SYMBOL(ioread32); EXPORT_SYMBOL(ioread32be); +#ifdef __powerpc64__ +u64 ioread64(void __iomem *addr) +{ + return readq(addr); +} +u64 ioread64be(void __iomem *addr) +{ + return readq_be(addr); +} +EXPORT_SYMBOL(ioread64); +EXPORT_SYMBOL(ioread64be); +#endif /* __powerpc64__ */ void iowrite8(u8 val, void __iomem *addr) { @@ -64,6 +76,18 @@ EXPORT_SYMBOL(iowrite16); EXPORT_SYMBOL(iowrite16be); EXPORT_SYMBOL(iowrite32); EXPORT_SYMBOL(iowrite32be); +#ifdef __powerpc64__ +void iowrite64(u64 val, void __iomem *addr) +{ + writeq(val, addr); +} +void iowrite64be(u64 val, void __iomem *addr) +{ + writeq_be(val, addr); +} +EXPORT_SYMBOL(iowrite64); +EXPORT_SYMBOL(iowrite64be); +#endif /* __powerpc64__ */ /* * These are the "repeat read/write" functions. Note the diff --git a/arch/powerpc/kernel/nvram_64.c b/arch/powerpc/kernel/nvram_64.c index 856f9a7944cd..64174bf95611 100644 --- a/arch/powerpc/kernel/nvram_64.c +++ b/arch/powerpc/kernel/nvram_64.c @@ -444,7 +444,8 @@ static int nvram_pstore_write(enum pstore_type_id type, */ static ssize_t nvram_pstore_read(u64 *id, enum pstore_type_id *type, int *count, struct timespec *time, char **buf, - bool *compressed, struct pstore_info *psi) + bool *compressed, ssize_t *ecc_notice_size, + struct pstore_info *psi) { struct oops_log_info *oops_hdr; unsigned int err_type, id_no, size = 0; @@ -545,6 +546,7 @@ static ssize_t nvram_pstore_read(u64 *id, enum pstore_type_id *type, return -ENOMEM; kfree(buff); + *ecc_notice_size = 0; if (err_type == ERR_TYPE_KERNEL_PANIC_GZ) *compressed = true; else diff --git a/arch/powerpc/kernel/pci_64.c b/arch/powerpc/kernel/pci_64.c index 3759df52bd67..a5ae49a2dcc4 100644 --- a/arch/powerpc/kernel/pci_64.c +++ b/arch/powerpc/kernel/pci_64.c @@ -47,7 +47,6 @@ static int __init pcibios_init(void) printk(KERN_INFO "PCI: Probing PCI hardware\n"); - pci_io_base = ISA_IO_BASE; /* For now, override phys_mem_access_prot. If we need it,g * later, we may move that initialization to each ppc_md */ diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index e2f12cbcade9..0b93893424f5 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -1505,6 +1505,16 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp) current->thread.regs = regs - 1; } +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + /* + * Clear any transactional state, we're exec()ing. The cause is + * not important as there will never be a recheckpoint so it's not + * user visible. + */ + if (MSR_TM_SUSPENDED(mfmsr())) + tm_reclaim_current(0); +#endif + memset(regs->gpr, 0, sizeof(regs->gpr)); regs->ctr = 0; regs->link = 0; diff --git a/arch/powerpc/kernel/tm.S b/arch/powerpc/kernel/tm.S index bf8f34a58670..b7019b559ddb 100644 --- a/arch/powerpc/kernel/tm.S +++ b/arch/powerpc/kernel/tm.S @@ -110,17 +110,11 @@ _GLOBAL(tm_reclaim) std r3, STK_PARAM(R3)(r1) SAVE_NVGPRS(r1) - /* We need to setup MSR for VSX register save instructions. Here we - * also clear the MSR RI since when we do the treclaim, we won't have a - * valid kernel pointer for a while. We clear RI here as it avoids - * adding another mtmsr closer to the treclaim. This makes the region - * maked as non-recoverable wider than it needs to be but it saves on - * inserting another mtmsrd later. - */ + /* We need to setup MSR for VSX register save instructions. */ mfmsr r14 mr r15, r14 ori r15, r15, MSR_FP - li r16, MSR_RI + li r16, 0 ori r16, r16, MSR_EE /* IRQs hard off */ andc r15, r15, r16 oris r15, r15, MSR_VEC@h @@ -176,7 +170,17 @@ dont_backup_fp: 1: tdeqi r6, 0 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0 - /* The moment we treclaim, ALL of our GPRs will switch + /* Clear MSR RI since we are about to change r1, EE is already off. */ + li r4, 0 + mtmsrd r4, 1 + + /* + * BE CAREFUL HERE: + * At this point we can't take an SLB miss since we have MSR_RI + * off. Load only to/from the stack/paca which are in SLB bolted regions + * until we turn MSR RI back on. + * + * The moment we treclaim, ALL of our GPRs will switch * to user register state. (FPRs, CCR etc. also!) * Use an sprg and a tm_scratch in the PACA to shuffle. */ @@ -197,6 +201,11 @@ dont_backup_fp: /* Store the PPR in r11 and reset to decent value */ std r11, GPR11(r1) /* Temporary stash */ + + /* Reset MSR RI so we can take SLB faults again */ + li r11, MSR_RI + mtmsrd r11, 1 + mfspr r11, SPRN_PPR HMT_MEDIUM @@ -397,11 +406,6 @@ restore_gprs: ld r5, THREAD_TM_DSCR(r3) ld r6, THREAD_TM_PPR(r3) - /* Clear the MSR RI since we are about to change R1. EE is already off - */ - li r4, 0 - mtmsrd r4, 1 - REST_GPR(0, r7) /* GPR0 */ REST_2GPRS(2, r7) /* GPR2-3 */ REST_GPR(4, r7) /* GPR4 */ @@ -439,10 +443,33 @@ restore_gprs: ld r6, _CCR(r7) mtcr r6 - REST_GPR(1, r7) /* GPR1 */ - REST_GPR(5, r7) /* GPR5-7 */ REST_GPR(6, r7) - ld r7, GPR7(r7) + + /* + * Store r1 and r5 on the stack so that we can access them + * after we clear MSR RI. + */ + + REST_GPR(5, r7) + std r5, -8(r1) + ld r5, GPR1(r7) + std r5, -16(r1) + + REST_GPR(7, r7) + + /* Clear MSR RI since we are about to change r1. EE is already off */ + li r5, 0 + mtmsrd r5, 1 + + /* + * BE CAREFUL HERE: + * At this point we can't take an SLB miss since we have MSR_RI + * off. Load only to/from the stack/paca which are in SLB bolted regions + * until we turn MSR RI back on. + */ + + ld r5, -8(r1) + ld r1, -16(r1) /* Commit register state as checkpointed state: */ TRECHKPT diff --git a/arch/powerpc/mm/copro_fault.c b/arch/powerpc/mm/copro_fault.c index 6527882ce05e..bb0354222b11 100644 --- a/arch/powerpc/mm/copro_fault.c +++ b/arch/powerpc/mm/copro_fault.c @@ -75,7 +75,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, } ret = 0; - *flt = handle_mm_fault(mm, vma, ea, is_write ? FAULT_FLAG_WRITE : 0); + *flt = handle_mm_fault(vma, ea, is_write ? FAULT_FLAG_WRITE : 0); if (unlikely(*flt & VM_FAULT_ERROR)) { if (*flt & VM_FAULT_OOM) { ret = -ENOMEM; diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index a67c6d781c52..a4db22f65021 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -429,7 +429,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if (unlikely(fault & (VM_FAULT_RETRY|VM_FAULT_ERROR))) { if (fault & VM_FAULT_SIGSEGV) goto bad_area; diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c index 40e05e7f43de..f8a871a72985 100644 --- a/arch/powerpc/mm/hash_native_64.c +++ b/arch/powerpc/mm/hash_native_64.c @@ -316,8 +316,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, DBG_LOW(" -> hit\n"); /* Update the HPTE */ hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & - ~(HPTE_R_PP | HPTE_R_N)) | - (newpp & (HPTE_R_PP | HPTE_R_N | + ~(HPTE_R_PPP | HPTE_R_N)) | + (newpp & (HPTE_R_PPP | HPTE_R_N | HPTE_R_C))); } native_unlock_hpte(hptep); @@ -385,8 +385,8 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, /* Update the HPTE */ hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & - ~(HPTE_R_PP | HPTE_R_N)) | - (newpp & (HPTE_R_PP | HPTE_R_N))); + ~(HPTE_R_PPP | HPTE_R_N)) | + (newpp & (HPTE_R_PPP | HPTE_R_N))); /* * Ensure it is out of the tlb too. Bolted entries base and * actual page size will be same. diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index b2740c67e172..2971ea18c768 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -201,9 +201,8 @@ unsigned long htab_convert_pte_flags(unsigned long pteflags) /* * We can't allow hardware to update hpte bits. Hence always * set 'R' bit and set 'C' if it is a write fault - * Memory coherence is always enabled */ - rflags |= HPTE_R_R | HPTE_R_M; + rflags |= HPTE_R_R; if (pteflags & _PAGE_DIRTY) rflags |= HPTE_R_C; @@ -213,10 +212,15 @@ unsigned long htab_convert_pte_flags(unsigned long pteflags) if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_TOLERANT) rflags |= HPTE_R_I; - if ((pteflags & _PAGE_CACHE_CTL ) == _PAGE_NON_IDEMPOTENT) + else if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_NON_IDEMPOTENT) rflags |= (HPTE_R_I | HPTE_R_G); - if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_SAO) - rflags |= (HPTE_R_I | HPTE_R_W); + else if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_SAO) + rflags |= (HPTE_R_W | HPTE_R_I | HPTE_R_M); + else + /* + * Add memory coherence if cache inhibited is not set + */ + rflags |= HPTE_R_M; return rflags; } @@ -918,6 +922,10 @@ void __init hash__early_init_mmu(void) vmemmap = (struct page *)H_VMEMMAP_BASE; ioremap_bot = IOREMAP_BASE; +#ifdef CONFIG_PCI + pci_io_base = ISA_IO_BASE; +#endif + /* Initialize the MMU Hash table and create the linear mapping * of memory. Has to be done before SLB initialization as this is * currently where the page size encoding is obtained. diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 5aac1a3f86cd..119d18611500 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -73,7 +73,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, cachep = PGT_CACHE(pdshift - pshift); #endif - new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT); + new = kmem_cache_zalloc(cachep, GFP_KERNEL); BUG_ON(pshift > HUGEPD_SHIFT_MASK); BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK); diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c index 227b2a6c4544..196222227e82 100644 --- a/arch/powerpc/mm/mmu_context_book3s64.c +++ b/arch/powerpc/mm/mmu_context_book3s64.c @@ -65,7 +65,7 @@ static int radix__init_new_context(struct mm_struct *mm, int index) /* * set the process table entry, */ - rts_field = 3ull << PPC_BITLSHIFT(2); + rts_field = radix__get_tree_size(); process_tb[index].prtb0 = cpu_to_be64(rts_field | __pa(mm->pgd) | RADIX_PGD_INDEX_SIZE); return 0; } diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c index c939e6e57a9e..7931e1496f0d 100644 --- a/arch/powerpc/mm/pgtable-radix.c +++ b/arch/powerpc/mm/pgtable-radix.c @@ -160,9 +160,8 @@ redo: process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT); /* * Fill in the process table. - * we support 52 bits, hence 52-28 = 24, 11000 */ - rts_field = 3ull << PPC_BITLSHIFT(2); + rts_field = radix__get_tree_size(); process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE); /* * Fill in the partition table. We are suppose to use effective address @@ -176,10 +175,8 @@ redo: static void __init radix_init_partition_table(void) { unsigned long rts_field; - /* - * we support 52 bits, hence 52-28 = 24, 11000 - */ - rts_field = 3ull << PPC_BITLSHIFT(2); + + rts_field = radix__get_tree_size(); BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large."); partition_tb = early_alloc_pgtable(1UL << PATB_SIZE_SHIFT); @@ -331,6 +328,11 @@ void __init radix__early_init_mmu(void) __vmalloc_end = RADIX_VMALLOC_END; vmemmap = (struct page *)RADIX_VMEMMAP_BASE; ioremap_bot = IOREMAP_BASE; + +#ifdef CONFIG_PCI + pci_io_base = ISA_IO_BASE; +#endif + /* * For now radix also use the same frag size */ diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index bf7bf32b54f8..7f922f557936 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -84,7 +84,7 @@ __init_refok pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long add pte_t *pte; if (slab_is_available()) { - pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); } else { pte = __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE)); if (pte) @@ -97,7 +97,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *ptepage; - gfp_t flags = GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO; + gfp_t flags = GFP_KERNEL | __GFP_ZERO; ptepage = alloc_pages(flags, 0); if (!ptepage) diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index e009e0604a8a..f5e8d4edb808 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -350,8 +350,7 @@ static pte_t *get_from_cache(struct mm_struct *mm) static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel) { void *ret = NULL; - struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | - __GFP_REPEAT | __GFP_ZERO); + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO); if (!page) return NULL; if (!kernel && !pgtable_page_ctor(page)) { diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c index 54efba2fd66e..ab2f60e812e2 100644 --- a/arch/powerpc/mm/tlb-radix.c +++ b/arch/powerpc/mm/tlb-radix.c @@ -18,16 +18,20 @@ static DEFINE_RAW_SPINLOCK(native_tlbie_lock); -static inline void __tlbiel_pid(unsigned long pid, int set) +#define RIC_FLUSH_TLB 0 +#define RIC_FLUSH_PWC 1 +#define RIC_FLUSH_ALL 2 + +static inline void __tlbiel_pid(unsigned long pid, int set, + unsigned long ric) { - unsigned long rb,rs,ric,prs,r; + unsigned long rb,rs,prs,r; rb = PPC_BIT(53); /* IS = 1 */ rb |= set << PPC_BITLSHIFT(51); rs = ((unsigned long)pid) << PPC_BITLSHIFT(31); prs = 1; /* process scoped */ r = 1; /* raidx format */ - ric = 2; /* invalidate all the caches */ asm volatile("ptesync": : :"memory"); asm volatile(".long 0x7c000224 | (%0 << 11) | (%1 << 16) |" @@ -39,25 +43,24 @@ static inline void __tlbiel_pid(unsigned long pid, int set) /* * We use 128 set in radix mode and 256 set in hpt mode. */ -static inline void _tlbiel_pid(unsigned long pid) +static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) { int set; for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) { - __tlbiel_pid(pid, set); + __tlbiel_pid(pid, set, ric); } return; } -static inline void _tlbie_pid(unsigned long pid) +static inline void _tlbie_pid(unsigned long pid, unsigned long ric) { - unsigned long rb,rs,ric,prs,r; + unsigned long rb,rs,prs,r; rb = PPC_BIT(53); /* IS = 1 */ rs = pid << PPC_BITLSHIFT(31); prs = 1; /* process scoped */ r = 1; /* raidx format */ - ric = 2; /* invalidate all the caches */ asm volatile("ptesync": : :"memory"); asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 16) |" @@ -67,16 +70,15 @@ static inline void _tlbie_pid(unsigned long pid) } static inline void _tlbiel_va(unsigned long va, unsigned long pid, - unsigned long ap) + unsigned long ap, unsigned long ric) { - unsigned long rb,rs,ric,prs,r; + unsigned long rb,rs,prs,r; rb = va & ~(PPC_BITMASK(52, 63)); rb |= ap << PPC_BITLSHIFT(58); rs = pid << PPC_BITLSHIFT(31); prs = 1; /* process scoped */ r = 1; /* raidx format */ - ric = 0; /* no cluster flush yet */ asm volatile("ptesync": : :"memory"); asm volatile(".long 0x7c000224 | (%0 << 11) | (%1 << 16) |" @@ -86,16 +88,15 @@ static inline void _tlbiel_va(unsigned long va, unsigned long pid, } static inline void _tlbie_va(unsigned long va, unsigned long pid, - unsigned long ap) + unsigned long ap, unsigned long ric) { - unsigned long rb,rs,ric,prs,r; + unsigned long rb,rs,prs,r; rb = va & ~(PPC_BITMASK(52, 63)); rb |= ap << PPC_BITLSHIFT(58); rs = pid << PPC_BITLSHIFT(31); prs = 1; /* process scoped */ r = 1; /* raidx format */ - ric = 0; /* no cluster flush yet */ asm volatile("ptesync": : :"memory"); asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 16) |" @@ -122,11 +123,26 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm) preempt_disable(); pid = mm->context.id; if (pid != MMU_NO_CONTEXT) - _tlbiel_pid(pid); + _tlbiel_pid(pid, RIC_FLUSH_ALL); preempt_enable(); } EXPORT_SYMBOL(radix__local_flush_tlb_mm); +void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr) +{ + unsigned long pid; + struct mm_struct *mm = tlb->mm; + + preempt_disable(); + + pid = mm->context.id; + if (pid != MMU_NO_CONTEXT) + _tlbiel_pid(pid, RIC_FLUSH_PWC); + + preempt_enable(); +} +EXPORT_SYMBOL(radix__local_flush_tlb_pwc); + void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, unsigned long ap, int nid) { @@ -135,7 +151,7 @@ void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, preempt_disable(); pid = mm ? mm->context.id : 0; if (pid != MMU_NO_CONTEXT) - _tlbiel_va(vmaddr, pid, ap); + _tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB); preempt_enable(); } @@ -172,16 +188,42 @@ void radix__flush_tlb_mm(struct mm_struct *mm) if (lock_tlbie) raw_spin_lock(&native_tlbie_lock); - _tlbie_pid(pid); + _tlbie_pid(pid, RIC_FLUSH_ALL); if (lock_tlbie) raw_spin_unlock(&native_tlbie_lock); } else - _tlbiel_pid(pid); + _tlbiel_pid(pid, RIC_FLUSH_ALL); no_context: preempt_enable(); } EXPORT_SYMBOL(radix__flush_tlb_mm); +void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr) +{ + unsigned long pid; + struct mm_struct *mm = tlb->mm; + + preempt_disable(); + + pid = mm->context.id; + if (unlikely(pid == MMU_NO_CONTEXT)) + goto no_context; + + if (!mm_is_core_local(mm)) { + int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); + + if (lock_tlbie) + raw_spin_lock(&native_tlbie_lock); + _tlbie_pid(pid, RIC_FLUSH_PWC); + if (lock_tlbie) + raw_spin_unlock(&native_tlbie_lock); + } else + _tlbiel_pid(pid, RIC_FLUSH_PWC); +no_context: + preempt_enable(); +} +EXPORT_SYMBOL(radix__flush_tlb_pwc); + void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, unsigned long ap, int nid) { @@ -196,11 +238,11 @@ void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, if (lock_tlbie) raw_spin_lock(&native_tlbie_lock); - _tlbie_va(vmaddr, pid, ap); + _tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB); if (lock_tlbie) raw_spin_unlock(&native_tlbie_lock); } else - _tlbiel_va(vmaddr, pid, ap); + _tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB); bail: preempt_enable(); } @@ -224,7 +266,7 @@ void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) if (lock_tlbie) raw_spin_lock(&native_tlbie_lock); - _tlbie_pid(0); + _tlbie_pid(0, RIC_FLUSH_ALL); if (lock_tlbie) raw_spin_unlock(&native_tlbie_lock); } diff --git a/arch/powerpc/platforms/40x/Kconfig b/arch/powerpc/platforms/40x/Kconfig index 6e287f1294fa..e3257f24a8a1 100644 --- a/arch/powerpc/platforms/40x/Kconfig +++ b/arch/powerpc/platforms/40x/Kconfig @@ -137,7 +137,7 @@ config STB03xxx config PPC4xx_GPIO bool "PPC4xx GPIO support" depends on 40x - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB help Enable gpiolib support for ppc40x based boards diff --git a/arch/powerpc/platforms/44x/Kconfig b/arch/powerpc/platforms/44x/Kconfig index 5538e57c36c1..48fc18041ff6 100644 --- a/arch/powerpc/platforms/44x/Kconfig +++ b/arch/powerpc/platforms/44x/Kconfig @@ -273,7 +273,7 @@ config PPC44x_SIMPLE config PPC4xx_GPIO bool "PPC4xx GPIO support" depends on 44x - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB help Enable gpiolib support for ppc440 based boards diff --git a/arch/powerpc/platforms/512x/Kconfig b/arch/powerpc/platforms/512x/Kconfig index f09016f6b3a6..bf7ae5cbd07a 100644 --- a/arch/powerpc/platforms/512x/Kconfig +++ b/arch/powerpc/platforms/512x/Kconfig @@ -6,7 +6,6 @@ config PPC_MPC512x select IPIC select PPC_PCI_CHOICE select FSL_PCI if PCI - select ARCH_WANT_OPTIONAL_GPIOLIB select USB_EHCI_BIG_ENDIAN_MMIO if USB_EHCI_HCD select USB_EHCI_BIG_ENDIAN_DESC if USB_EHCI_HCD diff --git a/arch/powerpc/platforms/83xx/Kconfig b/arch/powerpc/platforms/83xx/Kconfig index 2bdc8c862c46..4ef7f1cd05b7 100644 --- a/arch/powerpc/platforms/83xx/Kconfig +++ b/arch/powerpc/platforms/83xx/Kconfig @@ -116,7 +116,6 @@ endif # used for usb & gpio config PPC_MPC831x bool - select ARCH_WANT_OPTIONAL_GPIOLIB # used for math-emu config PPC_MPC832x @@ -125,9 +124,7 @@ config PPC_MPC832x # used for usb & gpio config PPC_MPC834x bool - select ARCH_WANT_OPTIONAL_GPIOLIB # used for usb & gpio config PPC_MPC837x bool - select ARCH_WANT_OPTIONAL_GPIOLIB diff --git a/arch/powerpc/platforms/85xx/Kconfig b/arch/powerpc/platforms/85xx/Kconfig index e626461a63bd..df25a3ed489d 100644 --- a/arch/powerpc/platforms/85xx/Kconfig +++ b/arch/powerpc/platforms/85xx/Kconfig @@ -225,7 +225,7 @@ config GE_IMP3A select DEFAULT_UIMAGE select SWIOTLB select MMIO_NVRAM - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select GE_FPGA help This option enables support for the GE Intelligent Platforms IMP3A @@ -272,7 +272,7 @@ config CORENET_GENERIC select PPC_E500MC select PHYS_64BIT select SWIOTLB - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select GPIO_MPC8XXX select HAS_RAPIDIO select PPC_EPAPR_HV_PIC diff --git a/arch/powerpc/platforms/86xx/Kconfig b/arch/powerpc/platforms/86xx/Kconfig index 1afd1e4a2dd2..3988f16e46c1 100644 --- a/arch/powerpc/platforms/86xx/Kconfig +++ b/arch/powerpc/platforms/86xx/Kconfig @@ -4,7 +4,6 @@ menuconfig PPC_86xx depends on 6xx select FSL_SOC select ALTIVEC - select ARCH_WANT_OPTIONAL_GPIOLIB help The Freescale E600 SoCs have 74xx cores. @@ -37,7 +36,7 @@ config GEF_PPC9A bool "GE PPC9A" select DEFAULT_UIMAGE select MMIO_NVRAM - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select GE_FPGA help This option enables support for the GE PPC9A. @@ -46,7 +45,7 @@ config GEF_SBC310 bool "GE SBC310" select DEFAULT_UIMAGE select MMIO_NVRAM - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select GE_FPGA help This option enables support for the GE SBC310. @@ -55,7 +54,7 @@ config GEF_SBC610 bool "GE SBC610" select DEFAULT_UIMAGE select MMIO_NVRAM - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select GE_FPGA select HAS_RAPIDIO help diff --git a/arch/powerpc/platforms/8xx/Kconfig b/arch/powerpc/platforms/8xx/Kconfig index 157250426b56..564d99bb2a26 100644 --- a/arch/powerpc/platforms/8xx/Kconfig +++ b/arch/powerpc/platforms/8xx/Kconfig @@ -109,7 +109,7 @@ config 8xx_COPYBACK config 8xx_GPIO bool "GPIO API Support" - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB help Saying Y here will cause the ports on an MPC8xx processor to be used with the GPIO API. If you say N here, the kernel needs less memory. diff --git a/arch/powerpc/platforms/Kconfig b/arch/powerpc/platforms/Kconfig index 46a3533d3acb..3663f71fd913 100644 --- a/arch/powerpc/platforms/Kconfig +++ b/arch/powerpc/platforms/Kconfig @@ -275,7 +275,7 @@ config TAU_AVERAGE config QE_GPIO bool "QE GPIO support" depends on QUICC_ENGINE - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB help Say Y here if you're going to use hardware that connects to the QE GPIOs. @@ -285,7 +285,7 @@ config CPM2 depends on (FSL_SOC_BOOKE && PPC32) || 8260 select CPM select PPC_PCI_CHOICE - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB help The CPM2 (Communications Processor Module) is a coprocessor on embedded CPUs made by Freescale. Selecting this option means that @@ -324,7 +324,7 @@ config OF_RTC config SIMPLE_GPIO bool "Support for simple, memory-mapped GPIO controllers" depends on PPC - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB help Say Y here to support simple, memory-mapped GPIO controllers. These are usually BCSRs used to control board's switches, LEDs, @@ -334,7 +334,7 @@ config SIMPLE_GPIO config MCU_MPC8349EMITX bool "MPC8349E-mITX MCU driver" depends on I2C=y && PPC_83xx - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB help Say Y here to enable soft power-off functionality on the Freescale boards with the MPC8349E-mITX-compatible MCU chips. This driver will diff --git a/arch/powerpc/platforms/cell/cpufreq_spudemand.c b/arch/powerpc/platforms/cell/cpufreq_spudemand.c index 82607d621aca..88301e53f085 100644 --- a/arch/powerpc/platforms/cell/cpufreq_spudemand.c +++ b/arch/powerpc/platforms/cell/cpufreq_spudemand.c @@ -85,61 +85,57 @@ static void spu_gov_cancel_work(struct spu_gov_info_struct *info) cancel_delayed_work_sync(&info->work); } -static int spu_gov_govern(struct cpufreq_policy *policy, unsigned int event) +static int spu_gov_start(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; - struct spu_gov_info_struct *info, *affected_info; + struct spu_gov_info_struct *info = &per_cpu(spu_gov_info, cpu); + struct spu_gov_info_struct *affected_info; int i; - int ret = 0; - info = &per_cpu(spu_gov_info, cpu); - - switch (event) { - case CPUFREQ_GOV_START: - if (!cpu_online(cpu)) { - printk(KERN_ERR "cpu %d is not online\n", cpu); - ret = -EINVAL; - break; - } + if (!cpu_online(cpu)) { + printk(KERN_ERR "cpu %d is not online\n", cpu); + return -EINVAL; + } - if (!policy->cur) { - printk(KERN_ERR "no cpu specified in policy\n"); - ret = -EINVAL; - break; - } + if (!policy->cur) { + printk(KERN_ERR "no cpu specified in policy\n"); + return -EINVAL; + } - /* initialize spu_gov_info for all affected cpus */ - for_each_cpu(i, policy->cpus) { - affected_info = &per_cpu(spu_gov_info, i); - affected_info->policy = policy; - } + /* initialize spu_gov_info for all affected cpus */ + for_each_cpu(i, policy->cpus) { + affected_info = &per_cpu(spu_gov_info, i); + affected_info->policy = policy; + } - info->poll_int = POLL_TIME; + info->poll_int = POLL_TIME; - /* setup timer */ - spu_gov_init_work(info); + /* setup timer */ + spu_gov_init_work(info); - break; + return 0; +} - case CPUFREQ_GOV_STOP: - /* cancel timer */ - spu_gov_cancel_work(info); +static void spu_gov_stop(struct cpufreq_policy *policy) +{ + unsigned int cpu = policy->cpu; + struct spu_gov_info_struct *info = &per_cpu(spu_gov_info, cpu); + int i; - /* clean spu_gov_info for all affected cpus */ - for_each_cpu (i, policy->cpus) { - info = &per_cpu(spu_gov_info, i); - info->policy = NULL; - } + /* cancel timer */ + spu_gov_cancel_work(info); - break; + /* clean spu_gov_info for all affected cpus */ + for_each_cpu (i, policy->cpus) { + info = &per_cpu(spu_gov_info, i); + info->policy = NULL; } - - return ret; } static struct cpufreq_governor spu_governor = { .name = "spudemand", - .governor = spu_gov_govern, + .start = spu_gov_start, + .stop = spu_gov_stop, .owner = THIS_MODULE, }; diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c index ff75d70f7285..f9af6461521a 100644 --- a/arch/powerpc/sysdev/axonram.c +++ b/arch/powerpc/sysdev/axonram.c @@ -223,7 +223,6 @@ static int axon_ram_probe(struct platform_device *device) bank->disk->first_minor = azfs_minor; bank->disk->fops = &axon_ram_devops; bank->disk->private_data = bank; - bank->disk->driverfs_dev = &device->dev; sprintf(bank->disk->disk_name, "%s%d", AXON_RAM_DEVICE_NAME, axon_ram_bank_id); @@ -238,7 +237,7 @@ static int axon_ram_probe(struct platform_device *device) set_capacity(bank->disk, bank->size >> AXON_RAM_SECTOR_SHIFT); blk_queue_make_request(bank->disk->queue, axon_ram_make_request); blk_queue_logical_block_size(bank->disk->queue, AXON_RAM_SECTOR_SIZE); - add_disk(bank->disk); + device_add_disk(&device->dev, bank->disk); bank->irq_id = irq_of_parse_and_map(device->dev.of_node, 0); if (bank->irq_id == NO_IRQ) { diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index a8c259059adf..9e607bf2d640 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -72,6 +72,7 @@ config S390 select ARCH_HAS_DEVMEM_IS_ALLOWED select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_HAS_KCOV select ARCH_HAS_SG_CHAIN select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_INLINE_READ_LOCK @@ -163,6 +164,7 @@ config S390 select NO_BOOTMEM select OLD_SIGACTION select OLD_SIGSUSPEND3 + select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE select TTY select VIRT_CPU_ACCOUNTING @@ -477,6 +479,9 @@ config SCHED_MC config SCHED_BOOK def_bool n +config SCHED_DRAWER + def_bool n + config SCHED_TOPOLOGY def_bool y prompt "Topology scheduler support" @@ -484,6 +489,7 @@ config SCHED_TOPOLOGY select SCHED_SMT select SCHED_MC select SCHED_BOOK + select SCHED_DRAWER help Topology scheduler support improves the CPU scheduler's decision making when dealing with machines that have multi-threading, @@ -605,16 +611,6 @@ config PCI_NR_FUNCTIONS This allows you to specify the maximum number of PCI functions which this kernel will support. -config PCI_NR_MSI - int "Maximum number of MSI interrupts (64-32768)" - range 64 32768 - default "256" - help - This defines the number of virtual interrupts the kernel will - provide for MSI interrupts. If you configure your system to have - too few drivers will fail to allocate MSI interrupts for all - PCI devices. - source "drivers/pci/Kconfig" endif # PCI diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile index 1dd210347e12..98ec652cc332 100644 --- a/arch/s390/boot/compressed/Makefile +++ b/arch/s390/boot/compressed/Makefile @@ -4,6 +4,8 @@ # create a compressed vmlinux image from the original vmlinux # +KCOV_INSTRUMENT := n + targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 targets += misc.o piggy.o sizes.h head.o diff --git a/arch/s390/configs/default_defconfig b/arch/s390/configs/default_defconfig index d5ec71b2ed02..889ea3450210 100644 --- a/arch/s390/configs/default_defconfig +++ b/arch/s390/configs/default_defconfig @@ -678,6 +678,7 @@ CONFIG_CRYPTO_SHA512_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_GHASH_S390=m +CONFIG_CRYPTO_CRC32_S390=m CONFIG_ASYMMETRIC_KEY_TYPE=y CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=m CONFIG_X509_CERTIFICATE_PARSER=m diff --git a/arch/s390/configs/gcov_defconfig b/arch/s390/configs/gcov_defconfig index f46a35115d2d..1bcfd764910a 100644 --- a/arch/s390/configs/gcov_defconfig +++ b/arch/s390/configs/gcov_defconfig @@ -616,6 +616,7 @@ CONFIG_CRYPTO_SHA512_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_GHASH_S390=m +CONFIG_CRYPTO_CRC32_S390=m CONFIG_ASYMMETRIC_KEY_TYPE=y CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=m CONFIG_X509_CERTIFICATE_PARSER=m diff --git a/arch/s390/configs/performance_defconfig b/arch/s390/configs/performance_defconfig index ba0f2a58b8cd..13ff090139c8 100644 --- a/arch/s390/configs/performance_defconfig +++ b/arch/s390/configs/performance_defconfig @@ -615,6 +615,7 @@ CONFIG_CRYPTO_SHA512_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_GHASH_S390=m +CONFIG_CRYPTO_CRC32_S390=m CONFIG_ASYMMETRIC_KEY_TYPE=y CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=m CONFIG_X509_CERTIFICATE_PARSER=m diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 7f0b7cda6259..d1033de4c4ee 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -9,3 +9,6 @@ obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o +obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o + +crc32-vx_s390-y := crc32-vx.o crc32le-vx.o crc32be-vx.o diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 7554a8bb2adc..2ea18b050309 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -44,7 +45,7 @@ struct s390_aes_ctx { long dec; int key_len; union { - struct crypto_blkcipher *blk; + struct crypto_skcipher *blk; struct crypto_cipher *cip; } fallback; }; @@ -63,7 +64,7 @@ struct s390_xts_ctx { long enc; long dec; int key_len; - struct crypto_blkcipher *fallback; + struct crypto_skcipher *fallback; }; /* @@ -237,16 +238,16 @@ static int setkey_fallback_blk(struct crypto_tfm *tfm, const u8 *key, struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); unsigned int ret; - sctx->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - sctx->fallback.blk->base.crt_flags |= (tfm->crt_flags & - CRYPTO_TFM_REQ_MASK); + crypto_skcipher_clear_flags(sctx->fallback.blk, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(sctx->fallback.blk, tfm->crt_flags & + CRYPTO_TFM_REQ_MASK); + + ret = crypto_skcipher_setkey(sctx->fallback.blk, key, len); + + tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->crt_flags |= crypto_skcipher_get_flags(sctx->fallback.blk) & + CRYPTO_TFM_RES_MASK; - ret = crypto_blkcipher_setkey(sctx->fallback.blk, key, len); - if (ret) { - tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm->crt_flags |= (sctx->fallback.blk->base.crt_flags & - CRYPTO_TFM_RES_MASK); - } return ret; } @@ -255,15 +256,17 @@ static int fallback_blk_dec(struct blkcipher_desc *desc, unsigned int nbytes) { unsigned int ret; - struct crypto_blkcipher *tfm; - struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct crypto_blkcipher *tfm = desc->tfm; + struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(tfm); + SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk); - tfm = desc->tfm; - desc->tfm = sctx->fallback.blk; + skcipher_request_set_tfm(req, sctx->fallback.blk); + skcipher_request_set_callback(req, desc->flags, NULL, NULL); + skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes); + ret = crypto_skcipher_decrypt(req); - desc->tfm = tfm; + skcipher_request_zero(req); return ret; } @@ -272,15 +275,15 @@ static int fallback_blk_enc(struct blkcipher_desc *desc, unsigned int nbytes) { unsigned int ret; - struct crypto_blkcipher *tfm; - struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); + struct crypto_blkcipher *tfm = desc->tfm; + struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(tfm); + SKCIPHER_REQUEST_ON_STACK(req, sctx->fallback.blk); - tfm = desc->tfm; - desc->tfm = sctx->fallback.blk; + skcipher_request_set_tfm(req, sctx->fallback.blk); + skcipher_request_set_callback(req, desc->flags, NULL, NULL); + skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes); - - desc->tfm = tfm; + ret = crypto_skcipher_encrypt(req); return ret; } @@ -370,8 +373,9 @@ static int fallback_init_blk(struct crypto_tfm *tfm) const char *name = tfm->__crt_alg->cra_name; struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - sctx->fallback.blk = crypto_alloc_blkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + sctx->fallback.blk = crypto_alloc_skcipher(name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(sctx->fallback.blk)) { pr_err("Allocating AES fallback algorithm %s failed\n", @@ -386,8 +390,7 @@ static void fallback_exit_blk(struct crypto_tfm *tfm) { struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm); - crypto_free_blkcipher(sctx->fallback.blk); - sctx->fallback.blk = NULL; + crypto_free_skcipher(sctx->fallback.blk); } static struct crypto_alg ecb_aes_alg = { @@ -536,16 +539,16 @@ static int xts_fallback_setkey(struct crypto_tfm *tfm, const u8 *key, struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm); unsigned int ret; - xts_ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - xts_ctx->fallback->base.crt_flags |= (tfm->crt_flags & - CRYPTO_TFM_REQ_MASK); + crypto_skcipher_clear_flags(xts_ctx->fallback, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(xts_ctx->fallback, tfm->crt_flags & + CRYPTO_TFM_REQ_MASK); + + ret = crypto_skcipher_setkey(xts_ctx->fallback, key, len); + + tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->crt_flags |= crypto_skcipher_get_flags(xts_ctx->fallback) & + CRYPTO_TFM_RES_MASK; - ret = crypto_blkcipher_setkey(xts_ctx->fallback, key, len); - if (ret) { - tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm->crt_flags |= (xts_ctx->fallback->base.crt_flags & - CRYPTO_TFM_RES_MASK); - } return ret; } @@ -553,16 +556,18 @@ static int xts_fallback_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); - struct crypto_blkcipher *tfm; + struct crypto_blkcipher *tfm = desc->tfm; + struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(tfm); + SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback); unsigned int ret; - tfm = desc->tfm; - desc->tfm = xts_ctx->fallback; + skcipher_request_set_tfm(req, xts_ctx->fallback); + skcipher_request_set_callback(req, desc->flags, NULL, NULL); + skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes); + ret = crypto_skcipher_decrypt(req); - desc->tfm = tfm; + skcipher_request_zero(req); return ret; } @@ -570,16 +575,18 @@ static int xts_fallback_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm); - struct crypto_blkcipher *tfm; + struct crypto_blkcipher *tfm = desc->tfm; + struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(tfm); + SKCIPHER_REQUEST_ON_STACK(req, xts_ctx->fallback); unsigned int ret; - tfm = desc->tfm; - desc->tfm = xts_ctx->fallback; + skcipher_request_set_tfm(req, xts_ctx->fallback); + skcipher_request_set_callback(req, desc->flags, NULL, NULL); + skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes); + ret = crypto_skcipher_encrypt(req); - desc->tfm = tfm; + skcipher_request_zero(req); return ret; } @@ -700,8 +707,9 @@ static int xts_fallback_init(struct crypto_tfm *tfm) const char *name = tfm->__crt_alg->cra_name; struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm); - xts_ctx->fallback = crypto_alloc_blkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + xts_ctx->fallback = crypto_alloc_skcipher(name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(xts_ctx->fallback)) { pr_err("Allocating XTS fallback algorithm %s failed\n", @@ -715,8 +723,7 @@ static void xts_fallback_exit(struct crypto_tfm *tfm) { struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm); - crypto_free_blkcipher(xts_ctx->fallback); - xts_ctx->fallback = NULL; + crypto_free_skcipher(xts_ctx->fallback); } static struct crypto_alg xts_aes_alg = { diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c new file mode 100644 index 000000000000..577ae1d4ae89 --- /dev/null +++ b/arch/s390/crypto/crc32-vx.c @@ -0,0 +1,310 @@ +/* + * Crypto-API module for CRC-32 algorithms implemented with the + * z/Architecture Vector Extension Facility. + * + * Copyright IBM Corp. 2015 + * Author(s): Hendrik Brueckner + */ +#define KMSG_COMPONENT "crc32-vx" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include +#include +#include +#include +#include + + +#define CRC32_BLOCK_SIZE 1 +#define CRC32_DIGEST_SIZE 4 + +#define VX_MIN_LEN 64 +#define VX_ALIGNMENT 16L +#define VX_ALIGN_MASK (VX_ALIGNMENT - 1) + +struct crc_ctx { + u32 key; +}; + +struct crc_desc_ctx { + u32 crc; +}; + +/* Prototypes for functions in assembly files */ +u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); +u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size); +u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); + +/* + * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension + * + * Creates a function to perform a particular CRC-32 computation. Depending + * on the message buffer, the hardware-accelerated or software implementation + * is used. Note that the message buffer is aligned to improve fetch + * operations of VECTOR LOAD MULTIPLE instructions. + * + */ +#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \ + static u32 __pure ___fname(u32 crc, \ + unsigned char const *data, size_t datalen) \ + { \ + struct kernel_fpu vxstate; \ + unsigned long prealign, aligned, remaining; \ + \ + if ((unsigned long)data & VX_ALIGN_MASK) { \ + prealign = VX_ALIGNMENT - \ + ((unsigned long)data & VX_ALIGN_MASK); \ + datalen -= prealign; \ + crc = ___crc32_sw(crc, data, prealign); \ + data = (void *)((unsigned long)data + prealign); \ + } \ + \ + if (datalen < VX_MIN_LEN) \ + return ___crc32_sw(crc, data, datalen); \ + \ + aligned = datalen & ~VX_ALIGN_MASK; \ + remaining = datalen & VX_ALIGN_MASK; \ + \ + kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \ + crc = ___crc32_vx(crc, data, aligned); \ + kernel_fpu_end(&vxstate); \ + \ + if (remaining) \ + crc = ___crc32_sw(crc, data + aligned, remaining); \ + \ + return crc; \ + } + +DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le) +DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be) +DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le) + + +static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm) +{ + struct crc_ctx *mctx = crypto_tfm_ctx(tfm); + + mctx->key = 0; + return 0; +} + +static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm) +{ + struct crc_ctx *mctx = crypto_tfm_ctx(tfm); + + mctx->key = ~0; + return 0; +} + +static int crc32_vx_init(struct shash_desc *desc) +{ + struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm); + struct crc_desc_ctx *ctx = shash_desc_ctx(desc); + + ctx->crc = mctx->key; + return 0; +} + +static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey, + unsigned int newkeylen) +{ + struct crc_ctx *mctx = crypto_shash_ctx(tfm); + + if (newkeylen != sizeof(mctx->key)) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + mctx->key = le32_to_cpu(*(__le32 *)newkey); + return 0; +} + +static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey, + unsigned int newkeylen) +{ + struct crc_ctx *mctx = crypto_shash_ctx(tfm); + + if (newkeylen != sizeof(mctx->key)) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + mctx->key = be32_to_cpu(*(__be32 *)newkey); + return 0; +} + +static int crc32le_vx_final(struct shash_desc *desc, u8 *out) +{ + struct crc_desc_ctx *ctx = shash_desc_ctx(desc); + + *(__le32 *)out = cpu_to_le32p(&ctx->crc); + return 0; +} + +static int crc32be_vx_final(struct shash_desc *desc, u8 *out) +{ + struct crc_desc_ctx *ctx = shash_desc_ctx(desc); + + *(__be32 *)out = cpu_to_be32p(&ctx->crc); + return 0; +} + +static int crc32c_vx_final(struct shash_desc *desc, u8 *out) +{ + struct crc_desc_ctx *ctx = shash_desc_ctx(desc); + + /* + * Perform a final XOR with 0xFFFFFFFF to be in sync + * with the generic crc32c shash implementation. + */ + *(__le32 *)out = ~cpu_to_le32p(&ctx->crc); + return 0; +} + +static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len, + u8 *out) +{ + *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len)); + return 0; +} + +static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len, + u8 *out) +{ + *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len)); + return 0; +} + +static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len, + u8 *out) +{ + /* + * Perform a final XOR with 0xFFFFFFFF to be in sync + * with the generic crc32c shash implementation. + */ + *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len)); + return 0; +} + + +#define CRC32_VX_FINUP(alg, func) \ + static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data, \ + unsigned int datalen, u8 *out) \ + { \ + return __ ## alg ## _vx_finup(shash_desc_ctx(desc), \ + data, datalen, out); \ + } + +CRC32_VX_FINUP(crc32le, crc32_le_vx) +CRC32_VX_FINUP(crc32be, crc32_be_vx) +CRC32_VX_FINUP(crc32c, crc32c_le_vx) + +#define CRC32_VX_DIGEST(alg, func) \ + static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \ + unsigned int len, u8 *out) \ + { \ + return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm), \ + data, len, out); \ + } + +CRC32_VX_DIGEST(crc32le, crc32_le_vx) +CRC32_VX_DIGEST(crc32be, crc32_be_vx) +CRC32_VX_DIGEST(crc32c, crc32c_le_vx) + +#define CRC32_VX_UPDATE(alg, func) \ + static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \ + unsigned int datalen) \ + { \ + struct crc_desc_ctx *ctx = shash_desc_ctx(desc); \ + ctx->crc = func(ctx->crc, data, datalen); \ + return 0; \ + } + +CRC32_VX_UPDATE(crc32le, crc32_le_vx) +CRC32_VX_UPDATE(crc32be, crc32_be_vx) +CRC32_VX_UPDATE(crc32c, crc32c_le_vx) + + +static struct shash_alg crc32_vx_algs[] = { + /* CRC-32 LE */ + { + .init = crc32_vx_init, + .setkey = crc32_vx_setkey, + .update = crc32le_vx_update, + .final = crc32le_vx_final, + .finup = crc32le_vx_finup, + .digest = crc32le_vx_digest, + .descsize = sizeof(struct crc_desc_ctx), + .digestsize = CRC32_DIGEST_SIZE, + .base = { + .cra_name = "crc32", + .cra_driver_name = "crc32-vx", + .cra_priority = 200, + .cra_blocksize = CRC32_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crc_ctx), + .cra_module = THIS_MODULE, + .cra_init = crc32_vx_cra_init_zero, + }, + }, + /* CRC-32 BE */ + { + .init = crc32_vx_init, + .setkey = crc32be_vx_setkey, + .update = crc32be_vx_update, + .final = crc32be_vx_final, + .finup = crc32be_vx_finup, + .digest = crc32be_vx_digest, + .descsize = sizeof(struct crc_desc_ctx), + .digestsize = CRC32_DIGEST_SIZE, + .base = { + .cra_name = "crc32be", + .cra_driver_name = "crc32be-vx", + .cra_priority = 200, + .cra_blocksize = CRC32_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crc_ctx), + .cra_module = THIS_MODULE, + .cra_init = crc32_vx_cra_init_zero, + }, + }, + /* CRC-32C LE */ + { + .init = crc32_vx_init, + .setkey = crc32_vx_setkey, + .update = crc32c_vx_update, + .final = crc32c_vx_final, + .finup = crc32c_vx_finup, + .digest = crc32c_vx_digest, + .descsize = sizeof(struct crc_desc_ctx), + .digestsize = CRC32_DIGEST_SIZE, + .base = { + .cra_name = "crc32c", + .cra_driver_name = "crc32c-vx", + .cra_priority = 200, + .cra_blocksize = CRC32_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crc_ctx), + .cra_module = THIS_MODULE, + .cra_init = crc32_vx_cra_init_invert, + }, + }, +}; + + +static int __init crc_vx_mod_init(void) +{ + return crypto_register_shashes(crc32_vx_algs, + ARRAY_SIZE(crc32_vx_algs)); +} + +static void __exit crc_vx_mod_exit(void) +{ + crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs)); +} + +module_cpu_feature_match(VXRS, crc_vx_mod_init); +module_exit(crc_vx_mod_exit); + +MODULE_AUTHOR("Hendrik Brueckner "); +MODULE_LICENSE("GPL"); + +MODULE_ALIAS_CRYPTO("crc32"); +MODULE_ALIAS_CRYPTO("crc32-vx"); +MODULE_ALIAS_CRYPTO("crc32c"); +MODULE_ALIAS_CRYPTO("crc32c-vx"); diff --git a/arch/s390/crypto/crc32be-vx.S b/arch/s390/crypto/crc32be-vx.S new file mode 100644 index 000000000000..8013989cd2e5 --- /dev/null +++ b/arch/s390/crypto/crc32be-vx.S @@ -0,0 +1,207 @@ +/* + * Hardware-accelerated CRC-32 variants for Linux on z Systems + * + * Use the z/Architecture Vector Extension Facility to accelerate the + * computing of CRC-32 checksums. + * + * This CRC-32 implementation algorithm processes the most-significant + * bit first (BE). + * + * Copyright IBM Corp. 2015 + * Author(s): Hendrik Brueckner + */ + +#include +#include + +/* Vector register range containing CRC-32 constants */ +#define CONST_R1R2 %v9 +#define CONST_R3R4 %v10 +#define CONST_R5 %v11 +#define CONST_R6 %v12 +#define CONST_RU_POLY %v13 +#define CONST_CRC_POLY %v14 + +.data +.align 8 + +/* + * The CRC-32 constant block contains reduction constants to fold and + * process particular chunks of the input data stream in parallel. + * + * For the CRC-32 variants, the constants are precomputed according to + * these defintions: + * + * R1 = x4*128+64 mod P(x) + * R2 = x4*128 mod P(x) + * R3 = x128+64 mod P(x) + * R4 = x128 mod P(x) + * R5 = x96 mod P(x) + * R6 = x64 mod P(x) + * + * Barret reduction constant, u, is defined as floor(x**64 / P(x)). + * + * where P(x) is the polynomial in the normal domain and the P'(x) is the + * polynomial in the reversed (bitreflected) domain. + * + * Note that the constant definitions below are extended in order to compute + * intermediate results with a single VECTOR GALOIS FIELD MULTIPLY instruction. + * The righmost doubleword can be 0 to prevent contribution to the result or + * can be multiplied by 1 to perform an XOR without the need for a separate + * VECTOR EXCLUSIVE OR instruction. + * + * CRC-32 (IEEE 802.3 Ethernet, ...) polynomials: + * + * P(x) = 0x04C11DB7 + * P'(x) = 0xEDB88320 + */ + +.Lconstants_CRC_32_BE: + .quad 0x08833794c, 0x0e6228b11 # R1, R2 + .quad 0x0c5b9cd4c, 0x0e8a45605 # R3, R4 + .quad 0x0f200aa66, 1 << 32 # R5, x32 + .quad 0x0490d678d, 1 # R6, 1 + .quad 0x104d101df, 0 # u + .quad 0x104C11DB7, 0 # P(x) + +.previous + +.text +/* + * The CRC-32 function(s) use these calling conventions: + * + * Parameters: + * + * %r2: Initial CRC value, typically ~0; and final CRC (return) value. + * %r3: Input buffer pointer, performance might be improved if the + * buffer is on a doubleword boundary. + * %r4: Length of the buffer, must be 64 bytes or greater. + * + * Register usage: + * + * %r5: CRC-32 constant pool base pointer. + * V0: Initial CRC value and intermediate constants and results. + * V1..V4: Data for CRC computation. + * V5..V8: Next data chunks that are fetched from the input buffer. + * + * V9..V14: CRC-32 constants. + */ +ENTRY(crc32_be_vgfm_16) + /* Load CRC-32 constants */ + larl %r5,.Lconstants_CRC_32_BE + VLM CONST_R1R2,CONST_CRC_POLY,0,%r5 + + /* Load the initial CRC value into the leftmost word of V0. */ + VZERO %v0 + VLVGF %v0,%r2,0 + + /* Load a 64-byte data chunk and XOR with CRC */ + VLM %v1,%v4,0,%r3 /* 64-bytes into V1..V4 */ + VX %v1,%v0,%v1 /* V1 ^= CRC */ + aghi %r3,64 /* BUF = BUF + 64 */ + aghi %r4,-64 /* LEN = LEN - 64 */ + + /* Check remaining buffer size and jump to proper folding method */ + cghi %r4,64 + jl .Lless_than_64bytes + +.Lfold_64bytes_loop: + /* Load the next 64-byte data chunk into V5 to V8 */ + VLM %v5,%v8,0,%r3 + + /* + * Perform a GF(2) multiplication of the doublewords in V1 with + * the reduction constants in V0. The intermediate result is + * then folded (accumulated) with the next data chunk in V5 and + * stored in V1. Repeat this step for the register contents + * in V2, V3, and V4 respectively. + */ + VGFMAG %v1,CONST_R1R2,%v1,%v5 + VGFMAG %v2,CONST_R1R2,%v2,%v6 + VGFMAG %v3,CONST_R1R2,%v3,%v7 + VGFMAG %v4,CONST_R1R2,%v4,%v8 + + /* Adjust buffer pointer and length for next loop */ + aghi %r3,64 /* BUF = BUF + 64 */ + aghi %r4,-64 /* LEN = LEN - 64 */ + + cghi %r4,64 + jnl .Lfold_64bytes_loop + +.Lless_than_64bytes: + /* Fold V1 to V4 into a single 128-bit value in V1 */ + VGFMAG %v1,CONST_R3R4,%v1,%v2 + VGFMAG %v1,CONST_R3R4,%v1,%v3 + VGFMAG %v1,CONST_R3R4,%v1,%v4 + + /* Check whether to continue with 64-bit folding */ + cghi %r4,16 + jl .Lfinal_fold + +.Lfold_16bytes_loop: + + VL %v2,0,,%r3 /* Load next data chunk */ + VGFMAG %v1,CONST_R3R4,%v1,%v2 /* Fold next data chunk */ + + /* Adjust buffer pointer and size for folding next data chunk */ + aghi %r3,16 + aghi %r4,-16 + + /* Process remaining data chunks */ + cghi %r4,16 + jnl .Lfold_16bytes_loop + +.Lfinal_fold: + /* + * The R5 constant is used to fold a 128-bit value into an 96-bit value + * that is XORed with the next 96-bit input data chunk. To use a single + * VGFMG instruction, multiply the rightmost 64-bit with x^32 (1<<32) to + * form an intermediate 96-bit value (with appended zeros) which is then + * XORed with the intermediate reduction result. + */ + VGFMG %v1,CONST_R5,%v1 + + /* + * Further reduce the remaining 96-bit value to a 64-bit value using a + * single VGFMG, the rightmost doubleword is multiplied with 0x1. The + * intermediate result is then XORed with the product of the leftmost + * doubleword with R6. The result is a 64-bit value and is subject to + * the Barret reduction. + */ + VGFMG %v1,CONST_R6,%v1 + + /* + * The input values to the Barret reduction are the degree-63 polynomial + * in V1 (R(x)), degree-32 generator polynomial, and the reduction + * constant u. The Barret reduction result is the CRC value of R(x) mod + * P(x). + * + * The Barret reduction algorithm is defined as: + * + * 1. T1(x) = floor( R(x) / x^32 ) GF2MUL u + * 2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x) + * 3. C(x) = R(x) XOR T2(x) mod x^32 + * + * Note: To compensate the division by x^32, use the vector unpack + * instruction to move the leftmost word into the leftmost doubleword + * of the vector register. The rightmost doubleword is multiplied + * with zero to not contribute to the intermedate results. + */ + + /* T1(x) = floor( R(x) / x^32 ) GF2MUL u */ + VUPLLF %v2,%v1 + VGFMG %v2,CONST_RU_POLY,%v2 + + /* + * Compute the GF(2) product of the CRC polynomial in VO with T1(x) in + * V2 and XOR the intermediate result, T2(x), with the value in V1. + * The final result is in the rightmost word of V2. + */ + VUPLLF %v2,%v2 + VGFMAG %v2,CONST_CRC_POLY,%v2,%v1 + +.Ldone: + VLGVF %r2,%v2,3 + br %r14 + +.previous diff --git a/arch/s390/crypto/crc32le-vx.S b/arch/s390/crypto/crc32le-vx.S new file mode 100644 index 000000000000..17f2504c2633 --- /dev/null +++ b/arch/s390/crypto/crc32le-vx.S @@ -0,0 +1,268 @@ +/* + * Hardware-accelerated CRC-32 variants for Linux on z Systems + * + * Use the z/Architecture Vector Extension Facility to accelerate the + * computing of bitreflected CRC-32 checksums for IEEE 802.3 Ethernet + * and Castagnoli. + * + * This CRC-32 implementation algorithm is bitreflected and processes + * the least-significant bit first (Little-Endian). + * + * Copyright IBM Corp. 2015 + * Author(s): Hendrik Brueckner + */ + +#include +#include + +/* Vector register range containing CRC-32 constants */ +#define CONST_PERM_LE2BE %v9 +#define CONST_R2R1 %v10 +#define CONST_R4R3 %v11 +#define CONST_R5 %v12 +#define CONST_RU_POLY %v13 +#define CONST_CRC_POLY %v14 + +.data +.align 8 + +/* + * The CRC-32 constant block contains reduction constants to fold and + * process particular chunks of the input data stream in parallel. + * + * For the CRC-32 variants, the constants are precomputed according to + * these definitions: + * + * R1 = [(x4*128+32 mod P'(x) << 32)]' << 1 + * R2 = [(x4*128-32 mod P'(x) << 32)]' << 1 + * R3 = [(x128+32 mod P'(x) << 32)]' << 1 + * R4 = [(x128-32 mod P'(x) << 32)]' << 1 + * R5 = [(x64 mod P'(x) << 32)]' << 1 + * R6 = [(x32 mod P'(x) << 32)]' << 1 + * + * The bitreflected Barret reduction constant, u', is defined as + * the bit reversal of floor(x**64 / P(x)). + * + * where P(x) is the polynomial in the normal domain and the P'(x) is the + * polynomial in the reversed (bitreflected) domain. + * + * CRC-32 (IEEE 802.3 Ethernet, ...) polynomials: + * + * P(x) = 0x04C11DB7 + * P'(x) = 0xEDB88320 + * + * CRC-32C (Castagnoli) polynomials: + * + * P(x) = 0x1EDC6F41 + * P'(x) = 0x82F63B78 + */ + +.Lconstants_CRC_32_LE: + .octa 0x0F0E0D0C0B0A09080706050403020100 # BE->LE mask + .quad 0x1c6e41596, 0x154442bd4 # R2, R1 + .quad 0x0ccaa009e, 0x1751997d0 # R4, R3 + .octa 0x163cd6124 # R5 + .octa 0x1F7011641 # u' + .octa 0x1DB710641 # P'(x) << 1 + +.Lconstants_CRC_32C_LE: + .octa 0x0F0E0D0C0B0A09080706050403020100 # BE->LE mask + .quad 0x09e4addf8, 0x740eef02 # R2, R1 + .quad 0x14cd00bd6, 0xf20c0dfe # R4, R3 + .octa 0x0dd45aab8 # R5 + .octa 0x0dea713f1 # u' + .octa 0x105ec76f0 # P'(x) << 1 + +.previous + + +.text + +/* + * The CRC-32 functions use these calling conventions: + * + * Parameters: + * + * %r2: Initial CRC value, typically ~0; and final CRC (return) value. + * %r3: Input buffer pointer, performance might be improved if the + * buffer is on a doubleword boundary. + * %r4: Length of the buffer, must be 64 bytes or greater. + * + * Register usage: + * + * %r5: CRC-32 constant pool base pointer. + * V0: Initial CRC value and intermediate constants and results. + * V1..V4: Data for CRC computation. + * V5..V8: Next data chunks that are fetched from the input buffer. + * V9: Constant for BE->LE conversion and shift operations + * + * V10..V14: CRC-32 constants. + */ + +ENTRY(crc32_le_vgfm_16) + larl %r5,.Lconstants_CRC_32_LE + j crc32_le_vgfm_generic + +ENTRY(crc32c_le_vgfm_16) + larl %r5,.Lconstants_CRC_32C_LE + j crc32_le_vgfm_generic + + +crc32_le_vgfm_generic: + /* Load CRC-32 constants */ + VLM CONST_PERM_LE2BE,CONST_CRC_POLY,0,%r5 + + /* + * Load the initial CRC value. + * + * The CRC value is loaded into the rightmost word of the + * vector register and is later XORed with the LSB portion + * of the loaded input data. + */ + VZERO %v0 /* Clear V0 */ + VLVGF %v0,%r2,3 /* Load CRC into rightmost word */ + + /* Load a 64-byte data chunk and XOR with CRC */ + VLM %v1,%v4,0,%r3 /* 64-bytes into V1..V4 */ + VPERM %v1,%v1,%v1,CONST_PERM_LE2BE + VPERM %v2,%v2,%v2,CONST_PERM_LE2BE + VPERM %v3,%v3,%v3,CONST_PERM_LE2BE + VPERM %v4,%v4,%v4,CONST_PERM_LE2BE + + VX %v1,%v0,%v1 /* V1 ^= CRC */ + aghi %r3,64 /* BUF = BUF + 64 */ + aghi %r4,-64 /* LEN = LEN - 64 */ + + cghi %r4,64 + jl .Lless_than_64bytes + +.Lfold_64bytes_loop: + /* Load the next 64-byte data chunk into V5 to V8 */ + VLM %v5,%v8,0,%r3 + VPERM %v5,%v5,%v5,CONST_PERM_LE2BE + VPERM %v6,%v6,%v6,CONST_PERM_LE2BE + VPERM %v7,%v7,%v7,CONST_PERM_LE2BE + VPERM %v8,%v8,%v8,CONST_PERM_LE2BE + + /* + * Perform a GF(2) multiplication of the doublewords in V1 with + * the R1 and R2 reduction constants in V0. The intermediate result + * is then folded (accumulated) with the next data chunk in V5 and + * stored in V1. Repeat this step for the register contents + * in V2, V3, and V4 respectively. + */ + VGFMAG %v1,CONST_R2R1,%v1,%v5 + VGFMAG %v2,CONST_R2R1,%v2,%v6 + VGFMAG %v3,CONST_R2R1,%v3,%v7 + VGFMAG %v4,CONST_R2R1,%v4,%v8 + + aghi %r3,64 /* BUF = BUF + 64 */ + aghi %r4,-64 /* LEN = LEN - 64 */ + + cghi %r4,64 + jnl .Lfold_64bytes_loop + +.Lless_than_64bytes: + /* + * Fold V1 to V4 into a single 128-bit value in V1. Multiply V1 with R3 + * and R4 and accumulating the next 128-bit chunk until a single 128-bit + * value remains. + */ + VGFMAG %v1,CONST_R4R3,%v1,%v2 + VGFMAG %v1,CONST_R4R3,%v1,%v3 + VGFMAG %v1,CONST_R4R3,%v1,%v4 + + cghi %r4,16 + jl .Lfinal_fold + +.Lfold_16bytes_loop: + + VL %v2,0,,%r3 /* Load next data chunk */ + VPERM %v2,%v2,%v2,CONST_PERM_LE2BE + VGFMAG %v1,CONST_R4R3,%v1,%v2 /* Fold next data chunk */ + + aghi %r3,16 + aghi %r4,-16 + + cghi %r4,16 + jnl .Lfold_16bytes_loop + +.Lfinal_fold: + /* + * Set up a vector register for byte shifts. The shift value must + * be loaded in bits 1-4 in byte element 7 of a vector register. + * Shift by 8 bytes: 0x40 + * Shift by 4 bytes: 0x20 + */ + VLEIB %v9,0x40,7 + + /* + * Prepare V0 for the next GF(2) multiplication: shift V0 by 8 bytes + * to move R4 into the rightmost doubleword and set the leftmost + * doubleword to 0x1. + */ + VSRLB %v0,CONST_R4R3,%v9 + VLEIG %v0,1,0 + + /* + * Compute GF(2) product of V1 and V0. The rightmost doubleword + * of V1 is multiplied with R4. The leftmost doubleword of V1 is + * multiplied by 0x1 and is then XORed with rightmost product. + * Implicitly, the intermediate leftmost product becomes padded + */ + VGFMG %v1,%v0,%v1 + + /* + * Now do the final 32-bit fold by multiplying the rightmost word + * in V1 with R5 and XOR the result with the remaining bits in V1. + * + * To achieve this by a single VGFMAG, right shift V1 by a word + * and store the result in V2 which is then accumulated. Use the + * vector unpack instruction to load the rightmost half of the + * doubleword into the rightmost doubleword element of V1; the other + * half is loaded in the leftmost doubleword. + * The vector register with CONST_R5 contains the R5 constant in the + * rightmost doubleword and the leftmost doubleword is zero to ignore + * the leftmost product of V1. + */ + VLEIB %v9,0x20,7 /* Shift by words */ + VSRLB %v2,%v1,%v9 /* Store remaining bits in V2 */ + VUPLLF %v1,%v1 /* Split rightmost doubleword */ + VGFMAG %v1,CONST_R5,%v1,%v2 /* V1 = (V1 * R5) XOR V2 */ + + /* + * Apply a Barret reduction to compute the final 32-bit CRC value. + * + * The input values to the Barret reduction are the degree-63 polynomial + * in V1 (R(x)), degree-32 generator polynomial, and the reduction + * constant u. The Barret reduction result is the CRC value of R(x) mod + * P(x). + * + * The Barret reduction algorithm is defined as: + * + * 1. T1(x) = floor( R(x) / x^32 ) GF2MUL u + * 2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x) + * 3. C(x) = R(x) XOR T2(x) mod x^32 + * + * Note: The leftmost doubleword of vector register containing + * CONST_RU_POLY is zero and, thus, the intermediate GF(2) product + * is zero and does not contribute to the final result. + */ + + /* T1(x) = floor( R(x) / x^32 ) GF2MUL u */ + VUPLLF %v2,%v1 + VGFMG %v2,CONST_RU_POLY,%v2 + + /* + * Compute the GF(2) product of the CRC polynomial with T1(x) in + * V2 and XOR the intermediate result, T2(x), with the value in V1. + * The final result is stored in word element 2 of V2. + */ + VUPLLF %v2,%v2 + VGFMAG %v2,CONST_CRC_POLY,%v2,%v1 + +.Ldone: + VLGVF %r2,%v2,2 + br %r14 + +.previous diff --git a/arch/s390/defconfig b/arch/s390/defconfig index 3f571ea89509..ccccebeeaaf6 100644 --- a/arch/s390/defconfig +++ b/arch/s390/defconfig @@ -225,12 +225,16 @@ CONFIG_CRYPTO_DEFLATE=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m CONFIG_CRYPTO_ANSI_CPRNG=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m CONFIG_ZCRYPT=m CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA256_S390=m CONFIG_CRYPTO_SHA512_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_AES_S390=m +CONFIG_CRYPTO_CRC32_S390=m CONFIG_CRC7=m # CONFIG_XZ_DEC_X86 is not set # CONFIG_XZ_DEC_POWERPC is not set diff --git a/arch/s390/hypfs/hypfs_diag.c b/arch/s390/hypfs/hypfs_diag.c index 045035796ca7..67d43a0eabb4 100644 --- a/arch/s390/hypfs/hypfs_diag.c +++ b/arch/s390/hypfs/hypfs_diag.c @@ -337,25 +337,27 @@ static inline __u64 phys_cpu__ctidx(enum diag204_format type, void *hdr) /* Diagnose 204 functions */ -static inline int __diag204(unsigned long subcode, unsigned long size, void *addr) +static inline int __diag204(unsigned long *subcode, unsigned long size, void *addr) { - register unsigned long _subcode asm("0") = subcode; + register unsigned long _subcode asm("0") = *subcode; register unsigned long _size asm("1") = size; asm volatile( " diag %2,%0,0x204\n" - "0:\n" + "0: nopr %%r7\n" EX_TABLE(0b,0b) : "+d" (_subcode), "+d" (_size) : "d" (addr) : "memory"); - if (_subcode) - return -1; + *subcode = _subcode; return _size; } static int diag204(unsigned long subcode, unsigned long size, void *addr) { diag_stat_inc(DIAG_STAT_X204); - return __diag204(subcode, size, addr); + size = __diag204(&subcode, size, addr); + if (subcode) + return -1; + return size; } /* diff --git a/arch/s390/hypfs/hypfs_vm.c b/arch/s390/hypfs/hypfs_vm.c index 44feac38ccfc..012919d9833b 100644 --- a/arch/s390/hypfs/hypfs_vm.c +++ b/arch/s390/hypfs/hypfs_vm.c @@ -70,7 +70,7 @@ static int diag2fc(int size, char* query, void *addr) diag_stat_inc(DIAG_STAT_X2FC); asm volatile( " diag %0,%1,0x2fc\n" - "0:\n" + "0: nopr %%r7\n" EX_TABLE(0b,0b) : "=d" (residual_cnt), "+d" (rc) : "0" (&parm_list) : "memory"); diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index 911064aa59b2..d28cc2f5b7b2 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -93,6 +93,11 @@ static inline int atomic_add_return(int i, atomic_t *v) return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i; } +static inline int atomic_fetch_add(int i, atomic_t *v) +{ + return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER); +} + static inline void atomic_add(int i, atomic_t *v) { #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES @@ -114,22 +119,27 @@ static inline void atomic_add(int i, atomic_t *v) #define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0) #define atomic_sub(_i, _v) atomic_add(-(int)(_i), _v) #define atomic_sub_return(_i, _v) atomic_add_return(-(int)(_i), _v) +#define atomic_fetch_sub(_i, _v) atomic_fetch_add(-(int)(_i), _v) #define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0) #define atomic_dec(_v) atomic_sub(1, _v) #define atomic_dec_return(_v) atomic_sub_return(1, _v) #define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) -#define ATOMIC_OP(op, OP) \ +#define ATOMIC_OPS(op, OP) \ static inline void atomic_##op(int i, atomic_t *v) \ { \ __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_NO_BARRIER); \ +} \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + return __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_BARRIER); \ } -ATOMIC_OP(and, AND) -ATOMIC_OP(or, OR) -ATOMIC_OP(xor, XOR) +ATOMIC_OPS(and, AND) +ATOMIC_OPS(or, OR) +ATOMIC_OPS(xor, XOR) -#undef ATOMIC_OP +#undef ATOMIC_OPS #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) @@ -236,6 +246,11 @@ static inline long long atomic64_add_return(long long i, atomic64_t *v) return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i; } +static inline long long atomic64_fetch_add(long long i, atomic64_t *v) +{ + return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER); +} + static inline void atomic64_add(long long i, atomic64_t *v) { #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES @@ -264,17 +279,21 @@ static inline long long atomic64_cmpxchg(atomic64_t *v, return old; } -#define ATOMIC64_OP(op, OP) \ +#define ATOMIC64_OPS(op, OP) \ static inline void atomic64_##op(long i, atomic64_t *v) \ { \ __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER); \ +} \ +static inline long atomic64_fetch_##op(long i, atomic64_t *v) \ +{ \ + return __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_BARRIER); \ } -ATOMIC64_OP(and, AND) -ATOMIC64_OP(or, OR) -ATOMIC64_OP(xor, XOR) +ATOMIC64_OPS(and, AND) +ATOMIC64_OPS(or, OR) +ATOMIC64_OPS(xor, XOR) -#undef ATOMIC64_OP +#undef ATOMIC64_OPS #undef __ATOMIC64_LOOP static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u) @@ -315,6 +334,7 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) #define atomic64_inc_return(_v) atomic64_add_return(1, _v) #define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) #define atomic64_sub_return(_i, _v) atomic64_add_return(-(long long)(_i), _v) +#define atomic64_fetch_sub(_i, _v) atomic64_fetch_add(-(long long)(_i), _v) #define atomic64_sub(_i, _v) atomic64_add(-(long long)(_i), _v) #define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) #define atomic64_dec(_v) atomic64_sub(1, _v) diff --git a/arch/s390/include/asm/cache.h b/arch/s390/include/asm/cache.h index 22da3b34c655..05219a5e0b2f 100644 --- a/arch/s390/include/asm/cache.h +++ b/arch/s390/include/asm/cache.h @@ -13,9 +13,6 @@ #define L1_CACHE_SHIFT 8 #define NET_SKB_PAD 32 -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) - -/* Read-only memory is marked before mark_rodata_ro() is called. */ -#define __ro_after_init __read_mostly +#define __read_mostly __section(.data..read_mostly) #endif diff --git a/arch/s390/include/asm/cio.h b/arch/s390/include/asm/cio.h index d1e7b0a0feeb..f7ed88cc066e 100644 --- a/arch/s390/include/asm/cio.h +++ b/arch/s390/include/asm/cio.h @@ -320,7 +320,7 @@ struct cio_iplinfo { extern int cio_get_iplinfo(struct cio_iplinfo *iplinfo); /* Function from drivers/s390/cio/chsc.c */ -int chsc_sstpc(void *page, unsigned int op, u16 ctrl); +int chsc_sstpc(void *page, unsigned int op, u16 ctrl, u64 *clock_delta); int chsc_sstpi(void *page, void *result, size_t size); #endif diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h index 9dd04b9e9782..03516476127b 100644 --- a/arch/s390/include/asm/cpu_mf.h +++ b/arch/s390/include/asm/cpu_mf.h @@ -169,16 +169,27 @@ static inline int lcctl(u64 ctl) } /* Extract CPU counter */ -static inline int ecctr(u64 ctr, u64 *val) +static inline int __ecctr(u64 ctr, u64 *content) { - register u64 content asm("4") = 0; + register u64 _content asm("4") = 0; int cc; asm volatile ( " .insn rre,0xb2e40000,%0,%2\n" " ipm %1\n" " srl %1,28\n" - : "=d" (content), "=d" (cc) : "d" (ctr) : "cc"); + : "=d" (_content), "=d" (cc) : "d" (ctr) : "cc"); + *content = _content; + return cc; +} + +/* Extract CPU counter */ +static inline int ecctr(u64 ctr, u64 *val) +{ + u64 content; + int cc; + + cc = __ecctr(ctr, &content); if (!cc) *val = content; return cc; diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index 5fac921c1c42..86cae09e076a 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -49,7 +49,7 @@ static inline void diag10_range(unsigned long start_pfn, unsigned long num_pfn) diag_stat_inc(DIAG_STAT_X010); asm volatile( "0: diag %0,%1,0x10\n" - "1:\n" + "1: nopr %%r7\n" EX_TABLE(0b, 1b) EX_TABLE(1b, 1b) : : "a" (start_addr), "a" (end_addr)); diff --git a/arch/s390/include/asm/etr.h b/arch/s390/include/asm/etr.h deleted file mode 100644 index 105f90e63a0e..000000000000 --- a/arch/s390/include/asm/etr.h +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Copyright IBM Corp. 2006 - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) - */ -#ifndef __S390_ETR_H -#define __S390_ETR_H - -/* ETR attachment control register */ -struct etr_eacr { - unsigned int e0 : 1; /* port 0 stepping control */ - unsigned int e1 : 1; /* port 1 stepping control */ - unsigned int _pad0 : 5; /* must be 00100 */ - unsigned int dp : 1; /* data port control */ - unsigned int p0 : 1; /* port 0 change recognition control */ - unsigned int p1 : 1; /* port 1 change recognition control */ - unsigned int _pad1 : 3; /* must be 000 */ - unsigned int ea : 1; /* ETR alert control */ - unsigned int es : 1; /* ETR sync check control */ - unsigned int sl : 1; /* switch to local control */ -} __attribute__ ((packed)); - -/* Port state returned by steai */ -enum etr_psc { - etr_psc_operational = 0, - etr_psc_semi_operational = 1, - etr_psc_protocol_error = 4, - etr_psc_no_symbols = 8, - etr_psc_no_signal = 12, - etr_psc_pps_mode = 13 -}; - -/* Logical port state returned by stetr */ -enum etr_lpsc { - etr_lpsc_operational_step = 0, - etr_lpsc_operational_alt = 1, - etr_lpsc_semi_operational = 2, - etr_lpsc_protocol_error = 4, - etr_lpsc_no_symbol_sync = 8, - etr_lpsc_no_signal = 12, - etr_lpsc_pps_mode = 13 -}; - -/* ETR status words */ -struct etr_esw { - struct etr_eacr eacr; /* attachment control register */ - unsigned int y : 1; /* stepping mode */ - unsigned int _pad0 : 5; /* must be 00000 */ - unsigned int p : 1; /* stepping port number */ - unsigned int q : 1; /* data port number */ - unsigned int psc0 : 4; /* port 0 state code */ - unsigned int psc1 : 4; /* port 1 state code */ -} __attribute__ ((packed)); - -/* Second level data register status word */ -struct etr_slsw { - unsigned int vv1 : 1; /* copy of validity bit data frame 1 */ - unsigned int vv2 : 1; /* copy of validity bit data frame 2 */ - unsigned int vv3 : 1; /* copy of validity bit data frame 3 */ - unsigned int vv4 : 1; /* copy of validity bit data frame 4 */ - unsigned int _pad0 : 19; /* must by all zeroes */ - unsigned int n : 1; /* EAF port number */ - unsigned int v1 : 1; /* validity bit ETR data frame 1 */ - unsigned int v2 : 1; /* validity bit ETR data frame 2 */ - unsigned int v3 : 1; /* validity bit ETR data frame 3 */ - unsigned int v4 : 1; /* validity bit ETR data frame 4 */ - unsigned int _pad1 : 4; /* must be 0000 */ -} __attribute__ ((packed)); - -/* ETR data frames */ -struct etr_edf1 { - unsigned int u : 1; /* untuned bit */ - unsigned int _pad0 : 1; /* must be 0 */ - unsigned int r : 1; /* service request bit */ - unsigned int _pad1 : 4; /* must be 0000 */ - unsigned int a : 1; /* time adjustment bit */ - unsigned int net_id : 8; /* ETR network id */ - unsigned int etr_id : 8; /* id of ETR which sends data frames */ - unsigned int etr_pn : 8; /* port number of ETR output port */ -} __attribute__ ((packed)); - -struct etr_edf2 { - unsigned int etv : 32; /* Upper 32 bits of TOD. */ -} __attribute__ ((packed)); - -struct etr_edf3 { - unsigned int rc : 8; /* failure reason code */ - unsigned int _pad0 : 3; /* must be 000 */ - unsigned int c : 1; /* ETR coupled bit */ - unsigned int tc : 4; /* ETR type code */ - unsigned int blto : 8; /* biased local time offset */ - /* (blto - 128) * 15 = minutes */ - unsigned int buo : 8; /* biased utc offset */ - /* (buo - 128) = leap seconds */ -} __attribute__ ((packed)); - -struct etr_edf4 { - unsigned int ed : 8; /* ETS device dependent data */ - unsigned int _pad0 : 1; /* must be 0 */ - unsigned int buc : 5; /* biased ut1 correction */ - /* (buc - 16) * 0.1 seconds */ - unsigned int em : 6; /* ETS error magnitude */ - unsigned int dc : 6; /* ETS drift code */ - unsigned int sc : 6; /* ETS steering code */ -} __attribute__ ((packed)); - -/* - * ETR attachment information block, two formats - * format 1 has 4 reserved words with a size of 64 bytes - * format 2 has 16 reserved words with a size of 96 bytes - */ -struct etr_aib { - struct etr_esw esw; - struct etr_slsw slsw; - unsigned long long tsp; - struct etr_edf1 edf1; - struct etr_edf2 edf2; - struct etr_edf3 edf3; - struct etr_edf4 edf4; - unsigned int reserved[16]; -} __attribute__ ((packed,aligned(8))); - -/* ETR interruption parameter */ -struct etr_irq_parm { - unsigned int _pad0 : 8; - unsigned int pc0 : 1; /* port 0 state change */ - unsigned int pc1 : 1; /* port 1 state change */ - unsigned int _pad1 : 3; - unsigned int eai : 1; /* ETR alert indication */ - unsigned int _pad2 : 18; -} __attribute__ ((packed)); - -/* Query TOD offset result */ -struct etr_ptff_qto { - unsigned long long physical_clock; - unsigned long long tod_offset; - unsigned long long logical_tod_offset; - unsigned long long tod_epoch_difference; -} __attribute__ ((packed)); - -/* Inline assembly helper functions */ -static inline int etr_setr(struct etr_eacr *ctrl) -{ - int rc = -EOPNOTSUPP; - - asm volatile( - " .insn s,0xb2160000,%1\n" - "0: la %0,0\n" - "1:\n" - EX_TABLE(0b,1b) - : "+d" (rc) : "Q" (*ctrl)); - return rc; -} - -/* Stores a format 1 aib with 64 bytes */ -static inline int etr_stetr(struct etr_aib *aib) -{ - int rc = -EOPNOTSUPP; - - asm volatile( - " .insn s,0xb2170000,%1\n" - "0: la %0,0\n" - "1:\n" - EX_TABLE(0b,1b) - : "+d" (rc) : "Q" (*aib)); - return rc; -} - -/* Stores a format 2 aib with 96 bytes for specified port */ -static inline int etr_steai(struct etr_aib *aib, unsigned int func) -{ - register unsigned int reg0 asm("0") = func; - int rc = -EOPNOTSUPP; - - asm volatile( - " .insn s,0xb2b30000,%1\n" - "0: la %0,0\n" - "1:\n" - EX_TABLE(0b,1b) - : "+d" (rc) : "Q" (*aib), "d" (reg0)); - return rc; -} - -/* Function codes for the steai instruction. */ -#define ETR_STEAI_STEPPING_PORT 0x10 -#define ETR_STEAI_ALTERNATE_PORT 0x11 -#define ETR_STEAI_PORT_0 0x12 -#define ETR_STEAI_PORT_1 0x13 - -static inline int etr_ptff(void *ptff_block, unsigned int func) -{ - register unsigned int reg0 asm("0") = func; - register unsigned long reg1 asm("1") = (unsigned long) ptff_block; - int rc = -EOPNOTSUPP; - - asm volatile( - " .word 0x0104\n" - " ipm %0\n" - " srl %0,28\n" - : "=d" (rc), "=m" (ptff_block) - : "d" (reg0), "d" (reg1), "m" (ptff_block) : "cc"); - return rc; -} - -/* Function codes for the ptff instruction. */ -#define ETR_PTFF_QAF 0x00 /* query available functions */ -#define ETR_PTFF_QTO 0x01 /* query tod offset */ -#define ETR_PTFF_QSI 0x02 /* query steering information */ -#define ETR_PTFF_ATO 0x40 /* adjust tod offset */ -#define ETR_PTFF_STO 0x41 /* set tod offset */ -#define ETR_PTFF_SFS 0x42 /* set fine steering rate */ -#define ETR_PTFF_SGS 0x43 /* set gross steering rate */ - -/* Functions needed by the machine check handler */ -int etr_switch_to_local(void); -int etr_sync_check(void); -void etr_queue_work(void); - -/* notifier for syncs */ -extern struct atomic_notifier_head s390_epoch_delta_notifier; - -/* STP interruption parameter */ -struct stp_irq_parm { - unsigned int _pad0 : 14; - unsigned int tsc : 1; /* Timing status change */ - unsigned int lac : 1; /* Link availability change */ - unsigned int tcpc : 1; /* Time control parameter change */ - unsigned int _pad2 : 15; -} __attribute__ ((packed)); - -#define STP_OP_SYNC 1 -#define STP_OP_CTRL 3 - -struct stp_sstpi { - unsigned int rsvd0; - unsigned int rsvd1 : 8; - unsigned int stratum : 8; - unsigned int vbits : 16; - unsigned int leaps : 16; - unsigned int tmd : 4; - unsigned int ctn : 4; - unsigned int rsvd2 : 3; - unsigned int c : 1; - unsigned int tst : 4; - unsigned int tzo : 16; - unsigned int dsto : 16; - unsigned int ctrl : 16; - unsigned int rsvd3 : 16; - unsigned int tto; - unsigned int rsvd4; - unsigned int ctnid[3]; - unsigned int rsvd5; - unsigned int todoff[4]; - unsigned int rsvd6[48]; -} __attribute__ ((packed)); - -/* Functions needed by the machine check handler */ -int stp_sync_check(void); -int stp_island_check(void); -void stp_queue_work(void); - -#endif /* __S390_ETR_H */ diff --git a/arch/s390/include/asm/fcx.h b/arch/s390/include/asm/fcx.h index 7ecb92b469b6..04cb4b4bcc5f 100644 --- a/arch/s390/include/asm/fcx.h +++ b/arch/s390/include/asm/fcx.h @@ -6,7 +6,7 @@ */ #ifndef _ASM_S390_FCX_H -#define _ASM_S390_FCX_H _ASM_S390_FCX_H +#define _ASM_S390_FCX_H #include diff --git a/arch/s390/include/asm/fpu/api.h b/arch/s390/include/asm/fpu/api.h index 5e04f3cbd320..6aba6fc406ad 100644 --- a/arch/s390/include/asm/fpu/api.h +++ b/arch/s390/include/asm/fpu/api.h @@ -1,6 +1,41 @@ /* * In-kernel FPU support functions * + * + * Consider these guidelines before using in-kernel FPU functions: + * + * 1. Use kernel_fpu_begin() and kernel_fpu_end() to enclose all in-kernel + * use of floating-point or vector registers and instructions. + * + * 2. For kernel_fpu_begin(), specify the vector register range you want to + * use with the KERNEL_VXR_* constants. Consider these usage guidelines: + * + * a) If your function typically runs in process-context, use the lower + * half of the vector registers, for example, specify KERNEL_VXR_LOW. + * b) If your function typically runs in soft-irq or hard-irq context, + * prefer using the upper half of the vector registers, for example, + * specify KERNEL_VXR_HIGH. + * + * If you adhere to these guidelines, an interrupted process context + * does not require to save and restore vector registers because of + * disjoint register ranges. + * + * Also note that the __kernel_fpu_begin()/__kernel_fpu_end() functions + * includes logic to save and restore up to 16 vector registers at once. + * + * 3. You can nest kernel_fpu_begin()/kernel_fpu_end() by using different + * struct kernel_fpu states. Vector registers that are in use by outer + * levels are saved and restored. You can minimize the save and restore + * effort by choosing disjoint vector register ranges. + * + * 5. To use vector floating-point instructions, specify the KERNEL_FPC + * flag to save and restore floating-point controls in addition to any + * vector register range. + * + * 6. To use floating-point registers and instructions only, specify the + * KERNEL_FPR flag. This flag triggers a save and restore of vector + * registers V0 to V15 and floating-point controls. + * * Copyright IBM Corp. 2015 * Author(s): Hendrik Brueckner */ @@ -8,6 +43,8 @@ #ifndef _ASM_S390_FPU_API_H #define _ASM_S390_FPU_API_H +#include + void save_fpu_regs(void); static inline int test_fp_ctl(u32 fpc) @@ -22,9 +59,47 @@ static inline int test_fp_ctl(u32 fpc) " la %0,0\n" "1:\n" EX_TABLE(0b,1b) - : "=d" (rc), "=d" (orig_fpc) + : "=d" (rc), "=&d" (orig_fpc) : "d" (fpc), "0" (-EINVAL)); return rc; } +#define KERNEL_VXR_V0V7 1 +#define KERNEL_VXR_V8V15 2 +#define KERNEL_VXR_V16V23 4 +#define KERNEL_VXR_V24V31 8 +#define KERNEL_FPR 16 +#define KERNEL_FPC 256 + +#define KERNEL_VXR_LOW (KERNEL_VXR_V0V7|KERNEL_VXR_V8V15) +#define KERNEL_VXR_MID (KERNEL_VXR_V8V15|KERNEL_VXR_V16V23) +#define KERNEL_VXR_HIGH (KERNEL_VXR_V16V23|KERNEL_VXR_V24V31) + +#define KERNEL_FPU_MASK (KERNEL_VXR_LOW|KERNEL_VXR_HIGH|KERNEL_FPR) + +struct kernel_fpu; + +/* + * Note the functions below must be called with preemption disabled. + * Do not enable preemption before calling __kernel_fpu_end() to prevent + * an corruption of an existing kernel FPU state. + * + * Prefer using the kernel_fpu_begin()/kernel_fpu_end() pair of functions. + */ +void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags); +void __kernel_fpu_end(struct kernel_fpu *state); + + +static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) +{ + preempt_disable(); + __kernel_fpu_begin(state, flags); +} + +static inline void kernel_fpu_end(struct kernel_fpu *state) +{ + __kernel_fpu_end(state); + preempt_enable(); +} + #endif /* _ASM_S390_FPU_API_H */ diff --git a/arch/s390/include/asm/fpu/types.h b/arch/s390/include/asm/fpu/types.h index fe937c9b6471..bce255ead72b 100644 --- a/arch/s390/include/asm/fpu/types.h +++ b/arch/s390/include/asm/fpu/types.h @@ -24,4 +24,14 @@ struct fpu { /* VX array structure for address operand constraints in inline assemblies */ struct vx_array { __vector128 _[__NUM_VXRS]; }; +/* In-kernel FPU state structure */ +struct kernel_fpu { + u32 mask; + u32 fpc; + union { + freg_t fprs[__NUM_FPRS]; + __vector128 vxrs[__NUM_VXRS]; + }; +}; + #endif /* _ASM_S390_FPU_TYPES_H */ diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index d9be7c0c1291..4c7fac75090e 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -41,7 +41,10 @@ static inline int prepare_hugepage_range(struct file *file, static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_val(*ptep) = _SEGMENT_ENTRY_EMPTY; + if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + pte_val(*ptep) = _REGION3_ENTRY_EMPTY; + else + pte_val(*ptep) = _SEGMENT_ENTRY_EMPTY; } static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, diff --git a/arch/s390/include/asm/ipl.h b/arch/s390/include/asm/ipl.h index 6fc44dca193e..4da22b2f0521 100644 --- a/arch/s390/include/asm/ipl.h +++ b/arch/s390/include/asm/ipl.h @@ -141,11 +141,11 @@ extern void setup_ipl(void); * DIAG 308 support */ enum diag308_subcode { - DIAG308_REL_HSA = 2, - DIAG308_IPL = 3, - DIAG308_DUMP = 4, - DIAG308_SET = 5, - DIAG308_STORE = 6, + DIAG308_REL_HSA = 2, + DIAG308_LOAD_CLEAR = 3, + DIAG308_LOAD_NORMAL_DUMP = 4, + DIAG308_SET = 5, + DIAG308_STORE = 6, }; enum diag308_ipl_type { diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index f97b055de76a..70c9bce766f5 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -7,11 +7,8 @@ #define NR_IRQS_BASE 3 -#ifdef CONFIG_PCI_NR_MSI -# define NR_IRQS (NR_IRQS_BASE + CONFIG_PCI_NR_MSI) -#else -# define NR_IRQS NR_IRQS_BASE -#endif +#define NR_IRQS NR_IRQS_BASE +#define NR_IRQS_LEGACY NR_IRQS_BASE /* External interruption codes */ #define EXT_IRQ_INTERRUPT_KEY 0x0040 diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index 7f9fd5e3f1bf..9be198f5ee79 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -4,6 +4,7 @@ #ifndef __ASSEMBLY__ #include +#include #define JUMP_LABEL_NOP_SIZE 6 #define JUMP_LABEL_NOP_OFFSET 2 diff --git a/arch/s390/include/asm/kprobes.h b/arch/s390/include/asm/kprobes.h index b47ad3b642cc..591e5a5279b0 100644 --- a/arch/s390/include/asm/kprobes.h +++ b/arch/s390/include/asm/kprobes.h @@ -43,9 +43,9 @@ typedef u16 kprobe_opcode_t; #define MAX_INSN_SIZE 0x0003 #define MAX_STACK_SIZE 64 #define MIN_STACK_SIZE(ADDR) (((MAX_STACK_SIZE) < \ - (((unsigned long)current_thread_info()) + THREAD_SIZE - (ADDR))) \ + (((unsigned long)task_stack_page(current)) + THREAD_SIZE - (ADDR))) \ ? (MAX_STACK_SIZE) \ - : (((unsigned long)current_thread_info()) + THREAD_SIZE - (ADDR))) + : (((unsigned long)task_stack_page(current)) + THREAD_SIZE - (ADDR))) #define kretprobe_blacklist_size 0 diff --git a/arch/s390/include/asm/mathemu.h b/arch/s390/include/asm/mathemu.h deleted file mode 100644 index 614dfaf47f71..000000000000 --- a/arch/s390/include/asm/mathemu.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * IEEE floating point emulation. - * - * S390 version - * Copyright IBM Corp. 1999 - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) - */ - -#ifndef __MATHEMU__ -#define __MATHEMU__ - -extern int math_emu_b3(__u8 *, struct pt_regs *); -extern int math_emu_ed(__u8 *, struct pt_regs *); -extern int math_emu_ldr(__u8 *); -extern int math_emu_ler(__u8 *); -extern int math_emu_std(__u8 *, struct pt_regs *); -extern int math_emu_ld(__u8 *, struct pt_regs *); -extern int math_emu_ste(__u8 *, struct pt_regs *); -extern int math_emu_le(__u8 *, struct pt_regs *); -extern int math_emu_lfpc(__u8 *, struct pt_regs *); -extern int math_emu_stfpc(__u8 *, struct pt_regs *); -extern int math_emu_srnm(__u8 *, struct pt_regs *); - -#endif /* __MATHEMU__ */ - - - - diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index 081b2ad99d73..18226437a832 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -6,7 +6,7 @@ typedef struct { cpumask_t cpu_attach_mask; - atomic_t attach_count; + atomic_t flush_count; unsigned int flush_mm; spinlock_t list_lock; struct list_head pgtable_list; diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index c837b79b455d..f77c638bf397 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -19,7 +19,7 @@ static inline int init_new_context(struct task_struct *tsk, INIT_LIST_HEAD(&mm->context.pgtable_list); INIT_LIST_HEAD(&mm->context.gmap_list); cpumask_clear(&mm->context.cpu_attach_mask); - atomic_set(&mm->context.attach_count, 0); + atomic_set(&mm->context.flush_count, 0); mm->context.flush_mm = 0; #ifdef CONFIG_PGSTE mm->context.alloc_pgste = page_table_allocate_pgste; @@ -90,15 +90,12 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, S390_lowcore.user_asce = next->context.asce; if (prev == next) return; - if (MACHINE_HAS_TLB_LC) - cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); + cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); + cpumask_set_cpu(cpu, mm_cpumask(next)); /* Clear old ASCE by loading the kernel ASCE. */ __ctl_load(S390_lowcore.kernel_asce, 1, 1); __ctl_load(S390_lowcore.kernel_asce, 7, 7); - atomic_inc(&next->context.attach_count); - atomic_dec(&prev->context.attach_count); - if (MACHINE_HAS_TLB_LC) - cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); + cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); } #define finish_arch_post_lock_switch finish_arch_post_lock_switch @@ -110,10 +107,9 @@ static inline void finish_arch_post_lock_switch(void) load_kernel_asce(); if (mm) { preempt_disable(); - while (atomic_read(&mm->context.attach_count) >> 16) + while (atomic_read(&mm->context.flush_count)) cpu_relax(); - cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); if (mm->context.flush_mm) __tlb_flush_mm(mm); preempt_enable(); @@ -128,7 +124,6 @@ static inline void activate_mm(struct mm_struct *prev, struct mm_struct *next) { switch_mm(prev, next, current); - cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); set_user_asce(next); } diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index 53eacbd4f09b..b2146c4119b2 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h @@ -21,6 +21,7 @@ #define HPAGE_SIZE (1UL << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) +#define HUGE_MAX_HSTATE 2 #define ARCH_HAS_SETCLEAR_HUGE_PTE #define ARCH_HAS_HUGE_PTE_TYPE @@ -30,11 +31,12 @@ #include #ifndef __ASSEMBLY__ +void __storage_key_init_range(unsigned long start, unsigned long end); + static inline void storage_key_init_range(unsigned long start, unsigned long end) { -#if PAGE_DEFAULT_KEY - __storage_key_init_range(start, end); -#endif + if (PAGE_DEFAULT_KEY) + __storage_key_init_range(start, end); } #define clear_page(page) memset((page), 0, PAGE_SIZE) diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index 1f7ff85c5e4c..c64c0befd3f3 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -86,16 +86,4 @@ struct sf_raw_sample { u8 padding[]; /* Padding to next multiple of 8 */ } __packed; -/* Perf hardware reserve and release functions */ -#ifdef CONFIG_PERF_EVENTS -int perf_reserve_sampling(void); -void perf_release_sampling(void); -#else /* CONFIG_PERF_EVENTS */ -static inline int perf_reserve_sampling(void) -{ - return 0; -} -static inline void perf_release_sampling(void) {} -#endif /* CONFIG_PERF_EVENTS */ - #endif /* _ASM_S390_PERF_EVENT_H */ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 18d2beb89340..ea1533e07271 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -28,12 +28,33 @@ #include #include #include +#include #include #include -extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096))); +extern pgd_t swapper_pg_dir[]; extern void paging_init(void); extern void vmem_map_init(void); +pmd_t *vmem_pmd_alloc(void); +pte_t *vmem_pte_alloc(void); + +enum { + PG_DIRECT_MAP_4K = 0, + PG_DIRECT_MAP_1M, + PG_DIRECT_MAP_2G, + PG_DIRECT_MAP_MAX +}; + +extern atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX]; + +static inline void update_page_count(int level, long count) +{ + if (IS_ENABLED(CONFIG_PROC_FS)) + atomic_long_add(count, &direct_pages_count[level]); +} + +struct seq_file; +void arch_report_meminfo(struct seq_file *m); /* * The S390 doesn't have any external MMU info: the kernel page @@ -270,8 +291,23 @@ static inline int is_module_addr(void *addr) #define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH) #define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID) -#define _REGION3_ENTRY_LARGE 0x400 /* RTTE-format control, large page */ -#define _REGION3_ENTRY_RO 0x200 /* page protection bit */ +#define _REGION3_ENTRY_ORIGIN_LARGE ~0x7fffffffUL /* large page address */ +#define _REGION3_ENTRY_ORIGIN ~0x7ffUL/* region third table origin */ + +#define _REGION3_ENTRY_DIRTY 0x2000 /* SW region dirty bit */ +#define _REGION3_ENTRY_YOUNG 0x1000 /* SW region young bit */ +#define _REGION3_ENTRY_LARGE 0x0400 /* RTTE-format control, large page */ +#define _REGION3_ENTRY_READ 0x0002 /* SW region read bit */ +#define _REGION3_ENTRY_WRITE 0x0001 /* SW region write bit */ + +#ifdef CONFIG_MEM_SOFT_DIRTY +#define _REGION3_ENTRY_SOFT_DIRTY 0x4000 /* SW region soft dirty bit */ +#else +#define _REGION3_ENTRY_SOFT_DIRTY 0x0000 /* SW region soft dirty bit */ +#endif + +#define _REGION_ENTRY_BITS 0xfffffffffffff227UL +#define _REGION_ENTRY_BITS_LARGE 0xffffffff8000fe27UL /* Bits in the segment table entry */ #define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL @@ -297,7 +333,8 @@ static inline int is_module_addr(void *addr) #endif /* - * Segment table entry encoding (R = read-only, I = invalid, y = young bit): + * Segment table and region3 table entry encoding + * (R = read-only, I = invalid, y = young bit): * dy..R...I...rw * prot-none, clean, old 00..1...1...00 * prot-none, clean, young 01..1...1...00 @@ -391,6 +428,33 @@ static inline int is_module_addr(void *addr) _SEGMENT_ENTRY_READ) #define SEGMENT_WRITE __pgprot(_SEGMENT_ENTRY_READ | \ _SEGMENT_ENTRY_WRITE) +#define SEGMENT_KERNEL __pgprot(_SEGMENT_ENTRY | \ + _SEGMENT_ENTRY_LARGE | \ + _SEGMENT_ENTRY_READ | \ + _SEGMENT_ENTRY_WRITE | \ + _SEGMENT_ENTRY_YOUNG | \ + _SEGMENT_ENTRY_DIRTY) +#define SEGMENT_KERNEL_RO __pgprot(_SEGMENT_ENTRY | \ + _SEGMENT_ENTRY_LARGE | \ + _SEGMENT_ENTRY_READ | \ + _SEGMENT_ENTRY_YOUNG | \ + _SEGMENT_ENTRY_PROTECT) + +/* + * Region3 entry (large page) protection definitions. + */ + +#define REGION3_KERNEL __pgprot(_REGION_ENTRY_TYPE_R3 | \ + _REGION3_ENTRY_LARGE | \ + _REGION3_ENTRY_READ | \ + _REGION3_ENTRY_WRITE | \ + _REGION3_ENTRY_YOUNG | \ + _REGION3_ENTRY_DIRTY) +#define REGION3_KERNEL_RO __pgprot(_REGION_ENTRY_TYPE_R3 | \ + _REGION3_ENTRY_LARGE | \ + _REGION3_ENTRY_READ | \ + _REGION3_ENTRY_YOUNG | \ + _REGION_ENTRY_PROTECT) static inline int mm_has_pgste(struct mm_struct *mm) { @@ -424,6 +488,53 @@ static inline int mm_use_skey(struct mm_struct *mm) return 0; } +static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new) +{ + register unsigned long reg2 asm("2") = old; + register unsigned long reg3 asm("3") = new; + unsigned long address = (unsigned long)ptr | 1; + + asm volatile( + " csp %0,%3" + : "+d" (reg2), "+m" (*ptr) + : "d" (reg3), "d" (address) + : "cc"); +} + +static inline void cspg(unsigned long *ptr, unsigned long old, unsigned long new) +{ + register unsigned long reg2 asm("2") = old; + register unsigned long reg3 asm("3") = new; + unsigned long address = (unsigned long)ptr | 1; + + asm volatile( + " .insn rre,0xb98a0000,%0,%3" + : "+d" (reg2), "+m" (*ptr) + : "d" (reg3), "d" (address) + : "cc"); +} + +#define CRDTE_DTT_PAGE 0x00UL +#define CRDTE_DTT_SEGMENT 0x10UL +#define CRDTE_DTT_REGION3 0x14UL +#define CRDTE_DTT_REGION2 0x18UL +#define CRDTE_DTT_REGION1 0x1cUL + +static inline void crdte(unsigned long old, unsigned long new, + unsigned long table, unsigned long dtt, + unsigned long address, unsigned long asce) +{ + register unsigned long reg2 asm("2") = old; + register unsigned long reg3 asm("3") = new; + register unsigned long reg4 asm("4") = table | dtt; + register unsigned long reg5 asm("5") = address; + + asm volatile(".insn rrf,0xb98f0000,%0,%2,%4,0" + : "+d" (reg2) + : "d" (reg3), "d" (reg4), "d" (reg5), "a" (asce) + : "memory", "cc"); +} + /* * pgd/pmd/pte query functions */ @@ -465,7 +576,7 @@ static inline int pud_none(pud_t pud) { if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) return 0; - return (pud_val(pud) & _REGION_ENTRY_INVALID) != 0UL; + return pud_val(pud) == _REGION3_ENTRY_EMPTY; } static inline int pud_large(pud_t pud) @@ -475,17 +586,35 @@ static inline int pud_large(pud_t pud) return !!(pud_val(pud) & _REGION3_ENTRY_LARGE); } +static inline unsigned long pud_pfn(pud_t pud) +{ + unsigned long origin_mask; + + origin_mask = _REGION3_ENTRY_ORIGIN; + if (pud_large(pud)) + origin_mask = _REGION3_ENTRY_ORIGIN_LARGE; + return (pud_val(pud) & origin_mask) >> PAGE_SHIFT; +} + +static inline int pmd_large(pmd_t pmd) +{ + return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0; +} + +static inline int pmd_bad(pmd_t pmd) +{ + if (pmd_large(pmd)) + return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0; + return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0; +} + static inline int pud_bad(pud_t pud) { - /* - * With dynamic page table levels the pud can be a region table - * entry or a segment table entry. Check for the bit that are - * invalid for either table entry. - */ - unsigned long mask = - ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID & - ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH; - return (pud_val(pud) & mask) != 0; + if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) + return pmd_bad(__pmd(pud_val(pud))); + if (pud_large(pud)) + return (pud_val(pud) & ~_REGION_ENTRY_BITS_LARGE) != 0; + return (pud_val(pud) & ~_REGION_ENTRY_BITS) != 0; } static inline int pmd_present(pmd_t pmd) @@ -498,11 +627,6 @@ static inline int pmd_none(pmd_t pmd) return pmd_val(pmd) == _SEGMENT_ENTRY_INVALID; } -static inline int pmd_large(pmd_t pmd) -{ - return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0; -} - static inline unsigned long pmd_pfn(pmd_t pmd) { unsigned long origin_mask; @@ -513,13 +637,6 @@ static inline unsigned long pmd_pfn(pmd_t pmd) return (pmd_val(pmd) & origin_mask) >> PAGE_SHIFT; } -static inline int pmd_bad(pmd_t pmd) -{ - if (pmd_large(pmd)) - return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0; - return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0; -} - #define __HAVE_ARCH_PMD_WRITE static inline int pmd_write(pmd_t pmd) { @@ -963,6 +1080,7 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) #define pte_page(x) pfn_to_page(pte_pfn(x)) #define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) +#define pud_page(pud) pfn_to_page(pud_pfn(pud)) /* Find an entry in the lowest level page table.. */ #define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr)) @@ -970,20 +1088,6 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) #define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address) #define pte_unmap(pte) do { } while (0) -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) -static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot) -{ - /* - * pgprot is PAGE_NONE, PAGE_READ, or PAGE_WRITE (see __Pxxx / __Sxxx) - * Convert to segment table entry format. - */ - if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE)) - return pgprot_val(SEGMENT_NONE); - if (pgprot_val(pgprot) == pgprot_val(PAGE_READ)) - return pgprot_val(SEGMENT_READ); - return pgprot_val(SEGMENT_WRITE); -} - static inline pmd_t pmd_wrprotect(pmd_t pmd) { pmd_val(pmd) &= ~_SEGMENT_ENTRY_WRITE; @@ -1020,6 +1124,56 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd) return pmd; } +static inline pud_t pud_wrprotect(pud_t pud) +{ + pud_val(pud) &= ~_REGION3_ENTRY_WRITE; + pud_val(pud) |= _REGION_ENTRY_PROTECT; + return pud; +} + +static inline pud_t pud_mkwrite(pud_t pud) +{ + pud_val(pud) |= _REGION3_ENTRY_WRITE; + if (pud_large(pud) && !(pud_val(pud) & _REGION3_ENTRY_DIRTY)) + return pud; + pud_val(pud) &= ~_REGION_ENTRY_PROTECT; + return pud; +} + +static inline pud_t pud_mkclean(pud_t pud) +{ + if (pud_large(pud)) { + pud_val(pud) &= ~_REGION3_ENTRY_DIRTY; + pud_val(pud) |= _REGION_ENTRY_PROTECT; + } + return pud; +} + +static inline pud_t pud_mkdirty(pud_t pud) +{ + if (pud_large(pud)) { + pud_val(pud) |= _REGION3_ENTRY_DIRTY | + _REGION3_ENTRY_SOFT_DIRTY; + if (pud_val(pud) & _REGION3_ENTRY_WRITE) + pud_val(pud) &= ~_REGION_ENTRY_PROTECT; + } + return pud; +} + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) +static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot) +{ + /* + * pgprot is PAGE_NONE, PAGE_READ, or PAGE_WRITE (see __Pxxx / __Sxxx) + * Convert to segment table entry format. + */ + if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE)) + return pgprot_val(SEGMENT_NONE); + if (pgprot_val(pgprot) == pgprot_val(PAGE_READ)) + return pgprot_val(SEGMENT_READ); + return pgprot_val(SEGMENT_WRITE); +} + static inline pmd_t pmd_mkyoung(pmd_t pmd) { if (pmd_large(pmd)) { @@ -1068,15 +1222,8 @@ static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot) static inline void __pmdp_csp(pmd_t *pmdp) { - register unsigned long reg2 asm("2") = pmd_val(*pmdp); - register unsigned long reg3 asm("3") = pmd_val(*pmdp) | - _SEGMENT_ENTRY_INVALID; - register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5; - - asm volatile( - " csp %1,%3" - : "=m" (*pmdp) - : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc"); + csp((unsigned int *)pmdp + 1, pmd_val(*pmdp), + pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); } static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp) @@ -1091,6 +1238,19 @@ static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp) : "cc" ); } +static inline void __pudp_idte(unsigned long address, pud_t *pudp) +{ + unsigned long r3o; + + r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t); + r3o |= _ASCE_TYPE_REGION3; + asm volatile( + " .insn rrf,0xb98e0000,%2,%3,0,0" + : "=m" (*pudp) + : "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK)) + : "cc"); +} + static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp) { unsigned long sto; @@ -1103,8 +1263,22 @@ static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp) : "cc" ); } +static inline void __pudp_idte_local(unsigned long address, pud_t *pudp) +{ + unsigned long r3o; + + r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t); + r3o |= _ASCE_TYPE_REGION3; + asm volatile( + " .insn rrf,0xb98e0000,%2,%3,0,1" + : "=m" (*pudp) + : "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK)) + : "cc"); +} + pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t); pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t); +pud_t pudp_xchg_direct(struct mm_struct *, unsigned long, pud_t *, pud_t); #ifdef CONFIG_TRANSPARENT_HUGEPAGE diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 9d4d311d7e52..09529202ea77 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -77,7 +77,10 @@ static inline void get_cpu_id(struct cpuid *ptr) asm volatile("stidp %0" : "=Q" (*ptr)); } -extern void s390_adjust_jiffies(void); +void s390_adjust_jiffies(void); +void s390_update_cpu_mhz(void); +void cpu_detect_mhz_feature(void); + extern const struct seq_operations cpuinfo_op; extern int sysctl_ieee_emulation_warnings; extern void execve_tail(void); @@ -233,6 +236,18 @@ void cpu_relax(void); #define cpu_relax_lowlatency() barrier() +#define ECAG_CACHE_ATTRIBUTE 0 +#define ECAG_CPU_ATTRIBUTE 1 + +static inline unsigned long __ecag(unsigned int asi, unsigned char parm) +{ + unsigned long val; + + asm volatile(".insn rsy,0xeb000000004c,%0,0,0(%1)" /* ecag */ + : "=d" (val) : "a" (asi << 8 | parm)); + return val; +} + static inline void psw_set_key(unsigned int key) { asm volatile("spka 0(%0)" : : "d" (key)); diff --git a/arch/s390/include/asm/rwsem.h b/arch/s390/include/asm/rwsem.h index c75e4471e618..597e7e96b59e 100644 --- a/arch/s390/include/asm/rwsem.h +++ b/arch/s390/include/asm/rwsem.h @@ -207,41 +207,4 @@ static inline void __downgrade_write(struct rw_semaphore *sem) rwsem_downgrade_wake(sem); } -/* - * implement atomic add functionality - */ -static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) -{ - signed long old, new; - - asm volatile( - " lg %0,%2\n" - "0: lgr %1,%0\n" - " agr %1,%4\n" - " csg %0,%1,%2\n" - " jl 0b" - : "=&d" (old), "=&d" (new), "=Q" (sem->count) - : "Q" (sem->count), "d" (delta) - : "cc", "memory"); -} - -/* - * implement exchange and add functionality - */ -static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) -{ - signed long old, new; - - asm volatile( - " lg %0,%2\n" - "0: lgr %1,%0\n" - " agr %1,%4\n" - " csg %0,%1,%2\n" - " jl 0b" - : "=&d" (old), "=&d" (new), "=Q" (sem->count) - : "Q" (sem->count), "d" (delta) - : "cc", "memory"); - return new; -} - #endif /* _S390_RWSEM_H */ diff --git a/arch/s390/include/asm/sections.h b/arch/s390/include/asm/sections.h index fbd9116eb17b..5ce29fe100ba 100644 --- a/arch/s390/include/asm/sections.h +++ b/arch/s390/include/asm/sections.h @@ -4,5 +4,6 @@ #include extern char _eshared[], _ehead[]; +extern char __start_ro_after_init[], __end_ro_after_init[]; #endif diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index c0f0efbb6ab5..5e8d57e1cc5e 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -86,9 +86,13 @@ extern char vmpoff_cmd[]; #define CONSOLE_IS_SCLP (console_mode == 1) #define CONSOLE_IS_3215 (console_mode == 2) #define CONSOLE_IS_3270 (console_mode == 3) +#define CONSOLE_IS_VT220 (console_mode == 4) +#define CONSOLE_IS_HVC (console_mode == 5) #define SET_CONSOLE_SCLP do { console_mode = 1; } while (0) #define SET_CONSOLE_3215 do { console_mode = 2; } while (0) #define SET_CONSOLE_3270 do { console_mode = 3; } while (0) +#define SET_CONSOLE_VT220 do { console_mode = 4; } while (0) +#define SET_CONSOLE_HVC do { console_mode = 5; } while (0) #define NSS_NAME_SIZE 8 extern char kernel_nss_name[]; diff --git a/arch/s390/include/asm/sfp-machine.h b/arch/s390/include/asm/sfp-machine.h deleted file mode 100644 index 4e16aede4b06..000000000000 --- a/arch/s390/include/asm/sfp-machine.h +++ /dev/null @@ -1,142 +0,0 @@ -/* Machine-dependent software floating-point definitions. - S/390 kernel version. - Copyright (C) 1997,1998,1999 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Richard Henderson (rth@cygnus.com), - Jakub Jelinek (jj@ultra.linux.cz), - David S. Miller (davem@redhat.com) and - Peter Maydell (pmaydell@chiark.greenend.org.uk). - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Library General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Library General Public License for more details. - - You should have received a copy of the GNU Library General Public - License along with the GNU C Library; see the file COPYING.LIB. If - not, write to the Free Software Foundation, Inc., - 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ - -#ifndef _SFP_MACHINE_H -#define _SFP_MACHINE_H - - -#define _FP_W_TYPE_SIZE 32 -#define _FP_W_TYPE unsigned int -#define _FP_WS_TYPE signed int -#define _FP_I_TYPE int - -#define _FP_MUL_MEAT_S(R,X,Y) \ - _FP_MUL_MEAT_1_wide(_FP_WFRACBITS_S,R,X,Y,umul_ppmm) -#define _FP_MUL_MEAT_D(R,X,Y) \ - _FP_MUL_MEAT_2_wide(_FP_WFRACBITS_D,R,X,Y,umul_ppmm) -#define _FP_MUL_MEAT_Q(R,X,Y) \ - _FP_MUL_MEAT_4_wide(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm) - -#define _FP_DIV_MEAT_S(R,X,Y) _FP_DIV_MEAT_1_udiv(S,R,X,Y) -#define _FP_DIV_MEAT_D(R,X,Y) _FP_DIV_MEAT_2_udiv(D,R,X,Y) -#define _FP_DIV_MEAT_Q(R,X,Y) _FP_DIV_MEAT_4_udiv(Q,R,X,Y) - -#define _FP_NANFRAC_S ((_FP_QNANBIT_S << 1) - 1) -#define _FP_NANFRAC_D ((_FP_QNANBIT_D << 1) - 1), -1 -#define _FP_NANFRAC_Q ((_FP_QNANBIT_Q << 1) - 1), -1, -1, -1 -#define _FP_NANSIGN_S 0 -#define _FP_NANSIGN_D 0 -#define _FP_NANSIGN_Q 0 - -#define _FP_KEEPNANFRACP 1 - -/* - * If one NaN is signaling and the other is not, - * we choose that one, otherwise we choose X. - */ -#define _FP_CHOOSENAN(fs, wc, R, X, Y, OP) \ - do { \ - if ((_FP_FRAC_HIGH_RAW_##fs(X) & _FP_QNANBIT_##fs) \ - && !(_FP_FRAC_HIGH_RAW_##fs(Y) & _FP_QNANBIT_##fs)) \ - { \ - R##_s = Y##_s; \ - _FP_FRAC_COPY_##wc(R,Y); \ - } \ - else \ - { \ - R##_s = X##_s; \ - _FP_FRAC_COPY_##wc(R,X); \ - } \ - R##_c = FP_CLS_NAN; \ - } while (0) - -/* Some assembly to speed things up. */ -#define __FP_FRAC_ADD_3(r2,r1,r0,x2,x1,x0,y2,y1,y0) ({ \ - unsigned int __r2 = (x2) + (y2); \ - unsigned int __r1 = (x1); \ - unsigned int __r0 = (x0); \ - asm volatile( \ - " alr %2,%3\n" \ - " brc 12,0f\n" \ - " lhi 0,1\n" \ - " alr %1,0\n" \ - " brc 12,0f\n" \ - " alr %0,0\n" \ - "0:" \ - : "+&d" (__r2), "+&d" (__r1), "+&d" (__r0) \ - : "d" (y0), "i" (1) : "cc", "0" ); \ - asm volatile( \ - " alr %1,%2\n" \ - " brc 12,0f\n" \ - " ahi %0,1\n" \ - "0:" \ - : "+&d" (__r2), "+&d" (__r1) \ - : "d" (y1) : "cc"); \ - (r2) = __r2; \ - (r1) = __r1; \ - (r0) = __r0; \ -}) - -#define __FP_FRAC_SUB_3(r2,r1,r0,x2,x1,x0,y2,y1,y0) ({ \ - unsigned int __r2 = (x2) - (y2); \ - unsigned int __r1 = (x1); \ - unsigned int __r0 = (x0); \ - asm volatile( \ - " slr %2,%3\n" \ - " brc 3,0f\n" \ - " lhi 0,1\n" \ - " slr %1,0\n" \ - " brc 3,0f\n" \ - " slr %0,0\n" \ - "0:" \ - : "+&d" (__r2), "+&d" (__r1), "+&d" (__r0) \ - : "d" (y0) : "cc", "0"); \ - asm volatile( \ - " slr %1,%2\n" \ - " brc 3,0f\n" \ - " ahi %0,-1\n" \ - "0:" \ - : "+&d" (__r2), "+&d" (__r1) \ - : "d" (y1) : "cc"); \ - (r2) = __r2; \ - (r1) = __r1; \ - (r0) = __r0; \ -}) - -#define __FP_FRAC_DEC_3(x2,x1,x0,y2,y1,y0) __FP_FRAC_SUB_3(x2,x1,x0,x2,x1,x0,y2,y1,y0) - -/* Obtain the current rounding mode. */ -#define FP_ROUNDMODE mode - -/* Exception flags. */ -#define FP_EX_INVALID 0x800000 -#define FP_EX_DIVZERO 0x400000 -#define FP_EX_OVERFLOW 0x200000 -#define FP_EX_UNDERFLOW 0x100000 -#define FP_EX_INEXACT 0x080000 - -/* We write the results always */ -#define FP_INHIBIT_RESULTS 0 - -#endif diff --git a/arch/s390/include/asm/sfp-util.h b/arch/s390/include/asm/sfp-util.h deleted file mode 100644 index c8b7cf9d6279..000000000000 --- a/arch/s390/include/asm/sfp-util.h +++ /dev/null @@ -1,67 +0,0 @@ -#include -#include -#include -#include - -#define add_ssaaaa(sh, sl, ah, al, bh, bl) ({ \ - unsigned int __sh = (ah); \ - unsigned int __sl = (al); \ - asm volatile( \ - " alr %1,%3\n" \ - " brc 12,0f\n" \ - " ahi %0,1\n" \ - "0: alr %0,%2" \ - : "+&d" (__sh), "+d" (__sl) \ - : "d" (bh), "d" (bl) : "cc"); \ - (sh) = __sh; \ - (sl) = __sl; \ -}) - -#define sub_ddmmss(sh, sl, ah, al, bh, bl) ({ \ - unsigned int __sh = (ah); \ - unsigned int __sl = (al); \ - asm volatile( \ - " slr %1,%3\n" \ - " brc 3,0f\n" \ - " ahi %0,-1\n" \ - "0: slr %0,%2" \ - : "+&d" (__sh), "+d" (__sl) \ - : "d" (bh), "d" (bl) : "cc"); \ - (sh) = __sh; \ - (sl) = __sl; \ -}) - -/* a umul b = a mul b + (a>=2<<31) ? b<<32:0 + (b>=2<<31) ? a<<32:0 */ -#define umul_ppmm(wh, wl, u, v) ({ \ - unsigned int __wh = u; \ - unsigned int __wl = v; \ - asm volatile( \ - " ltr 1,%0\n" \ - " mr 0,%1\n" \ - " jnm 0f\n" \ - " alr 0,%1\n" \ - "0: ltr %1,%1\n" \ - " jnm 1f\n" \ - " alr 0,%0\n" \ - "1: lr %0,0\n" \ - " lr %1,1\n" \ - : "+d" (__wh), "+d" (__wl) \ - : : "0", "1", "cc"); \ - wh = __wh; \ - wl = __wl; \ -}) - -#define udiv_qrnnd(q, r, n1, n0, d) \ - do { unsigned long __n; \ - unsigned int __r, __d; \ - __n = ((unsigned long)(n1) << 32) + n0; \ - __d = (d); \ - (q) = __n / __d; \ - (r) = __n % __d; \ - } while (0) - -#define UDIV_NEEDS_NORMALIZATION 0 - -#define abort() BUG() - -#define __BYTE_ORDER __BIG_ENDIAN diff --git a/arch/s390/include/asm/sigp.h b/arch/s390/include/asm/sigp.h index 1c8f33fca356..72df5f2de6b0 100644 --- a/arch/s390/include/asm/sigp.h +++ b/arch/s390/include/asm/sigp.h @@ -37,8 +37,8 @@ #ifndef __ASSEMBLY__ -static inline int __pcpu_sigp(u16 addr, u8 order, unsigned long parm, - u32 *status) +static inline int ____pcpu_sigp(u16 addr, u8 order, unsigned long parm, + u32 *status) { register unsigned long reg1 asm ("1") = parm; int cc; @@ -48,8 +48,19 @@ static inline int __pcpu_sigp(u16 addr, u8 order, unsigned long parm, " ipm %0\n" " srl %0,28\n" : "=d" (cc), "+d" (reg1) : "d" (addr), "a" (order) : "cc"); + *status = reg1; + return cc; +} + +static inline int __pcpu_sigp(u16 addr, u8 order, unsigned long parm, + u32 *status) +{ + u32 _status; + int cc; + + cc = ____pcpu_sigp(addr, order, parm, &_status); if (status && cc == 1) - *status = reg1; + *status = _status; return cc; } diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h index 63ebf37d3143..7e9e09f600fa 100644 --- a/arch/s390/include/asm/spinlock.h +++ b/arch/s390/include/asm/spinlock.h @@ -10,6 +10,8 @@ #define __ASM_SPINLOCK_H #include +#include +#include #define SPINLOCK_LOCKVAL (S390_lowcore.spinlock_lockval) @@ -97,6 +99,7 @@ static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) { while (arch_spin_is_locked(lock)) arch_spin_relax(lock); + smp_acquire__after_ctrl_dep(); } /* diff --git a/arch/s390/include/asm/stp.h b/arch/s390/include/asm/stp.h new file mode 100644 index 000000000000..7689727585b2 --- /dev/null +++ b/arch/s390/include/asm/stp.h @@ -0,0 +1,51 @@ +/* + * Copyright IBM Corp. 2006 + * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) + */ +#ifndef __S390_STP_H +#define __S390_STP_H + +/* notifier for syncs */ +extern struct atomic_notifier_head s390_epoch_delta_notifier; + +/* STP interruption parameter */ +struct stp_irq_parm { + unsigned int _pad0 : 14; + unsigned int tsc : 1; /* Timing status change */ + unsigned int lac : 1; /* Link availability change */ + unsigned int tcpc : 1; /* Time control parameter change */ + unsigned int _pad2 : 15; +} __attribute__ ((packed)); + +#define STP_OP_SYNC 1 +#define STP_OP_CTRL 3 + +struct stp_sstpi { + unsigned int rsvd0; + unsigned int rsvd1 : 8; + unsigned int stratum : 8; + unsigned int vbits : 16; + unsigned int leaps : 16; + unsigned int tmd : 4; + unsigned int ctn : 4; + unsigned int rsvd2 : 3; + unsigned int c : 1; + unsigned int tst : 4; + unsigned int tzo : 16; + unsigned int dsto : 16; + unsigned int ctrl : 16; + unsigned int rsvd3 : 16; + unsigned int tto; + unsigned int rsvd4; + unsigned int ctnid[3]; + unsigned int rsvd5; + unsigned int todoff[4]; + unsigned int rsvd6[48]; +} __attribute__ ((packed)); + +/* Functions needed by the machine check handler */ +int stp_sync_check(void); +int stp_island_check(void); +void stp_queue_work(void); + +#endif /* __S390_STP_H */ diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index dcb6312a0b91..0bb08f341c09 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -52,6 +52,70 @@ static inline void store_clock_comparator(__u64 *time) void clock_comparator_work(void); +void __init ptff_init(void); + +extern unsigned char ptff_function_mask[16]; +extern unsigned long lpar_offset; +extern unsigned long initial_leap_seconds; + +/* Function codes for the ptff instruction. */ +#define PTFF_QAF 0x00 /* query available functions */ +#define PTFF_QTO 0x01 /* query tod offset */ +#define PTFF_QSI 0x02 /* query steering information */ +#define PTFF_QUI 0x04 /* query UTC information */ +#define PTFF_ATO 0x40 /* adjust tod offset */ +#define PTFF_STO 0x41 /* set tod offset */ +#define PTFF_SFS 0x42 /* set fine steering rate */ +#define PTFF_SGS 0x43 /* set gross steering rate */ + +/* Query TOD offset result */ +struct ptff_qto { + unsigned long long physical_clock; + unsigned long long tod_offset; + unsigned long long logical_tod_offset; + unsigned long long tod_epoch_difference; +} __packed; + +static inline int ptff_query(unsigned int nr) +{ + unsigned char *ptr; + + ptr = ptff_function_mask + (nr >> 3); + return (*ptr & (0x80 >> (nr & 7))) != 0; +} + +/* Query UTC information result */ +struct ptff_qui { + unsigned int tm : 2; + unsigned int ts : 2; + unsigned int : 28; + unsigned int pad_0x04; + unsigned long leap_event; + short old_leap; + short new_leap; + unsigned int pad_0x14; + unsigned long prt[5]; + unsigned long cst[3]; + unsigned int skew; + unsigned int pad_0x5c[41]; +} __packed; + +static inline int ptff(void *ptff_block, size_t len, unsigned int func) +{ + typedef struct { char _[len]; } addrtype; + register unsigned int reg0 asm("0") = func; + register unsigned long reg1 asm("1") = (unsigned long) ptff_block; + int rc; + + asm volatile( + " .word 0x0104\n" + " ipm %0\n" + " srl %0,28\n" + : "=d" (rc), "+m" (*(addrtype *) ptff_block) + : "d" (reg0), "d" (reg1) : "cc"); + return rc; +} + static inline unsigned long long local_tick_disable(void) { unsigned long long old; @@ -105,7 +169,7 @@ static inline cycles_t get_cycles(void) return (cycles_t) get_tod_clock() >> 2; } -int get_sync_clock(unsigned long long *clock); +int get_phys_clock(unsigned long long *clock); void init_cpu_timer(void); unsigned long long monotonic_clock(void); diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h index 7a92e69c50bc..15711de10403 100644 --- a/arch/s390/include/asm/tlb.h +++ b/arch/s390/include/asm/tlb.h @@ -87,10 +87,10 @@ static inline void tlb_finish_mmu(struct mmu_gather *tlb, * tlb_ptep_clear_flush. In both flush modes the tlb for a page cache page * has already been freed, so just do free_page_and_swap_cache. */ -static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) +static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page) { free_page_and_swap_cache(page); - return 1; /* avoid calling tlb_flush_mmu */ + return false; /* avoid calling tlb_flush_mmu */ } static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) @@ -98,6 +98,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) free_page_and_swap_cache(page); } +static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return __tlb_remove_page(tlb, page); +} + +static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, + struct page *page) +{ + return __tlb_remove_page(tlb, page); +} + +static inline void tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return tlb_remove_page(tlb, page); +} + /* * pte_free_tlb frees a pte table and clears the CRSTE for the * page table from the tlb. diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index a2e6ef32e054..1a691ef740cf 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -5,6 +5,7 @@ #include #include #include +#include /* * Flush all TLB entries on the local CPU. @@ -44,17 +45,9 @@ void smp_ptlb_all(void); */ static inline void __tlb_flush_global(void) { - register unsigned long reg2 asm("2"); - register unsigned long reg3 asm("3"); - register unsigned long reg4 asm("4"); - long dummy; - - dummy = 0; - reg2 = reg3 = 0; - reg4 = ((unsigned long) &dummy) + 1; - asm volatile( - " csp %0,%2" - : : "d" (reg2), "d" (reg3), "d" (reg4), "m" (dummy) : "cc" ); + unsigned int dummy = 0; + + csp(&dummy, 0, 0); } /* @@ -64,7 +57,7 @@ static inline void __tlb_flush_global(void) static inline void __tlb_flush_full(struct mm_struct *mm) { preempt_disable(); - atomic_add(0x10000, &mm->context.attach_count); + atomic_inc(&mm->context.flush_count); if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { /* Local TLB flush */ __tlb_flush_local(); @@ -76,21 +69,19 @@ static inline void __tlb_flush_full(struct mm_struct *mm) cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask); } - atomic_sub(0x10000, &mm->context.attach_count); + atomic_dec(&mm->context.flush_count); preempt_enable(); } /* - * Flush TLB entries for a specific ASCE on all CPUs. + * Flush TLB entries for a specific ASCE on all CPUs. Should never be used + * when more than one asce (e.g. gmap) ran on this mm. */ static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) { - int active, count; - preempt_disable(); - active = (mm == current->active_mm) ? 1 : 0; - count = atomic_add_return(0x10000, &mm->context.attach_count); - if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active && + atomic_inc(&mm->context.flush_count); + if (MACHINE_HAS_TLB_LC && cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { __tlb_flush_idte_local(asce); } else { @@ -103,7 +94,7 @@ static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask); } - atomic_sub(0x10000, &mm->context.attach_count); + atomic_dec(&mm->context.flush_count); preempt_enable(); } diff --git a/arch/s390/include/asm/topology.h b/arch/s390/include/asm/topology.h index 6b53962e807e..f15f5571ca2b 100644 --- a/arch/s390/include/asm/topology.h +++ b/arch/s390/include/asm/topology.h @@ -14,10 +14,12 @@ struct cpu_topology_s390 { unsigned short core_id; unsigned short socket_id; unsigned short book_id; + unsigned short drawer_id; unsigned short node_id; cpumask_t thread_mask; cpumask_t core_mask; cpumask_t book_mask; + cpumask_t drawer_mask; }; DECLARE_PER_CPU(struct cpu_topology_s390, cpu_topology); @@ -30,6 +32,8 @@ DECLARE_PER_CPU(struct cpu_topology_s390, cpu_topology); #define topology_core_cpumask(cpu) (&per_cpu(cpu_topology, cpu).core_mask) #define topology_book_id(cpu) (per_cpu(cpu_topology, cpu).book_id) #define topology_book_cpumask(cpu) (&per_cpu(cpu_topology, cpu).book_mask) +#define topology_drawer_id(cpu) (per_cpu(cpu_topology, cpu).drawer_id) +#define topology_drawer_cpumask(cpu) (&per_cpu(cpu_topology, cpu).drawer_mask) #define mc_capable() 1 diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index e0900ddf91dd..9b49cf1daa8f 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -151,8 +151,65 @@ unsigned long __must_check __copy_to_user(void __user *to, const void *from, __rc; \ }) -#define __put_user_fn(x, ptr, size) __put_get_user_asm(ptr, x, size, 0x810000UL) -#define __get_user_fn(x, ptr, size) __put_get_user_asm(x, ptr, size, 0x81UL) +static inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) +{ + unsigned long spec = 0x810000UL; + int rc; + + switch (size) { + case 1: + rc = __put_get_user_asm((unsigned char __user *)ptr, + (unsigned char *)x, + size, spec); + break; + case 2: + rc = __put_get_user_asm((unsigned short __user *)ptr, + (unsigned short *)x, + size, spec); + break; + case 4: + rc = __put_get_user_asm((unsigned int __user *)ptr, + (unsigned int *)x, + size, spec); + break; + case 8: + rc = __put_get_user_asm((unsigned long __user *)ptr, + (unsigned long *)x, + size, spec); + break; + }; + return rc; +} + +static inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size) +{ + unsigned long spec = 0x81UL; + int rc; + + switch (size) { + case 1: + rc = __put_get_user_asm((unsigned char *)x, + (unsigned char __user *)ptr, + size, spec); + break; + case 2: + rc = __put_get_user_asm((unsigned short *)x, + (unsigned short __user *)ptr, + size, spec); + break; + case 4: + rc = __put_get_user_asm((unsigned int *)x, + (unsigned int __user *)ptr, + size, spec); + break; + case 8: + rc = __put_get_user_asm((unsigned long *)x, + (unsigned long __user *)ptr, + size, spec); + break; + }; + return rc; +} #else /* CONFIG_HAVE_MARCH_Z10_FEATURES */ @@ -191,7 +248,7 @@ static inline int __get_user_fn(void *x, const void __user *ptr, unsigned long s __put_user_bad(); \ break; \ } \ - __pu_err; \ + __builtin_expect(__pu_err, 0); \ }) #define put_user(x, ptr) \ @@ -240,7 +297,7 @@ int __put_user_bad(void) __attribute__((noreturn)); __get_user_bad(); \ break; \ } \ - __gu_err; \ + __builtin_expect(__gu_err, 0); \ }) #define get_user(x, ptr) \ diff --git a/arch/s390/include/uapi/asm/ptrace.h b/arch/s390/include/uapi/asm/ptrace.h index a150f4fabe43..77630c74f13b 100644 --- a/arch/s390/include/uapi/asm/ptrace.h +++ b/arch/s390/include/uapi/asm/ptrace.h @@ -359,9 +359,9 @@ typedef struct per_cr_bits bits; } control_regs; /* - * Use these flags instead of setting em_instruction_fetch - * directly they are used so that single stepping can be - * switched on & off while not affecting other tracing + * The single_step and instruction_fetch bits are obsolete, + * the kernel always sets them to zero. To enable single + * stepping use ptrace(PTRACE_SINGLESTEP) instead. */ unsigned single_step : 1; unsigned instruction_fetch : 1; diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 2f5586ab8a6a..f37be37edd3a 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -2,6 +2,9 @@ # Makefile for the linux kernel. # +KCOV_INSTRUMENT_early.o := n +KCOV_INSTRUMENT_sclp.o := n + ifdef CONFIG_FUNCTION_TRACER # Don't trace early setup code and tracing code CFLAGS_REMOVE_early.o = $(CC_FLAGS_FTRACE) @@ -45,7 +48,7 @@ obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += debug.o irq.o ipl.o dis.o diag.o sclp.o vdso.o obj-y += sysinfo.o jump_label.o lgr.o os_info.o machine_kexec.o pgm_check.o -obj-y += runtime_instr.o cache.o dumpstack.o +obj-y += runtime_instr.o cache.o fpu.o dumpstack.o obj-y += entry.o reipl.o relocate_kernel.o extra-y += head.o head64.o vmlinux.lds diff --git a/arch/s390/kernel/cache.c b/arch/s390/kernel/cache.c index 77a84bd78be2..c8a83276a4dc 100644 --- a/arch/s390/kernel/cache.c +++ b/arch/s390/kernel/cache.c @@ -99,12 +99,7 @@ static inline enum cache_type get_cache_type(struct cache_info *ci, int level) static inline unsigned long ecag(int ai, int li, int ti) { - unsigned long cmd, val; - - cmd = ai << 4 | li << 1 | ti; - asm volatile(".insn rsy,0xeb000000004c,%0,0,0(%1)" /* ecag */ - : "=d" (val) : "a" (cmd)); - return val; + return __ecag(ECAG_CACHE_ATTRIBUTE, ai << 4 | li << 1 | ti); } static void ci_leaf_init(struct cacheinfo *this_leaf, int private, diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 8cb9bfdd3ea8..43446fa2a4e5 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index 69f9908ac44c..6693383bc01b 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -78,14 +78,10 @@ void dump_trace(dump_trace_func_t func, void *data, struct task_struct *task, sp = __dump_trace(func, data, sp, S390_lowcore.async_stack + frame_size - ASYNC_SIZE, S390_lowcore.async_stack + frame_size); - if (task) - __dump_trace(func, data, sp, - (unsigned long)task_stack_page(task), - (unsigned long)task_stack_page(task) + THREAD_SIZE); - else - __dump_trace(func, data, sp, - S390_lowcore.thread_info, - S390_lowcore.thread_info + THREAD_SIZE); + task = task ?: current; + __dump_trace(func, data, sp, + (unsigned long)task_stack_page(task), + (unsigned long)task_stack_page(task) + THREAD_SIZE); } EXPORT_SYMBOL_GPL(dump_trace); diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index a0684de5a93b..717b03aa16b5 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -231,6 +231,26 @@ static noinline __init void detect_machine_type(void) S390_lowcore.machine_flags |= MACHINE_FLAG_VM; } +static noinline __init void setup_arch_string(void) +{ + struct sysinfo_1_1_1 *mach = (struct sysinfo_1_1_1 *)&sysinfo_page; + + if (stsi(mach, 1, 1, 1)) + return; + EBCASC(mach->manufacturer, sizeof(mach->manufacturer)); + EBCASC(mach->type, sizeof(mach->type)); + EBCASC(mach->model, sizeof(mach->model)); + EBCASC(mach->model_capacity, sizeof(mach->model_capacity)); + dump_stack_set_arch_desc("%-16.16s %-4.4s %-16.16s %-16.16s (%s)", + mach->manufacturer, + mach->type, + mach->model, + mach->model_capacity, + MACHINE_IS_LPAR ? "LPAR" : + MACHINE_IS_VM ? "z/VM" : + MACHINE_IS_KVM ? "KVM" : "unknown"); +} + static __init void setup_topology(void) { int max_mnest; @@ -447,11 +467,13 @@ void __init startup_init(void) ipl_save_parameters(); rescue_initrd(); clear_bss_section(); + ptff_init(); init_kernel_storage_key(); lockdep_off(); setup_lowcore_early(); setup_facility_list(); detect_machine_type(); + setup_arch_string(); ipl_update_parameters(); setup_boot_command_line(); create_kernel_nss(); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 2d47f9cfcb36..c51650a1ed16 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -163,6 +163,16 @@ _PIF_WORK = (_PIF_PER_TRAP) .endm .section .kprobes.text, "ax" +.Ldummy: + /* + * This nop exists only in order to avoid that __switch_to starts at + * the beginning of the kprobes text section. In that case we would + * have several symbols at the same address. E.g. objdump would take + * an arbitrary symbol name when disassembling this code. + * With the added nop in between the __switch_to symbol is unique + * again. + */ + nop 0 /* * Scheduler resume function, called by switch_to @@ -175,7 +185,6 @@ ENTRY(__switch_to) stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task lgr %r1,%r2 aghi %r1,__TASK_thread # thread_struct of prev task - lg %r4,__TASK_thread_info(%r2) # get thread_info of prev lg %r5,__TASK_thread_info(%r3) # get thread_info of next stg %r15,__THREAD_ksp(%r1) # store kernel stack of prev lgr %r1,%r3 diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c new file mode 100644 index 000000000000..81d1d1887507 --- /dev/null +++ b/arch/s390/kernel/fpu.c @@ -0,0 +1,249 @@ +/* + * In-kernel vector facility support functions + * + * Copyright IBM Corp. 2015 + * Author(s): Hendrik Brueckner + */ +#include +#include +#include +#include +#include + +/* + * Per-CPU variable to maintain FPU register ranges that are in use + * by the kernel. + */ +static DEFINE_PER_CPU(u32, kernel_fpu_state); + +#define KERNEL_FPU_STATE_MASK (KERNEL_FPU_MASK|KERNEL_FPC) + + +void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) +{ + if (!__this_cpu_read(kernel_fpu_state)) { + /* + * Save user space FPU state and register contents. Multiple + * calls because of interruptions do not matter and return + * immediately. This also sets CIF_FPU to lazy restore FP/VX + * register contents when returning to user space. + */ + save_fpu_regs(); + } + + /* Update flags to use the vector facility for KERNEL_FPR */ + if (MACHINE_HAS_VX && (state->mask & KERNEL_FPR)) { + flags |= KERNEL_VXR_LOW | KERNEL_FPC; + flags &= ~KERNEL_FPR; + } + + /* Save and update current kernel VX state */ + state->mask = __this_cpu_read(kernel_fpu_state); + __this_cpu_or(kernel_fpu_state, flags & KERNEL_FPU_STATE_MASK); + + /* + * If this is the first call to __kernel_fpu_begin(), no additional + * work is required. + */ + if (!(state->mask & KERNEL_FPU_STATE_MASK)) + return; + + /* + * If KERNEL_FPR is still set, the vector facility is not available + * and, thus, save floating-point control and registers only. + */ + if (state->mask & KERNEL_FPR) { + asm volatile("stfpc %0" : "=Q" (state->fpc)); + asm volatile("std 0,%0" : "=Q" (state->fprs[0])); + asm volatile("std 1,%0" : "=Q" (state->fprs[1])); + asm volatile("std 2,%0" : "=Q" (state->fprs[2])); + asm volatile("std 3,%0" : "=Q" (state->fprs[3])); + asm volatile("std 4,%0" : "=Q" (state->fprs[4])); + asm volatile("std 5,%0" : "=Q" (state->fprs[5])); + asm volatile("std 6,%0" : "=Q" (state->fprs[6])); + asm volatile("std 7,%0" : "=Q" (state->fprs[7])); + asm volatile("std 8,%0" : "=Q" (state->fprs[8])); + asm volatile("std 9,%0" : "=Q" (state->fprs[9])); + asm volatile("std 10,%0" : "=Q" (state->fprs[10])); + asm volatile("std 11,%0" : "=Q" (state->fprs[11])); + asm volatile("std 12,%0" : "=Q" (state->fprs[12])); + asm volatile("std 13,%0" : "=Q" (state->fprs[13])); + asm volatile("std 14,%0" : "=Q" (state->fprs[14])); + asm volatile("std 15,%0" : "=Q" (state->fprs[15])); + return; + } + + /* + * If this is a nested call to __kernel_fpu_begin(), check the saved + * state mask to save and later restore the vector registers that + * are already in use. Let's start with checking floating-point + * controls. + */ + if (state->mask & KERNEL_FPC) + asm volatile("stfpc %0" : "=m" (state->fpc)); + + /* Test and save vector registers */ + asm volatile ( + /* + * Test if any vector register must be saved and, if so, + * test if all register can be saved. + */ + " tmll %[m],15\n" /* KERNEL_VXR_MASK */ + " jz 20f\n" /* no work -> done */ + " la 1,%[vxrs]\n" /* load save area */ + " jo 18f\n" /* -> save V0..V31 */ + + /* + * Test if V8..V23 can be saved at once... this speeds up + * for KERNEL_fpu_MID only. Otherwise continue to split the + * range of vector registers into two halves and test them + * separately. + */ + " tmll %[m],6\n" /* KERNEL_VXR_MID */ + " jo 17f\n" /* -> save V8..V23 */ + + /* Test and save the first half of 16 vector registers */ + "1: tmll %[m],3\n" /* KERNEL_VXR_LOW */ + " jz 10f\n" /* -> KERNEL_VXR_HIGH */ + " jo 2f\n" /* 11 -> save V0..V15 */ + " brc 4,3f\n" /* 01 -> save V0..V7 */ + " brc 2,4f\n" /* 10 -> save V8..V15 */ + + /* Test and save the second half of 16 vector registers */ + "10: tmll %[m],12\n" /* KERNEL_VXR_HIGH */ + " jo 19f\n" /* 11 -> save V16..V31 */ + " brc 4,11f\n" /* 01 -> save V16..V23 */ + " brc 2,12f\n" /* 10 -> save V24..V31 */ + " j 20f\n" /* 00 -> done */ + + /* + * Below are the vstm combinations to save multiple vector + * registers at once. + */ + "2: .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */ + " j 10b\n" /* -> VXR_HIGH */ + "3: .word 0xe707,0x1000,0x003e\n" /* vstm 0,7,0(1) */ + " j 10b\n" /* -> VXR_HIGH */ + "4: .word 0xe78f,0x1080,0x003e\n" /* vstm 8,15,128(1) */ + " j 10b\n" /* -> VXR_HIGH */ + "\n" + "11: .word 0xe707,0x1100,0x0c3e\n" /* vstm 16,23,256(1) */ + " j 20f\n" /* -> done */ + "12: .word 0xe78f,0x1180,0x0c3e\n" /* vstm 24,31,384(1) */ + " j 20f\n" /* -> done */ + "\n" + "17: .word 0xe787,0x1080,0x043e\n" /* vstm 8,23,128(1) */ + " nill %[m],249\n" /* m &= ~VXR_MID */ + " j 1b\n" /* -> VXR_LOW */ + "\n" + "18: .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */ + "19: .word 0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */ + "20:" + : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs) + : [m] "d" (state->mask) + : "1", "cc"); +} +EXPORT_SYMBOL(__kernel_fpu_begin); + +void __kernel_fpu_end(struct kernel_fpu *state) +{ + /* Just update the per-CPU state if there is nothing to restore */ + if (!(state->mask & KERNEL_FPU_STATE_MASK)) + goto update_fpu_state; + + /* + * If KERNEL_FPR is specified, the vector facility is not available + * and, thus, restore floating-point control and registers only. + */ + if (state->mask & KERNEL_FPR) { + asm volatile("lfpc %0" : : "Q" (state->fpc)); + asm volatile("ld 0,%0" : : "Q" (state->fprs[0])); + asm volatile("ld 1,%0" : : "Q" (state->fprs[1])); + asm volatile("ld 2,%0" : : "Q" (state->fprs[2])); + asm volatile("ld 3,%0" : : "Q" (state->fprs[3])); + asm volatile("ld 4,%0" : : "Q" (state->fprs[4])); + asm volatile("ld 5,%0" : : "Q" (state->fprs[5])); + asm volatile("ld 6,%0" : : "Q" (state->fprs[6])); + asm volatile("ld 7,%0" : : "Q" (state->fprs[7])); + asm volatile("ld 8,%0" : : "Q" (state->fprs[8])); + asm volatile("ld 9,%0" : : "Q" (state->fprs[9])); + asm volatile("ld 10,%0" : : "Q" (state->fprs[10])); + asm volatile("ld 11,%0" : : "Q" (state->fprs[11])); + asm volatile("ld 12,%0" : : "Q" (state->fprs[12])); + asm volatile("ld 13,%0" : : "Q" (state->fprs[13])); + asm volatile("ld 14,%0" : : "Q" (state->fprs[14])); + asm volatile("ld 15,%0" : : "Q" (state->fprs[15])); + goto update_fpu_state; + } + + /* Test and restore floating-point controls */ + if (state->mask & KERNEL_FPC) + asm volatile("lfpc %0" : : "Q" (state->fpc)); + + /* Test and restore (load) vector registers */ + asm volatile ( + /* + * Test if any vector registers must be loaded and, if so, + * test if all registers can be loaded at once. + */ + " tmll %[m],15\n" /* KERNEL_VXR_MASK */ + " jz 20f\n" /* no work -> done */ + " la 1,%[vxrs]\n" /* load load area */ + " jo 18f\n" /* -> load V0..V31 */ + + /* + * Test if V8..V23 can be restored at once... this speeds up + * for KERNEL_VXR_MID only. Otherwise continue to split the + * range of vector registers into two halves and test them + * separately. + */ + " tmll %[m],6\n" /* KERNEL_VXR_MID */ + " jo 17f\n" /* -> load V8..V23 */ + + /* Test and load the first half of 16 vector registers */ + "1: tmll %[m],3\n" /* KERNEL_VXR_LOW */ + " jz 10f\n" /* -> KERNEL_VXR_HIGH */ + " jo 2f\n" /* 11 -> load V0..V15 */ + " brc 4,3f\n" /* 01 -> load V0..V7 */ + " brc 2,4f\n" /* 10 -> load V8..V15 */ + + /* Test and load the second half of 16 vector registers */ + "10: tmll %[m],12\n" /* KERNEL_VXR_HIGH */ + " jo 19f\n" /* 11 -> load V16..V31 */ + " brc 4,11f\n" /* 01 -> load V16..V23 */ + " brc 2,12f\n" /* 10 -> load V24..V31 */ + " j 20f\n" /* 00 -> done */ + + /* + * Below are the vstm combinations to load multiple vector + * registers at once. + */ + "2: .word 0xe70f,0x1000,0x0036\n" /* vlm 0,15,0(1) */ + " j 10b\n" /* -> VXR_HIGH */ + "3: .word 0xe707,0x1000,0x0036\n" /* vlm 0,7,0(1) */ + " j 10b\n" /* -> VXR_HIGH */ + "4: .word 0xe78f,0x1080,0x0036\n" /* vlm 8,15,128(1) */ + " j 10b\n" /* -> VXR_HIGH */ + "\n" + "11: .word 0xe707,0x1100,0x0c36\n" /* vlm 16,23,256(1) */ + " j 20f\n" /* -> done */ + "12: .word 0xe78f,0x1180,0x0c36\n" /* vlm 24,31,384(1) */ + " j 20f\n" /* -> done */ + "\n" + "17: .word 0xe787,0x1080,0x0436\n" /* vlm 8,23,128(1) */ + " nill %[m],249\n" /* m &= ~VXR_MID */ + " j 1b\n" /* -> VXR_LOW */ + "\n" + "18: .word 0xe70f,0x1000,0x0036\n" /* vlm 0,15,0(1) */ + "19: .word 0xe70f,0x1100,0x0c36\n" /* vlm 16,31,256(1) */ + "20:" + : + : [vxrs] "Q" (*(struct vx_array *) &state->vxrs), + [m] "d" (state->mask) + : "1", "cc"); + +update_fpu_state: + /* Update current kernel VX state */ + __this_cpu_write(kernel_fpu_state, state->mask); +} +EXPORT_SYMBOL(__kernel_fpu_end); diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index f20abdb5630a..295bfb7124bc 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -121,9 +121,9 @@ static char *dump_type_str(enum dump_type type) * Must be in data section since the bss section * is not cleared when these are accessed. */ -static u8 ipl_ssid __attribute__((__section__(".data"))) = 0; -static u16 ipl_devno __attribute__((__section__(".data"))) = 0; -u32 ipl_flags __attribute__((__section__(".data"))) = 0; +static u8 ipl_ssid __section(.data) = 0; +static u16 ipl_devno __section(.data) = 0; +u32 ipl_flags __section(.data) = 0; enum ipl_method { REIPL_METHOD_CCW_CIO, @@ -174,7 +174,7 @@ static inline int __diag308(unsigned long subcode, void *addr) asm volatile( " diag %0,%2,0x308\n" - "0:\n" + "0: nopr %%r7\n" EX_TABLE(0b,0b) : "+d" (_addr), "+d" (_rc) : "d" (subcode) : "cc", "memory"); @@ -563,7 +563,7 @@ static struct kset *ipl_kset; static void __ipl_run(void *unused) { - diag308(DIAG308_IPL, NULL); + diag308(DIAG308_LOAD_CLEAR, NULL); if (MACHINE_IS_VM) __cpcmd("IPL", NULL, 0, NULL); else if (ipl_info.type == IPL_TYPE_CCW) @@ -1085,21 +1085,24 @@ static void __reipl_run(void *unused) break; case REIPL_METHOD_CCW_DIAG: diag308(DIAG308_SET, reipl_block_ccw); - diag308(DIAG308_IPL, NULL); + if (MACHINE_IS_LPAR) + diag308(DIAG308_LOAD_NORMAL_DUMP, NULL); + else + diag308(DIAG308_LOAD_CLEAR, NULL); break; case REIPL_METHOD_FCP_RW_DIAG: diag308(DIAG308_SET, reipl_block_fcp); - diag308(DIAG308_IPL, NULL); + diag308(DIAG308_LOAD_CLEAR, NULL); break; case REIPL_METHOD_FCP_RO_DIAG: - diag308(DIAG308_IPL, NULL); + diag308(DIAG308_LOAD_CLEAR, NULL); break; case REIPL_METHOD_FCP_RO_VM: __cpcmd("IPL", NULL, 0, NULL); break; case REIPL_METHOD_NSS_DIAG: diag308(DIAG308_SET, reipl_block_nss); - diag308(DIAG308_IPL, NULL); + diag308(DIAG308_LOAD_CLEAR, NULL); break; case REIPL_METHOD_NSS: get_ipl_string(buf, reipl_block_nss, REIPL_METHOD_NSS); @@ -1108,7 +1111,7 @@ static void __reipl_run(void *unused) case REIPL_METHOD_DEFAULT: if (MACHINE_IS_VM) __cpcmd("IPL", NULL, 0, NULL); - diag308(DIAG308_IPL, NULL); + diag308(DIAG308_LOAD_CLEAR, NULL); break; case REIPL_METHOD_FCP_DUMP: break; @@ -1423,7 +1426,7 @@ static void diag308_dump(void *dump_block) { diag308(DIAG308_SET, dump_block); while (1) { - if (diag308(DIAG308_DUMP, NULL) != 0x302) + if (diag308(DIAG308_LOAD_NORMAL_DUMP, NULL) != 0x302) break; udelay_simple(USEC_PER_SEC); } @@ -2064,12 +2067,5 @@ void s390_reset_system(void) S390_lowcore.program_new_psw.addr = (unsigned long) s390_base_pgm_handler; - /* - * Clear subchannel ID and number to signal new kernel that no CCW or - * SCSI IPL has been done (for kexec and kdump) - */ - S390_lowcore.subchannel_id = 0; - S390_lowcore.subchannel_nr = 0; - do_reset_calls(); } diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index c373a1d41d10..285d6561076d 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -127,9 +127,7 @@ int show_interrupts(struct seq_file *p, void *v) seq_printf(p, "CPU%d ", cpu); seq_putc(p, '\n'); } - if (index < NR_IRQS) { - if (index >= NR_IRQS_BASE) - goto out; + if (index < NR_IRQS_BASE) { seq_printf(p, "%s: ", irqclass_main_desc[index].name); irq = irqclass_main_desc[index].irq; for_each_online_cpu(cpu) @@ -137,6 +135,9 @@ int show_interrupts(struct seq_file *p, void *v) seq_putc(p, '\n'); goto out; } + if (index > NR_IRQS_BASE) + goto out; + for (index = 0; index < NR_ARCH_IRQS; index++) { seq_printf(p, "%s: ", irqclass_sub_desc[index].name); irq = irqclass_sub_desc[index].irq; diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 0e64f08d3d69..3074c1d83829 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -60,8 +61,6 @@ static int machine_kdump_pm_cb(struct notifier_block *nb, unsigned long action, static int __init machine_kdump_pm_init(void) { pm_notifier(machine_kdump_pm_cb, 0); - /* Create initial mapping for crashkernel memory */ - arch_kexec_unprotect_crashkres(); return 0; } arch_initcall(machine_kdump_pm_init); @@ -150,42 +149,40 @@ static int kdump_csum_valid(struct kimage *image) #ifdef CONFIG_CRASH_DUMP -/* - * Map or unmap crashkernel memory - */ -static void crash_map_pages(int enable) +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end) { - unsigned long size = resource_size(&crashk_res); - - BUG_ON(crashk_res.start % KEXEC_CRASH_MEM_ALIGN || - size % KEXEC_CRASH_MEM_ALIGN); - if (enable) - vmem_add_mapping(crashk_res.start, size); - else { - vmem_remove_mapping(crashk_res.start, size); - if (size) - os_info_crashkernel_add(crashk_res.start, size); - else - os_info_crashkernel_add(0, 0); - } + unsigned long addr, size; + + for (addr = begin; addr < end; addr += PAGE_SIZE) + free_reserved_page(pfn_to_page(addr >> PAGE_SHIFT)); + size = begin - crashk_res.start; + if (size) + os_info_crashkernel_add(crashk_res.start, size); + else + os_info_crashkernel_add(0, 0); +} + +static void crash_protect_pages(int protect) +{ + unsigned long size; + + if (!crashk_res.end) + return; + size = resource_size(&crashk_res); + if (protect) + set_memory_ro(crashk_res.start, size >> PAGE_SHIFT); + else + set_memory_rw(crashk_res.start, size >> PAGE_SHIFT); } -/* - * Unmap crashkernel memory - */ void arch_kexec_protect_crashkres(void) { - if (crashk_res.end) - crash_map_pages(0); + crash_protect_pages(1); } -/* - * Map crashkernel memory - */ void arch_kexec_unprotect_crashkres(void) { - if (crashk_res.end) - crash_map_pages(1); + crash_protect_pages(0); } #endif diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 07302ce37648..29376f0e725c 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include @@ -27,7 +27,6 @@ struct mcck_struct { unsigned int kill_task : 1; unsigned int channel_report : 1; unsigned int warning : 1; - unsigned int etr_queue : 1; unsigned int stp_queue : 1; unsigned long mcck_code; }; @@ -82,8 +81,6 @@ void s390_handle_mcck(void) if (xchg(&mchchk_wng_posted, 1) == 0) kill_cad_pid(SIGPWR, 1); } - if (mcck.etr_queue) - etr_queue_work(); if (mcck.stp_queue) stp_queue_work(); if (mcck.kill_task) { @@ -241,8 +238,6 @@ static int notrace s390_validate_registers(union mci mci) #define ED_STP_ISLAND 6 /* External damage STP island check */ #define ED_STP_SYNC 7 /* External damage STP sync check */ -#define ED_ETR_SYNC 12 /* External damage ETR sync check */ -#define ED_ETR_SWITCH 13 /* External damage ETR switch to local */ /* * machine check handler. @@ -325,15 +320,11 @@ void notrace s390_do_machine_check(struct pt_regs *regs) } if (mci.ed && mci.ec) { /* External damage */ - if (S390_lowcore.external_damage_code & (1U << ED_ETR_SYNC)) - mcck->etr_queue |= etr_sync_check(); - if (S390_lowcore.external_damage_code & (1U << ED_ETR_SWITCH)) - mcck->etr_queue |= etr_switch_to_local(); if (S390_lowcore.external_damage_code & (1U << ED_STP_SYNC)) mcck->stp_queue |= stp_sync_check(); if (S390_lowcore.external_damage_code & (1U << ED_STP_ISLAND)) mcck->stp_queue |= stp_island_check(); - if (mcck->etr_queue || mcck->stp_queue) + if (mcck->stp_queue) set_cpu_flag(CIF_MCCK_PENDING); } if (mci.se) diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 59215c518f37..7ec63b1d920d 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -649,6 +649,8 @@ static int cpumf_pmu_commit_txn(struct pmu *pmu) /* Performance monitoring unit for s390x */ static struct pmu cpumf_pmu = { + .task_ctx_nr = perf_sw_context, + .capabilities = PERF_PMU_CAP_NO_INTERRUPT, .pmu_enable = cpumf_pmu_enable, .pmu_disable = cpumf_pmu_disable, .event_init = cpumf_pmu_event_init, @@ -708,12 +710,6 @@ static int __init cpumf_pmu_init(void) goto out; } - /* The CPU measurement counter facility does not have overflow - * interrupts to do sampling. Sampling must be provided by - * external means, for example, by timers. - */ - cpumf_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; - cpumf_pmu.attr_groups = cpumf_cf_event_group(); rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", PERF_TYPE_RAW); if (rc) { diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index a8e832166417..53acf2d76fa9 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -601,17 +601,12 @@ static void release_pmc_hardware(void) irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); on_each_cpu(setup_pmc_cpu, &flags, 1); - perf_release_sampling(); } static int reserve_pmc_hardware(void) { int flags = PMC_INIT; - int err; - err = perf_reserve_sampling(); - if (err) - return err; on_each_cpu(setup_pmc_cpu, &flags, 1); if (flags & PMC_FAILURE) { release_pmc_hardware(); @@ -979,12 +974,15 @@ static int perf_push_sample(struct perf_event *event, struct sf_raw_sample *sfr) struct pt_regs regs; struct perf_sf_sde_regs *sde_regs; struct perf_sample_data data; - struct perf_raw_record raw; + struct perf_raw_record raw = { + .frag = { + .size = sfr->size, + .data = sfr, + }, + }; /* Setup perf sample */ perf_sample_data_init(&data, 0, event->hw.last_period); - raw.size = sfr->size; - raw.data = sfr; data.raw = &raw; /* Setup pt_regs to look like an CPU-measurement external interrupt diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c index 87035fa58bbe..17431f63de00 100644 --- a/arch/s390/kernel/perf_event.c +++ b/arch/s390/kernel/perf_event.c @@ -248,33 +248,3 @@ ssize_t cpumf_events_sysfs_show(struct device *dev, return sprintf(page, "event=0x%04llx,name=%s\n", pmu_attr->id, attr->attr.name); } - -/* Reserve/release functions for sharing perf hardware */ -static DEFINE_SPINLOCK(perf_hw_owner_lock); -static void *perf_sampling_owner; - -int perf_reserve_sampling(void) -{ - int err; - - err = 0; - spin_lock(&perf_hw_owner_lock); - if (perf_sampling_owner) { - pr_warn("The sampling facility is already reserved by %p\n", - perf_sampling_owner); - err = -EBUSY; - } else - perf_sampling_owner = __builtin_return_address(0); - spin_unlock(&perf_hw_owner_lock); - return err; -} -EXPORT_SYMBOL(perf_reserve_sampling); - -void perf_release_sampling(void) -{ - spin_lock(&perf_hw_owner_lock); - WARN_ON(!perf_sampling_owner); - perf_sampling_owner = NULL; - spin_unlock(&perf_hw_owner_lock); -} -EXPORT_SYMBOL(perf_release_sampling); diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index de7451065c34..81d0808085e6 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -13,12 +13,45 @@ #include #include #include +#include #include #include #include #include -static DEFINE_PER_CPU(struct cpuid, cpu_id); +struct cpu_info { + unsigned int cpu_mhz_dynamic; + unsigned int cpu_mhz_static; + struct cpuid cpu_id; +}; + +static DEFINE_PER_CPU(struct cpu_info, cpu_info); + +static bool machine_has_cpu_mhz; + +void __init cpu_detect_mhz_feature(void) +{ + if (test_facility(34) && __ecag(ECAG_CPU_ATTRIBUTE, 0) != -1UL) + machine_has_cpu_mhz = 1; +} + +static void update_cpu_mhz(void *arg) +{ + unsigned long mhz; + struct cpu_info *c; + + mhz = __ecag(ECAG_CPU_ATTRIBUTE, 0); + c = this_cpu_ptr(&cpu_info); + c->cpu_mhz_dynamic = mhz >> 32; + c->cpu_mhz_static = mhz & 0xffffffff; +} + +void s390_update_cpu_mhz(void) +{ + s390_adjust_jiffies(); + if (machine_has_cpu_mhz) + on_each_cpu(update_cpu_mhz, NULL, 0); +} void notrace cpu_relax(void) { @@ -35,9 +68,11 @@ EXPORT_SYMBOL(cpu_relax); */ void cpu_init(void) { - struct cpuid *id = this_cpu_ptr(&cpu_id); + struct cpuid *id = this_cpu_ptr(&cpu_info.cpu_id); get_cpu_id(id); + if (machine_has_cpu_mhz) + update_cpu_mhz(NULL); atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; BUG_ON(current->mm); @@ -53,10 +88,7 @@ int cpu_have_feature(unsigned int num) } EXPORT_SYMBOL(cpu_have_feature); -/* - * show_cpuinfo - Get information on one CPU for use by procfs. - */ -static int show_cpuinfo(struct seq_file *m, void *v) +static void show_cpu_summary(struct seq_file *m, void *v) { static const char *hwcap_str[] = { "esan3", "zarch", "stfle", "msa", "ldisp", "eimm", "dfp", @@ -65,34 +97,55 @@ static int show_cpuinfo(struct seq_file *m, void *v) static const char * const int_hwcap_str[] = { "sie" }; - unsigned long n = (unsigned long) v - 1; - int i; - - if (!n) { - s390_adjust_jiffies(); - seq_printf(m, "vendor_id : IBM/S390\n" - "# processors : %i\n" - "bogomips per cpu: %lu.%02lu\n", - num_online_cpus(), loops_per_jiffy/(500000/HZ), - (loops_per_jiffy/(5000/HZ))%100); - seq_puts(m, "features\t: "); - for (i = 0; i < ARRAY_SIZE(hwcap_str); i++) - if (hwcap_str[i] && (elf_hwcap & (1UL << i))) - seq_printf(m, "%s ", hwcap_str[i]); - for (i = 0; i < ARRAY_SIZE(int_hwcap_str); i++) - if (int_hwcap_str[i] && (int_hwcap & (1UL << i))) - seq_printf(m, "%s ", int_hwcap_str[i]); - seq_puts(m, "\n"); - show_cacheinfo(m); - } - if (cpu_online(n)) { - struct cpuid *id = &per_cpu(cpu_id, n); - seq_printf(m, "processor %li: " + int i, cpu; + + seq_printf(m, "vendor_id : IBM/S390\n" + "# processors : %i\n" + "bogomips per cpu: %lu.%02lu\n", + num_online_cpus(), loops_per_jiffy/(500000/HZ), + (loops_per_jiffy/(5000/HZ))%100); + seq_printf(m, "max thread id : %d\n", smp_cpu_mtid); + seq_puts(m, "features\t: "); + for (i = 0; i < ARRAY_SIZE(hwcap_str); i++) + if (hwcap_str[i] && (elf_hwcap & (1UL << i))) + seq_printf(m, "%s ", hwcap_str[i]); + for (i = 0; i < ARRAY_SIZE(int_hwcap_str); i++) + if (int_hwcap_str[i] && (int_hwcap & (1UL << i))) + seq_printf(m, "%s ", int_hwcap_str[i]); + seq_puts(m, "\n"); + show_cacheinfo(m); + for_each_online_cpu(cpu) { + struct cpuid *id = &per_cpu(cpu_info.cpu_id, cpu); + + seq_printf(m, "processor %d: " "version = %02X, " "identification = %06X, " "machine = %04X\n", - n, id->version, id->ident, id->machine); + cpu, id->version, id->ident, id->machine); } +} + +static void show_cpu_mhz(struct seq_file *m, unsigned long n) +{ + struct cpu_info *c = per_cpu_ptr(&cpu_info, n); + + seq_printf(m, "cpu MHz dynamic : %d\n", c->cpu_mhz_dynamic); + seq_printf(m, "cpu MHz static : %d\n", c->cpu_mhz_static); +} + +/* + * show_cpuinfo - Get information on one CPU for use by procfs. + */ +static int show_cpuinfo(struct seq_file *m, void *v) +{ + unsigned long n = (unsigned long) v - 1; + + if (!n) + show_cpu_summary(m, v); + if (!machine_has_cpu_mhz) + return 0; + seq_printf(m, "\ncpu number : %ld\n", n); + show_cpu_mhz(m, n); return 0; } @@ -126,4 +179,3 @@ const struct seq_operations cpuinfo_op = { .stop = c_stop, .show = show_cpuinfo, }; - diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index f31939147ccd..ba5f456edaa9 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -130,17 +130,14 @@ __setup("condev=", condev_setup); static void __init set_preferred_console(void) { - if (MACHINE_IS_KVM) { - if (sclp.has_vt220) - add_preferred_console("ttyS", 1, NULL); - else if (sclp.has_linemode) - add_preferred_console("ttyS", 0, NULL); - else - add_preferred_console("hvc", 0, NULL); - } else if (CONSOLE_IS_3215 || CONSOLE_IS_SCLP) + if (CONSOLE_IS_3215 || CONSOLE_IS_SCLP) add_preferred_console("ttyS", 0, NULL); else if (CONSOLE_IS_3270) add_preferred_console("tty3270", 0, NULL); + else if (CONSOLE_IS_VT220) + add_preferred_console("ttyS", 1, NULL); + else if (CONSOLE_IS_HVC) + add_preferred_console("hvc", 0, NULL); } static int __init conmode_setup(char *str) @@ -206,6 +203,15 @@ static void __init conmode_default(void) SET_CONSOLE_SCLP; #endif } + } else if (MACHINE_IS_KVM) { + if (sclp.has_vt220 && + config_enabled(CONFIG_SCLP_VT220_CONSOLE)) + SET_CONSOLE_VT220; + else if (sclp.has_linemode && + config_enabled(CONFIG_SCLP_CONSOLE)) + SET_CONSOLE_SCLP; + else + SET_CONSOLE_HVC; } else { #if defined(CONFIG_SCLP_CONSOLE) || defined(CONFIG_SCLP_VT220_CONSOLE) SET_CONSOLE_SCLP; @@ -289,7 +295,7 @@ static int __init parse_vmalloc(char *arg) } early_param("vmalloc", parse_vmalloc); -void *restart_stack __attribute__((__section__(".data"))); +void *restart_stack __section(.data); static void __init setup_lowcore(void) { @@ -432,6 +438,20 @@ static void __init setup_resources(void) } } } +#ifdef CONFIG_CRASH_DUMP + /* + * Re-add removed crash kernel memory as reserved memory. This makes + * sure it will be mapped with the identity mapping and struct pages + * will be created, so it can be resized later on. + * However add it later since the crash kernel resource should not be + * part of the System RAM resource. + */ + if (crashk_res.end) { + memblock_add(crashk_res.start, resource_size(&crashk_res)); + memblock_reserve(crashk_res.start, resource_size(&crashk_res)); + insert_resource(&iomem_resource, &crashk_res); + } +#endif } static void __init setup_memory_end(void) @@ -602,7 +622,6 @@ static void __init reserve_crashkernel(void) diag10_range(PFN_DOWN(crash_base), PFN_DOWN(crash_size)); crashk_res.start = crash_base; crashk_res.end = crash_base + crash_size - 1; - insert_resource(&iomem_resource, &crashk_res); memblock_remove(crash_base, crash_size); pr_info("Reserving %lluMB of memory at %lluMB " "for crashkernel (System RAM: %luMB)\n", @@ -901,6 +920,7 @@ void __init setup_arch(char **cmdline_p) setup_vmcoreinfo(); setup_lowcore(); smp_fill_possible_mask(); + cpu_detect_mhz_feature(); cpu_init(); numa_setup(); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 7b89a7572100..35531fe1c5ea 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -242,10 +242,8 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) { struct lowcore *lc = pcpu->lowcore; - if (MACHINE_HAS_TLB_LC) - cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask); + cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask); cpumask_set_cpu(cpu, mm_cpumask(&init_mm)); - atomic_inc(&init_mm.context.attach_count); lc->cpu_nr = cpu; lc->spinlock_lockval = arch_spin_lockval(cpu); lc->percpu_offset = __per_cpu_offset[cpu]; @@ -320,17 +318,11 @@ static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *), */ static int pcpu_set_smt(unsigned int mtid) { - register unsigned long reg1 asm ("1") = (unsigned long) mtid; int cc; if (smp_cpu_mtid == mtid) return 0; - asm volatile( - " sigp %1,0,%2 # sigp set multi-threading\n" - " ipm %0\n" - " srl %0,28\n" - : "=d" (cc) : "d" (reg1), "K" (SIGP_SET_MULTI_THREADING) - : "cc"); + cc = __pcpu_sigp(0, SIGP_SET_MULTI_THREADING, mtid, NULL); if (cc == 0) { smp_cpu_mtid = mtid; smp_cpu_mt_shift = 0; @@ -876,10 +868,8 @@ void __cpu_die(unsigned int cpu) while (!pcpu_stopped(pcpu)) cpu_relax(); pcpu_free_lowcore(pcpu); - atomic_dec(&init_mm.context.attach_count); cpumask_clear_cpu(cpu, mm_cpumask(&init_mm)); - if (MACHINE_HAS_TLB_LC) - cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask); + cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask); } void __noreturn cpu_die(void) @@ -897,7 +887,7 @@ void __init smp_fill_possible_mask(void) sclp_max = max(sclp.mtid, sclp.mtid_cp) + 1; sclp_max = min(smp_max_threads, sclp_max); - sclp_max = sclp.max_cores * sclp_max ?: nr_cpu_ids; + sclp_max = (sclp.max_cores * sclp_max) ?: nr_cpu_ids; possible = setup_possible_cpus ?: nr_cpu_ids; possible = min(possible, sclp_max); for (cpu = 0; cpu < possible && cpu < nr_cpu_ids; cpu++) diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index f7dba3887a54..050b8d067d3b 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -16,21 +16,11 @@ #include #include #include - -/* Sigh, math-emu. Don't ask. */ -#include -#include -#include +#include int topology_max_mnest; -/* - * stsi - store system information - * - * Returns the current configuration level if function code 0 was specified. - * Otherwise returns 0 on success or a negative value on error. - */ -int stsi(void *sysinfo, int fc, int sel1, int sel2) +static inline int __stsi(void *sysinfo, int fc, int sel1, int sel2, int *lvl) { register int r0 asm("0") = (fc << 28) | sel1; register int r1 asm("1") = sel2; @@ -45,9 +35,24 @@ int stsi(void *sysinfo, int fc, int sel1, int sel2) : "+d" (r0), "+d" (rc) : "d" (r1), "a" (sysinfo), "K" (-EOPNOTSUPP) : "cc", "memory"); + *lvl = ((unsigned int) r0) >> 28; + return rc; +} + +/* + * stsi - store system information + * + * Returns the current configuration level if function code 0 was specified. + * Otherwise returns 0 on success or a negative value on error. + */ +int stsi(void *sysinfo, int fc, int sel1, int sel2) +{ + int lvl, rc; + + rc = __stsi(sysinfo, fc, sel1, sel2, &lvl); if (rc) return rc; - return fc ? 0 : ((unsigned int) r0) >> 28; + return fc ? 0 : lvl; } EXPORT_SYMBOL(stsi); @@ -414,10 +419,8 @@ subsys_initcall(create_proc_service_level); void s390_adjust_jiffies(void) { struct sysinfo_1_2_2 *info; - const unsigned int fmil = 0x4b189680; /* 1e7 as 32-bit float. */ - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - unsigned int capability; + unsigned long capability; + struct kernel_fpu fpu; info = (void *) get_zeroed_page(GFP_KERNEL); if (!info) @@ -433,15 +436,25 @@ void s390_adjust_jiffies(void) * higher cpu capacity. Bogomips are the other way round. * To get to a halfway suitable number we divide 1e7 * by the cpu capability number. Yes, that means a floating - * point division .. math-emu here we come :-) + * point division .. */ - FP_UNPACK_SP(SA, &fmil); - if ((info->capability >> 23) == 0) - FP_FROM_INT_S(SB, (long) info->capability, 64, long); - else - FP_UNPACK_SP(SB, &info->capability); - FP_DIV_S(SR, SA, SB); - FP_TO_INT_S(capability, SR, 32, 0); + kernel_fpu_begin(&fpu, KERNEL_FPR); + asm volatile( + " sfpc %3\n" + " l %0,%1\n" + " tmlh %0,0xff80\n" + " jnz 0f\n" + " cefbr %%f2,%0\n" + " j 1f\n" + "0: le %%f2,%1\n" + "1: cefbr %%f0,%2\n" + " debr %%f0,%%f2\n" + " cgebr %0,5,%%f0\n" + : "=&d" (capability) + : "Q" (info->capability), "d" (10000000), "d" (0) + : "cc" + ); + kernel_fpu_end(&fpu); } else /* * Really old machine without stsi block for basic diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 9409d32f285e..4e9949800562 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -39,13 +39,14 @@ #include #include #include +#include #include #include #include #include #include #include -#include +#include #include #include "entry.h" @@ -61,6 +62,32 @@ static DEFINE_PER_CPU(struct clock_event_device, comparators); ATOMIC_NOTIFIER_HEAD(s390_epoch_delta_notifier); EXPORT_SYMBOL(s390_epoch_delta_notifier); +unsigned char ptff_function_mask[16]; +unsigned long lpar_offset; +unsigned long initial_leap_seconds; + +/* + * Get time offsets with PTFF + */ +void __init ptff_init(void) +{ + struct ptff_qto qto; + struct ptff_qui qui; + + if (!test_facility(28)) + return; + ptff(&ptff_function_mask, sizeof(ptff_function_mask), PTFF_QAF); + + /* get LPAR offset */ + if (ptff_query(PTFF_QTO) && ptff(&qto, sizeof(qto), PTFF_QTO) == 0) + lpar_offset = qto.tod_epoch_difference; + + /* get initial leap seconds */ + if (ptff_query(PTFF_QUI) && ptff(&qui, sizeof(qui), PTFF_QUI) == 0) + initial_leap_seconds = (unsigned long) + ((long) qui.old_leap * 4096000000L); +} + /* * Scheduler clock - returns current time in nanosec units. */ @@ -162,30 +189,32 @@ static void clock_comparator_interrupt(struct ext_code ext_code, set_clock_comparator(S390_lowcore.clock_comparator); } -static void etr_timing_alert(struct etr_irq_parm *); static void stp_timing_alert(struct stp_irq_parm *); static void timing_alert_interrupt(struct ext_code ext_code, unsigned int param32, unsigned long param64) { inc_irq_stat(IRQEXT_TLA); - if (param32 & 0x00c40000) - etr_timing_alert((struct etr_irq_parm *) ¶m32); if (param32 & 0x00038000) stp_timing_alert((struct stp_irq_parm *) ¶m32); } -static void etr_reset(void); static void stp_reset(void); void read_persistent_clock64(struct timespec64 *ts) { - tod_to_timeval(get_tod_clock() - TOD_UNIX_EPOCH, ts); + __u64 clock; + + clock = get_tod_clock() - initial_leap_seconds; + tod_to_timeval(clock - TOD_UNIX_EPOCH, ts); } void read_boot_clock64(struct timespec64 *ts) { - tod_to_timeval(sched_clock_base_cc - TOD_UNIX_EPOCH, ts); + __u64 clock; + + clock = sched_clock_base_cc - initial_leap_seconds; + tod_to_timeval(clock - TOD_UNIX_EPOCH, ts); } static cycle_t read_tod_clock(struct clocksource *cs) @@ -269,7 +298,6 @@ void update_vsyscall_tz(void) void __init time_init(void) { /* Reset time synchronization interfaces. */ - etr_reset(); stp_reset(); /* request the clock comparator external interrupt */ @@ -337,20 +365,20 @@ static unsigned long clock_sync_flags; #define CLOCK_SYNC_STP 3 /* - * The synchronous get_clock function. It will write the current clock - * value to the clock pointer and return 0 if the clock is in sync with - * the external time source. If the clock mode is local it will return - * -EOPNOTSUPP and -EAGAIN if the clock is not in sync with the external - * reference. + * The get_clock function for the physical clock. It will get the current + * TOD clock, subtract the LPAR offset and write the result to *clock. + * The function returns 0 if the clock is in sync with the external time + * source. If the clock mode is local it will return -EOPNOTSUPP and + * -EAGAIN if the clock is not in sync with the external reference. */ -int get_sync_clock(unsigned long long *clock) +int get_phys_clock(unsigned long long *clock) { atomic_t *sw_ptr; unsigned int sw0, sw1; sw_ptr = &get_cpu_var(clock_sync_word); sw0 = atomic_read(sw_ptr); - *clock = get_tod_clock(); + *clock = get_tod_clock() - lpar_offset; sw1 = atomic_read(sw_ptr); put_cpu_var(clock_sync_word); if (sw0 == sw1 && (sw0 & 0x80000000U)) @@ -364,7 +392,7 @@ int get_sync_clock(unsigned long long *clock) return -EACCES; return -EAGAIN; } -EXPORT_SYMBOL(get_sync_clock); +EXPORT_SYMBOL(get_phys_clock); /* * Make get_sync_clock return -EAGAIN. @@ -416,301 +444,6 @@ static void __init time_init_wq(void) time_sync_wq = create_singlethread_workqueue("timesync"); } -/* - * External Time Reference (ETR) code. - */ -static int etr_port0_online; -static int etr_port1_online; -static int etr_steai_available; - -static int __init early_parse_etr(char *p) -{ - if (strncmp(p, "off", 3) == 0) - etr_port0_online = etr_port1_online = 0; - else if (strncmp(p, "port0", 5) == 0) - etr_port0_online = 1; - else if (strncmp(p, "port1", 5) == 0) - etr_port1_online = 1; - else if (strncmp(p, "on", 2) == 0) - etr_port0_online = etr_port1_online = 1; - return 0; -} -early_param("etr", early_parse_etr); - -enum etr_event { - ETR_EVENT_PORT0_CHANGE, - ETR_EVENT_PORT1_CHANGE, - ETR_EVENT_PORT_ALERT, - ETR_EVENT_SYNC_CHECK, - ETR_EVENT_SWITCH_LOCAL, - ETR_EVENT_UPDATE, -}; - -/* - * Valid bit combinations of the eacr register are (x = don't care): - * e0 e1 dp p0 p1 ea es sl - * 0 0 x 0 0 0 0 0 initial, disabled state - * 0 0 x 0 1 1 0 0 port 1 online - * 0 0 x 1 0 1 0 0 port 0 online - * 0 0 x 1 1 1 0 0 both ports online - * 0 1 x 0 1 1 0 0 port 1 online and usable, ETR or PPS mode - * 0 1 x 0 1 1 0 1 port 1 online, usable and ETR mode - * 0 1 x 0 1 1 1 0 port 1 online, usable, PPS mode, in-sync - * 0 1 x 0 1 1 1 1 port 1 online, usable, ETR mode, in-sync - * 0 1 x 1 1 1 0 0 both ports online, port 1 usable - * 0 1 x 1 1 1 1 0 both ports online, port 1 usable, PPS mode, in-sync - * 0 1 x 1 1 1 1 1 both ports online, port 1 usable, ETR mode, in-sync - * 1 0 x 1 0 1 0 0 port 0 online and usable, ETR or PPS mode - * 1 0 x 1 0 1 0 1 port 0 online, usable and ETR mode - * 1 0 x 1 0 1 1 0 port 0 online, usable, PPS mode, in-sync - * 1 0 x 1 0 1 1 1 port 0 online, usable, ETR mode, in-sync - * 1 0 x 1 1 1 0 0 both ports online, port 0 usable - * 1 0 x 1 1 1 1 0 both ports online, port 0 usable, PPS mode, in-sync - * 1 0 x 1 1 1 1 1 both ports online, port 0 usable, ETR mode, in-sync - * 1 1 x 1 1 1 1 0 both ports online & usable, ETR, in-sync - * 1 1 x 1 1 1 1 1 both ports online & usable, ETR, in-sync - */ -static struct etr_eacr etr_eacr; -static u64 etr_tolec; /* time of last eacr update */ -static struct etr_aib etr_port0; -static int etr_port0_uptodate; -static struct etr_aib etr_port1; -static int etr_port1_uptodate; -static unsigned long etr_events; -static struct timer_list etr_timer; - -static void etr_timeout(unsigned long dummy); -static void etr_work_fn(struct work_struct *work); -static DEFINE_MUTEX(etr_work_mutex); -static DECLARE_WORK(etr_work, etr_work_fn); - -/* - * Reset ETR attachment. - */ -static void etr_reset(void) -{ - etr_eacr = (struct etr_eacr) { - .e0 = 0, .e1 = 0, ._pad0 = 4, .dp = 0, - .p0 = 0, .p1 = 0, ._pad1 = 0, .ea = 0, - .es = 0, .sl = 0 }; - if (etr_setr(&etr_eacr) == 0) { - etr_tolec = get_tod_clock(); - set_bit(CLOCK_SYNC_HAS_ETR, &clock_sync_flags); - if (etr_port0_online && etr_port1_online) - set_bit(CLOCK_SYNC_ETR, &clock_sync_flags); - } else if (etr_port0_online || etr_port1_online) { - pr_warn("The real or virtual hardware system does not provide an ETR interface\n"); - etr_port0_online = etr_port1_online = 0; - } -} - -static int __init etr_init(void) -{ - struct etr_aib aib; - - if (!test_bit(CLOCK_SYNC_HAS_ETR, &clock_sync_flags)) - return 0; - time_init_wq(); - /* Check if this machine has the steai instruction. */ - if (etr_steai(&aib, ETR_STEAI_STEPPING_PORT) == 0) - etr_steai_available = 1; - setup_timer(&etr_timer, etr_timeout, 0UL); - if (etr_port0_online) { - set_bit(ETR_EVENT_PORT0_CHANGE, &etr_events); - queue_work(time_sync_wq, &etr_work); - } - if (etr_port1_online) { - set_bit(ETR_EVENT_PORT1_CHANGE, &etr_events); - queue_work(time_sync_wq, &etr_work); - } - return 0; -} - -arch_initcall(etr_init); - -/* - * Two sorts of ETR machine checks. The architecture reads: - * "When a machine-check niterruption occurs and if a switch-to-local or - * ETR-sync-check interrupt request is pending but disabled, this pending - * disabled interruption request is indicated and is cleared". - * Which means that we can get etr_switch_to_local events from the machine - * check handler although the interruption condition is disabled. Lovely.. - */ - -/* - * Switch to local machine check. This is called when the last usable - * ETR port goes inactive. After switch to local the clock is not in sync. - */ -int etr_switch_to_local(void) -{ - if (!etr_eacr.sl) - return 0; - disable_sync_clock(NULL); - if (!test_and_set_bit(ETR_EVENT_SWITCH_LOCAL, &etr_events)) { - etr_eacr.es = etr_eacr.sl = 0; - etr_setr(&etr_eacr); - return 1; - } - return 0; -} - -/* - * ETR sync check machine check. This is called when the ETR OTE and the - * local clock OTE are farther apart than the ETR sync check tolerance. - * After a ETR sync check the clock is not in sync. The machine check - * is broadcasted to all cpus at the same time. - */ -int etr_sync_check(void) -{ - if (!etr_eacr.es) - return 0; - disable_sync_clock(NULL); - if (!test_and_set_bit(ETR_EVENT_SYNC_CHECK, &etr_events)) { - etr_eacr.es = 0; - etr_setr(&etr_eacr); - return 1; - } - return 0; -} - -void etr_queue_work(void) -{ - queue_work(time_sync_wq, &etr_work); -} - -/* - * ETR timing alert. There are two causes: - * 1) port state change, check the usability of the port - * 2) port alert, one of the ETR-data-validity bits (v1-v2 bits of the - * sldr-status word) or ETR-data word 1 (edf1) or ETR-data word 3 (edf3) - * or ETR-data word 4 (edf4) has changed. - */ -static void etr_timing_alert(struct etr_irq_parm *intparm) -{ - if (intparm->pc0) - /* ETR port 0 state change. */ - set_bit(ETR_EVENT_PORT0_CHANGE, &etr_events); - if (intparm->pc1) - /* ETR port 1 state change. */ - set_bit(ETR_EVENT_PORT1_CHANGE, &etr_events); - if (intparm->eai) - /* - * ETR port alert on either port 0, 1 or both. - * Both ports are not up-to-date now. - */ - set_bit(ETR_EVENT_PORT_ALERT, &etr_events); - queue_work(time_sync_wq, &etr_work); -} - -static void etr_timeout(unsigned long dummy) -{ - set_bit(ETR_EVENT_UPDATE, &etr_events); - queue_work(time_sync_wq, &etr_work); -} - -/* - * Check if the etr mode is pss. - */ -static inline int etr_mode_is_pps(struct etr_eacr eacr) -{ - return eacr.es && !eacr.sl; -} - -/* - * Check if the etr mode is etr. - */ -static inline int etr_mode_is_etr(struct etr_eacr eacr) -{ - return eacr.es && eacr.sl; -} - -/* - * Check if the port can be used for TOD synchronization. - * For PPS mode the port has to receive OTEs. For ETR mode - * the port has to receive OTEs, the ETR stepping bit has to - * be zero and the validity bits for data frame 1, 2, and 3 - * have to be 1. - */ -static int etr_port_valid(struct etr_aib *aib, int port) -{ - unsigned int psc; - - /* Check that this port is receiving OTEs. */ - if (aib->tsp == 0) - return 0; - - psc = port ? aib->esw.psc1 : aib->esw.psc0; - if (psc == etr_lpsc_pps_mode) - return 1; - if (psc == etr_lpsc_operational_step) - return !aib->esw.y && aib->slsw.v1 && - aib->slsw.v2 && aib->slsw.v3; - return 0; -} - -/* - * Check if two ports are on the same network. - */ -static int etr_compare_network(struct etr_aib *aib1, struct etr_aib *aib2) -{ - // FIXME: any other fields we have to compare? - return aib1->edf1.net_id == aib2->edf1.net_id; -} - -/* - * Wrapper for etr_stei that converts physical port states - * to logical port states to be consistent with the output - * of stetr (see etr_psc vs. etr_lpsc). - */ -static void etr_steai_cv(struct etr_aib *aib, unsigned int func) -{ - BUG_ON(etr_steai(aib, func) != 0); - /* Convert port state to logical port state. */ - if (aib->esw.psc0 == 1) - aib->esw.psc0 = 2; - else if (aib->esw.psc0 == 0 && aib->esw.p == 0) - aib->esw.psc0 = 1; - if (aib->esw.psc1 == 1) - aib->esw.psc1 = 2; - else if (aib->esw.psc1 == 0 && aib->esw.p == 1) - aib->esw.psc1 = 1; -} - -/* - * Check if the aib a2 is still connected to the same attachment as - * aib a1, the etv values differ by one and a2 is valid. - */ -static int etr_aib_follows(struct etr_aib *a1, struct etr_aib *a2, int p) -{ - int state_a1, state_a2; - - /* Paranoia check: e0/e1 should better be the same. */ - if (a1->esw.eacr.e0 != a2->esw.eacr.e0 || - a1->esw.eacr.e1 != a2->esw.eacr.e1) - return 0; - - /* Still connected to the same etr ? */ - state_a1 = p ? a1->esw.psc1 : a1->esw.psc0; - state_a2 = p ? a2->esw.psc1 : a2->esw.psc0; - if (state_a1 == etr_lpsc_operational_step) { - if (state_a2 != etr_lpsc_operational_step || - a1->edf1.net_id != a2->edf1.net_id || - a1->edf1.etr_id != a2->edf1.etr_id || - a1->edf1.etr_pn != a2->edf1.etr_pn) - return 0; - } else if (state_a2 != etr_lpsc_pps_mode) - return 0; - - /* The ETV value of a2 needs to be ETV of a1 + 1. */ - if (a1->edf2.etv + 1 != a2->edf2.etv) - return 0; - - if (!etr_port_valid(a2, p)) - return 0; - - return 1; -} - struct clock_sync_data { atomic_t cpus; int in_sync; @@ -747,688 +480,6 @@ static void clock_sync_cpu(struct clock_sync_data *sync) fixup_clock_comparator(sync->fixup_cc); } -/* - * Sync the TOD clock using the port referred to by aibp. This port - * has to be enabled and the other port has to be disabled. The - * last eacr update has to be more than 1.6 seconds in the past. - */ -static int etr_sync_clock(void *data) -{ - static int first; - unsigned long long clock, old_clock, clock_delta, delay, delta; - struct clock_sync_data *etr_sync; - struct etr_aib *sync_port, *aib; - int port; - int rc; - - etr_sync = data; - - if (xchg(&first, 1) == 1) { - /* Slave */ - clock_sync_cpu(etr_sync); - return 0; - } - - /* Wait until all other cpus entered the sync function. */ - while (atomic_read(&etr_sync->cpus) != 0) - cpu_relax(); - - port = etr_sync->etr_port; - aib = etr_sync->etr_aib; - sync_port = (port == 0) ? &etr_port0 : &etr_port1; - enable_sync_clock(); - - /* Set clock to next OTE. */ - __ctl_set_bit(14, 21); - __ctl_set_bit(0, 29); - clock = ((unsigned long long) (aib->edf2.etv + 1)) << 32; - old_clock = get_tod_clock(); - if (set_tod_clock(clock) == 0) { - __udelay(1); /* Wait for the clock to start. */ - __ctl_clear_bit(0, 29); - __ctl_clear_bit(14, 21); - etr_stetr(aib); - /* Adjust Linux timing variables. */ - delay = (unsigned long long) - (aib->edf2.etv - sync_port->edf2.etv) << 32; - delta = adjust_time(old_clock, clock, delay); - clock_delta = clock - old_clock; - atomic_notifier_call_chain(&s390_epoch_delta_notifier, 0, - &clock_delta); - etr_sync->fixup_cc = delta; - fixup_clock_comparator(delta); - /* Verify that the clock is properly set. */ - if (!etr_aib_follows(sync_port, aib, port)) { - /* Didn't work. */ - disable_sync_clock(NULL); - etr_sync->in_sync = -EAGAIN; - rc = -EAGAIN; - } else { - etr_sync->in_sync = 1; - rc = 0; - } - } else { - /* Could not set the clock ?!? */ - __ctl_clear_bit(0, 29); - __ctl_clear_bit(14, 21); - disable_sync_clock(NULL); - etr_sync->in_sync = -EAGAIN; - rc = -EAGAIN; - } - xchg(&first, 0); - return rc; -} - -static int etr_sync_clock_stop(struct etr_aib *aib, int port) -{ - struct clock_sync_data etr_sync; - struct etr_aib *sync_port; - int follows; - int rc; - - /* Check if the current aib is adjacent to the sync port aib. */ - sync_port = (port == 0) ? &etr_port0 : &etr_port1; - follows = etr_aib_follows(sync_port, aib, port); - memcpy(sync_port, aib, sizeof(*aib)); - if (!follows) - return -EAGAIN; - memset(&etr_sync, 0, sizeof(etr_sync)); - etr_sync.etr_aib = aib; - etr_sync.etr_port = port; - get_online_cpus(); - atomic_set(&etr_sync.cpus, num_online_cpus() - 1); - rc = stop_machine(etr_sync_clock, &etr_sync, cpu_online_mask); - put_online_cpus(); - return rc; -} - -/* - * Handle the immediate effects of the different events. - * The port change event is used for online/offline changes. - */ -static struct etr_eacr etr_handle_events(struct etr_eacr eacr) -{ - if (test_and_clear_bit(ETR_EVENT_SYNC_CHECK, &etr_events)) - eacr.es = 0; - if (test_and_clear_bit(ETR_EVENT_SWITCH_LOCAL, &etr_events)) - eacr.es = eacr.sl = 0; - if (test_and_clear_bit(ETR_EVENT_PORT_ALERT, &etr_events)) - etr_port0_uptodate = etr_port1_uptodate = 0; - - if (test_and_clear_bit(ETR_EVENT_PORT0_CHANGE, &etr_events)) { - if (eacr.e0) - /* - * Port change of an enabled port. We have to - * assume that this can have caused an stepping - * port switch. - */ - etr_tolec = get_tod_clock(); - eacr.p0 = etr_port0_online; - if (!eacr.p0) - eacr.e0 = 0; - etr_port0_uptodate = 0; - } - if (test_and_clear_bit(ETR_EVENT_PORT1_CHANGE, &etr_events)) { - if (eacr.e1) - /* - * Port change of an enabled port. We have to - * assume that this can have caused an stepping - * port switch. - */ - etr_tolec = get_tod_clock(); - eacr.p1 = etr_port1_online; - if (!eacr.p1) - eacr.e1 = 0; - etr_port1_uptodate = 0; - } - clear_bit(ETR_EVENT_UPDATE, &etr_events); - return eacr; -} - -/* - * Set up a timer that expires after the etr_tolec + 1.6 seconds if - * one of the ports needs an update. - */ -static void etr_set_tolec_timeout(unsigned long long now) -{ - unsigned long micros; - - if ((!etr_eacr.p0 || etr_port0_uptodate) && - (!etr_eacr.p1 || etr_port1_uptodate)) - return; - micros = (now > etr_tolec) ? ((now - etr_tolec) >> 12) : 0; - micros = (micros > 1600000) ? 0 : 1600000 - micros; - mod_timer(&etr_timer, jiffies + (micros * HZ) / 1000000 + 1); -} - -/* - * Set up a time that expires after 1/2 second. - */ -static void etr_set_sync_timeout(void) -{ - mod_timer(&etr_timer, jiffies + HZ/2); -} - -/* - * Update the aib information for one or both ports. - */ -static struct etr_eacr etr_handle_update(struct etr_aib *aib, - struct etr_eacr eacr) -{ - /* With both ports disabled the aib information is useless. */ - if (!eacr.e0 && !eacr.e1) - return eacr; - - /* Update port0 or port1 with aib stored in etr_work_fn. */ - if (aib->esw.q == 0) { - /* Information for port 0 stored. */ - if (eacr.p0 && !etr_port0_uptodate) { - etr_port0 = *aib; - if (etr_port0_online) - etr_port0_uptodate = 1; - } - } else { - /* Information for port 1 stored. */ - if (eacr.p1 && !etr_port1_uptodate) { - etr_port1 = *aib; - if (etr_port0_online) - etr_port1_uptodate = 1; - } - } - - /* - * Do not try to get the alternate port aib if the clock - * is not in sync yet. - */ - if (!eacr.es || !check_sync_clock()) - return eacr; - - /* - * If steai is available we can get the information about - * the other port immediately. If only stetr is available the - * data-port bit toggle has to be used. - */ - if (etr_steai_available) { - if (eacr.p0 && !etr_port0_uptodate) { - etr_steai_cv(&etr_port0, ETR_STEAI_PORT_0); - etr_port0_uptodate = 1; - } - if (eacr.p1 && !etr_port1_uptodate) { - etr_steai_cv(&etr_port1, ETR_STEAI_PORT_1); - etr_port1_uptodate = 1; - } - } else { - /* - * One port was updated above, if the other - * port is not uptodate toggle dp bit. - */ - if ((eacr.p0 && !etr_port0_uptodate) || - (eacr.p1 && !etr_port1_uptodate)) - eacr.dp ^= 1; - else - eacr.dp = 0; - } - return eacr; -} - -/* - * Write new etr control register if it differs from the current one. - * Return 1 if etr_tolec has been updated as well. - */ -static void etr_update_eacr(struct etr_eacr eacr) -{ - int dp_changed; - - if (memcmp(&etr_eacr, &eacr, sizeof(eacr)) == 0) - /* No change, return. */ - return; - /* - * The disable of an active port of the change of the data port - * bit can/will cause a change in the data port. - */ - dp_changed = etr_eacr.e0 > eacr.e0 || etr_eacr.e1 > eacr.e1 || - (etr_eacr.dp ^ eacr.dp) != 0; - etr_eacr = eacr; - etr_setr(&etr_eacr); - if (dp_changed) - etr_tolec = get_tod_clock(); -} - -/* - * ETR work. In this function you'll find the main logic. In - * particular this is the only function that calls etr_update_eacr(), - * it "controls" the etr control register. - */ -static void etr_work_fn(struct work_struct *work) -{ - unsigned long long now; - struct etr_eacr eacr; - struct etr_aib aib; - int sync_port; - - /* prevent multiple execution. */ - mutex_lock(&etr_work_mutex); - - /* Create working copy of etr_eacr. */ - eacr = etr_eacr; - - /* Check for the different events and their immediate effects. */ - eacr = etr_handle_events(eacr); - - /* Check if ETR is supposed to be active. */ - eacr.ea = eacr.p0 || eacr.p1; - if (!eacr.ea) { - /* Both ports offline. Reset everything. */ - eacr.dp = eacr.es = eacr.sl = 0; - on_each_cpu(disable_sync_clock, NULL, 1); - del_timer_sync(&etr_timer); - etr_update_eacr(eacr); - goto out_unlock; - } - - /* Store aib to get the current ETR status word. */ - BUG_ON(etr_stetr(&aib) != 0); - etr_port0.esw = etr_port1.esw = aib.esw; /* Copy status word. */ - now = get_tod_clock(); - - /* - * Update the port information if the last stepping port change - * or data port change is older than 1.6 seconds. - */ - if (now >= etr_tolec + (1600000 << 12)) - eacr = etr_handle_update(&aib, eacr); - - /* - * Select ports to enable. The preferred synchronization mode is PPS. - * If a port can be enabled depends on a number of things: - * 1) The port needs to be online and uptodate. A port is not - * disabled just because it is not uptodate, but it is only - * enabled if it is uptodate. - * 2) The port needs to have the same mode (pps / etr). - * 3) The port needs to be usable -> etr_port_valid() == 1 - * 4) To enable the second port the clock needs to be in sync. - * 5) If both ports are useable and are ETR ports, the network id - * has to be the same. - * The eacr.sl bit is used to indicate etr mode vs. pps mode. - */ - if (eacr.p0 && aib.esw.psc0 == etr_lpsc_pps_mode) { - eacr.sl = 0; - eacr.e0 = 1; - if (!etr_mode_is_pps(etr_eacr)) - eacr.es = 0; - if (!eacr.es || !eacr.p1 || aib.esw.psc1 != etr_lpsc_pps_mode) - eacr.e1 = 0; - // FIXME: uptodate checks ? - else if (etr_port0_uptodate && etr_port1_uptodate) - eacr.e1 = 1; - sync_port = (etr_port0_uptodate && - etr_port_valid(&etr_port0, 0)) ? 0 : -1; - } else if (eacr.p1 && aib.esw.psc1 == etr_lpsc_pps_mode) { - eacr.sl = 0; - eacr.e0 = 0; - eacr.e1 = 1; - if (!etr_mode_is_pps(etr_eacr)) - eacr.es = 0; - sync_port = (etr_port1_uptodate && - etr_port_valid(&etr_port1, 1)) ? 1 : -1; - } else if (eacr.p0 && aib.esw.psc0 == etr_lpsc_operational_step) { - eacr.sl = 1; - eacr.e0 = 1; - if (!etr_mode_is_etr(etr_eacr)) - eacr.es = 0; - if (!eacr.es || !eacr.p1 || - aib.esw.psc1 != etr_lpsc_operational_alt) - eacr.e1 = 0; - else if (etr_port0_uptodate && etr_port1_uptodate && - etr_compare_network(&etr_port0, &etr_port1)) - eacr.e1 = 1; - sync_port = (etr_port0_uptodate && - etr_port_valid(&etr_port0, 0)) ? 0 : -1; - } else if (eacr.p1 && aib.esw.psc1 == etr_lpsc_operational_step) { - eacr.sl = 1; - eacr.e0 = 0; - eacr.e1 = 1; - if (!etr_mode_is_etr(etr_eacr)) - eacr.es = 0; - sync_port = (etr_port1_uptodate && - etr_port_valid(&etr_port1, 1)) ? 1 : -1; - } else { - /* Both ports not usable. */ - eacr.es = eacr.sl = 0; - sync_port = -1; - } - - /* - * If the clock is in sync just update the eacr and return. - * If there is no valid sync port wait for a port update. - */ - if ((eacr.es && check_sync_clock()) || sync_port < 0) { - etr_update_eacr(eacr); - etr_set_tolec_timeout(now); - goto out_unlock; - } - - /* - * Prepare control register for clock syncing - * (reset data port bit, set sync check control. - */ - eacr.dp = 0; - eacr.es = 1; - - /* - * Update eacr and try to synchronize the clock. If the update - * of eacr caused a stepping port switch (or if we have to - * assume that a stepping port switch has occurred) or the - * clock syncing failed, reset the sync check control bit - * and set up a timer to try again after 0.5 seconds - */ - etr_update_eacr(eacr); - if (now < etr_tolec + (1600000 << 12) || - etr_sync_clock_stop(&aib, sync_port) != 0) { - /* Sync failed. Try again in 1/2 second. */ - eacr.es = 0; - etr_update_eacr(eacr); - etr_set_sync_timeout(); - } else - etr_set_tolec_timeout(now); -out_unlock: - mutex_unlock(&etr_work_mutex); -} - -/* - * Sysfs interface functions - */ -static struct bus_type etr_subsys = { - .name = "etr", - .dev_name = "etr", -}; - -static struct device etr_port0_dev = { - .id = 0, - .bus = &etr_subsys, -}; - -static struct device etr_port1_dev = { - .id = 1, - .bus = &etr_subsys, -}; - -/* - * ETR subsys attributes - */ -static ssize_t etr_stepping_port_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - return sprintf(buf, "%i\n", etr_port0.esw.p); -} - -static DEVICE_ATTR(stepping_port, 0400, etr_stepping_port_show, NULL); - -static ssize_t etr_stepping_mode_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - char *mode_str; - - if (etr_mode_is_pps(etr_eacr)) - mode_str = "pps"; - else if (etr_mode_is_etr(etr_eacr)) - mode_str = "etr"; - else - mode_str = "local"; - return sprintf(buf, "%s\n", mode_str); -} - -static DEVICE_ATTR(stepping_mode, 0400, etr_stepping_mode_show, NULL); - -/* - * ETR port attributes - */ -static inline struct etr_aib *etr_aib_from_dev(struct device *dev) -{ - if (dev == &etr_port0_dev) - return etr_port0_online ? &etr_port0 : NULL; - else - return etr_port1_online ? &etr_port1 : NULL; -} - -static ssize_t etr_online_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - unsigned int online; - - online = (dev == &etr_port0_dev) ? etr_port0_online : etr_port1_online; - return sprintf(buf, "%i\n", online); -} - -static ssize_t etr_online_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - unsigned int value; - - value = simple_strtoul(buf, NULL, 0); - if (value != 0 && value != 1) - return -EINVAL; - if (!test_bit(CLOCK_SYNC_HAS_ETR, &clock_sync_flags)) - return -EOPNOTSUPP; - mutex_lock(&clock_sync_mutex); - if (dev == &etr_port0_dev) { - if (etr_port0_online == value) - goto out; /* Nothing to do. */ - etr_port0_online = value; - if (etr_port0_online && etr_port1_online) - set_bit(CLOCK_SYNC_ETR, &clock_sync_flags); - else - clear_bit(CLOCK_SYNC_ETR, &clock_sync_flags); - set_bit(ETR_EVENT_PORT0_CHANGE, &etr_events); - queue_work(time_sync_wq, &etr_work); - } else { - if (etr_port1_online == value) - goto out; /* Nothing to do. */ - etr_port1_online = value; - if (etr_port0_online && etr_port1_online) - set_bit(CLOCK_SYNC_ETR, &clock_sync_flags); - else - clear_bit(CLOCK_SYNC_ETR, &clock_sync_flags); - set_bit(ETR_EVENT_PORT1_CHANGE, &etr_events); - queue_work(time_sync_wq, &etr_work); - } -out: - mutex_unlock(&clock_sync_mutex); - return count; -} - -static DEVICE_ATTR(online, 0600, etr_online_show, etr_online_store); - -static ssize_t etr_stepping_control_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - return sprintf(buf, "%i\n", (dev == &etr_port0_dev) ? - etr_eacr.e0 : etr_eacr.e1); -} - -static DEVICE_ATTR(stepping_control, 0400, etr_stepping_control_show, NULL); - -static ssize_t etr_mode_code_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - if (!etr_port0_online && !etr_port1_online) - /* Status word is not uptodate if both ports are offline. */ - return -ENODATA; - return sprintf(buf, "%i\n", (dev == &etr_port0_dev) ? - etr_port0.esw.psc0 : etr_port0.esw.psc1); -} - -static DEVICE_ATTR(state_code, 0400, etr_mode_code_show, NULL); - -static ssize_t etr_untuned_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct etr_aib *aib = etr_aib_from_dev(dev); - - if (!aib || !aib->slsw.v1) - return -ENODATA; - return sprintf(buf, "%i\n", aib->edf1.u); -} - -static DEVICE_ATTR(untuned, 0400, etr_untuned_show, NULL); - -static ssize_t etr_network_id_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct etr_aib *aib = etr_aib_from_dev(dev); - - if (!aib || !aib->slsw.v1) - return -ENODATA; - return sprintf(buf, "%i\n", aib->edf1.net_id); -} - -static DEVICE_ATTR(network, 0400, etr_network_id_show, NULL); - -static ssize_t etr_id_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct etr_aib *aib = etr_aib_from_dev(dev); - - if (!aib || !aib->slsw.v1) - return -ENODATA; - return sprintf(buf, "%i\n", aib->edf1.etr_id); -} - -static DEVICE_ATTR(id, 0400, etr_id_show, NULL); - -static ssize_t etr_port_number_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct etr_aib *aib = etr_aib_from_dev(dev); - - if (!aib || !aib->slsw.v1) - return -ENODATA; - return sprintf(buf, "%i\n", aib->edf1.etr_pn); -} - -static DEVICE_ATTR(port, 0400, etr_port_number_show, NULL); - -static ssize_t etr_coupled_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct etr_aib *aib = etr_aib_from_dev(dev); - - if (!aib || !aib->slsw.v3) - return -ENODATA; - return sprintf(buf, "%i\n", aib->edf3.c); -} - -static DEVICE_ATTR(coupled, 0400, etr_coupled_show, NULL); - -static ssize_t etr_local_time_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct etr_aib *aib = etr_aib_from_dev(dev); - - if (!aib || !aib->slsw.v3) - return -ENODATA; - return sprintf(buf, "%i\n", aib->edf3.blto); -} - -static DEVICE_ATTR(local_time, 0400, etr_local_time_show, NULL); - -static ssize_t etr_utc_offset_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct etr_aib *aib = etr_aib_from_dev(dev); - - if (!aib || !aib->slsw.v3) - return -ENODATA; - return sprintf(buf, "%i\n", aib->edf3.buo); -} - -static DEVICE_ATTR(utc_offset, 0400, etr_utc_offset_show, NULL); - -static struct device_attribute *etr_port_attributes[] = { - &dev_attr_online, - &dev_attr_stepping_control, - &dev_attr_state_code, - &dev_attr_untuned, - &dev_attr_network, - &dev_attr_id, - &dev_attr_port, - &dev_attr_coupled, - &dev_attr_local_time, - &dev_attr_utc_offset, - NULL -}; - -static int __init etr_register_port(struct device *dev) -{ - struct device_attribute **attr; - int rc; - - rc = device_register(dev); - if (rc) - goto out; - for (attr = etr_port_attributes; *attr; attr++) { - rc = device_create_file(dev, *attr); - if (rc) - goto out_unreg; - } - return 0; -out_unreg: - for (; attr >= etr_port_attributes; attr--) - device_remove_file(dev, *attr); - device_unregister(dev); -out: - return rc; -} - -static void __init etr_unregister_port(struct device *dev) -{ - struct device_attribute **attr; - - for (attr = etr_port_attributes; *attr; attr++) - device_remove_file(dev, *attr); - device_unregister(dev); -} - -static int __init etr_init_sysfs(void) -{ - int rc; - - rc = subsys_system_register(&etr_subsys, NULL); - if (rc) - goto out; - rc = device_create_file(etr_subsys.dev_root, &dev_attr_stepping_port); - if (rc) - goto out_unreg_subsys; - rc = device_create_file(etr_subsys.dev_root, &dev_attr_stepping_mode); - if (rc) - goto out_remove_stepping_port; - rc = etr_register_port(&etr_port0_dev); - if (rc) - goto out_remove_stepping_mode; - rc = etr_register_port(&etr_port1_dev); - if (rc) - goto out_remove_port0; - return 0; - -out_remove_port0: - etr_unregister_port(&etr_port0_dev); -out_remove_stepping_mode: - device_remove_file(etr_subsys.dev_root, &dev_attr_stepping_mode); -out_remove_stepping_port: - device_remove_file(etr_subsys.dev_root, &dev_attr_stepping_port); -out_unreg_subsys: - bus_unregister(&etr_subsys); -out: - return rc; -} - -device_initcall(etr_init_sysfs); - /* * Server Time Protocol (STP) code. */ @@ -1455,7 +506,7 @@ static void __init stp_reset(void) int rc; stp_page = (void *) get_zeroed_page(GFP_ATOMIC); - rc = chsc_sstpc(stp_page, STP_OP_CTRL, 0x0000); + rc = chsc_sstpc(stp_page, STP_OP_CTRL, 0x0000, NULL); if (rc == 0) set_bit(CLOCK_SYNC_HAS_STP, &clock_sync_flags); else if (stp_online) { @@ -1533,6 +584,7 @@ static int stp_sync_clock(void *data) static int first; unsigned long long old_clock, delta, new_clock, clock_delta; struct clock_sync_data *stp_sync; + struct ptff_qto qto; int rc; stp_sync = data; @@ -1554,11 +606,14 @@ static int stp_sync_clock(void *data) stp_info.todoff[2] || stp_info.todoff[3] || stp_info.tmd != 2) { old_clock = get_tod_clock(); - rc = chsc_sstpc(stp_page, STP_OP_SYNC, 0); + rc = chsc_sstpc(stp_page, STP_OP_SYNC, 0, &clock_delta); if (rc == 0) { - new_clock = get_tod_clock(); + new_clock = old_clock + clock_delta; delta = adjust_time(old_clock, new_clock, 0); - clock_delta = new_clock - old_clock; + if (ptff_query(PTFF_QTO) && + ptff(&qto, sizeof(qto), PTFF_QTO) == 0) + /* Update LPAR offset */ + lpar_offset = qto.tod_epoch_difference; atomic_notifier_call_chain(&s390_epoch_delta_notifier, 0, &clock_delta); fixup_clock_comparator(delta); @@ -1590,12 +645,12 @@ static void stp_work_fn(struct work_struct *work) mutex_lock(&stp_work_mutex); if (!stp_online) { - chsc_sstpc(stp_page, STP_OP_CTRL, 0x0000); + chsc_sstpc(stp_page, STP_OP_CTRL, 0x0000, NULL); del_timer_sync(&stp_timer); goto out_unlock; } - rc = chsc_sstpc(stp_page, STP_OP_CTRL, 0xb0e0); + rc = chsc_sstpc(stp_page, STP_OP_CTRL, 0xb0e0, NULL); if (rc) goto out_unlock; diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 64298a867589..e959c02e0cac 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -46,6 +46,7 @@ static DECLARE_WORK(topology_work, topology_work_fn); */ static struct mask_info socket_info; static struct mask_info book_info; +static struct mask_info drawer_info; DEFINE_PER_CPU(struct cpu_topology_s390, cpu_topology); EXPORT_PER_CPU_SYMBOL_GPL(cpu_topology); @@ -79,10 +80,10 @@ static cpumask_t cpu_thread_map(unsigned int cpu) return mask; } -static struct mask_info *add_cpus_to_mask(struct topology_core *tl_core, - struct mask_info *book, - struct mask_info *socket, - int one_socket_per_cpu) +static void add_cpus_to_mask(struct topology_core *tl_core, + struct mask_info *drawer, + struct mask_info *book, + struct mask_info *socket) { struct cpu_topology_s390 *topo; unsigned int core; @@ -97,21 +98,17 @@ static struct mask_info *add_cpus_to_mask(struct topology_core *tl_core, continue; for (i = 0; i <= smp_cpu_mtid; i++) { topo = &per_cpu(cpu_topology, lcpu + i); + topo->drawer_id = drawer->id; topo->book_id = book->id; + topo->socket_id = socket->id; topo->core_id = rcore; topo->thread_id = lcpu + i; + cpumask_set_cpu(lcpu + i, &drawer->mask); cpumask_set_cpu(lcpu + i, &book->mask); cpumask_set_cpu(lcpu + i, &socket->mask); - if (one_socket_per_cpu) - topo->socket_id = rcore; - else - topo->socket_id = socket->id; smp_cpu_set_polarization(lcpu + i, tl_core->pp); } - if (one_socket_per_cpu) - socket = socket->next; } - return socket; } static void clear_masks(void) @@ -128,6 +125,11 @@ static void clear_masks(void) cpumask_clear(&info->mask); info = info->next; } + info = &drawer_info; + while (info) { + cpumask_clear(&info->mask); + info = info->next; + } } static union topology_entry *next_tle(union topology_entry *tle) @@ -137,16 +139,22 @@ static union topology_entry *next_tle(union topology_entry *tle) return (union topology_entry *)((struct topology_container *)tle + 1); } -static void __tl_to_masks_generic(struct sysinfo_15_1_x *info) +static void tl_to_masks(struct sysinfo_15_1_x *info) { struct mask_info *socket = &socket_info; struct mask_info *book = &book_info; + struct mask_info *drawer = &drawer_info; union topology_entry *tle, *end; + clear_masks(); tle = info->tle; end = (union topology_entry *)((unsigned long)info + info->length); while (tle < end) { switch (tle->nl) { + case 3: + drawer = drawer->next; + drawer->id = tle->container.id; + break; case 2: book = book->next; book->id = tle->container.id; @@ -156,32 +164,7 @@ static void __tl_to_masks_generic(struct sysinfo_15_1_x *info) socket->id = tle->container.id; break; case 0: - add_cpus_to_mask(&tle->cpu, book, socket, 0); - break; - default: - clear_masks(); - return; - } - tle = next_tle(tle); - } -} - -static void __tl_to_masks_z10(struct sysinfo_15_1_x *info) -{ - struct mask_info *socket = &socket_info; - struct mask_info *book = &book_info; - union topology_entry *tle, *end; - - tle = info->tle; - end = (union topology_entry *)((unsigned long)info + info->length); - while (tle < end) { - switch (tle->nl) { - case 1: - book = book->next; - book->id = tle->container.id; - break; - case 0: - socket = add_cpus_to_mask(&tle->cpu, book, socket, 1); + add_cpus_to_mask(&tle->cpu, drawer, book, socket); break; default: clear_masks(); @@ -191,22 +174,6 @@ static void __tl_to_masks_z10(struct sysinfo_15_1_x *info) } } -static void tl_to_masks(struct sysinfo_15_1_x *info) -{ - struct cpuid cpu_id; - - get_cpu_id(&cpu_id); - clear_masks(); - switch (cpu_id.machine) { - case 0x2097: - case 0x2098: - __tl_to_masks_z10(info); - break; - default: - __tl_to_masks_generic(info); - } -} - static void topology_update_polarization_simple(void) { int cpu; @@ -257,11 +224,13 @@ static void update_cpu_masks(void) topo->thread_mask = cpu_thread_map(cpu); topo->core_mask = cpu_group_map(&socket_info, cpu); topo->book_mask = cpu_group_map(&book_info, cpu); + topo->drawer_mask = cpu_group_map(&drawer_info, cpu); if (!MACHINE_HAS_TOPOLOGY) { topo->thread_id = cpu; topo->core_id = cpu; topo->socket_id = cpu; topo->book_id = cpu; + topo->drawer_id = cpu; } } numa_update_cpu_topology(); @@ -269,10 +238,7 @@ static void update_cpu_masks(void) void store_topology(struct sysinfo_15_1_x *info) { - if (topology_max_mnest >= 3) - stsi(info, 15, 1, 3); - else - stsi(info, 15, 1, 2); + stsi(info, 15, 1, min(topology_max_mnest, 4)); } int arch_update_cpu_topology(void) @@ -442,6 +408,11 @@ static const struct cpumask *cpu_book_mask(int cpu) return &per_cpu(cpu_topology, cpu).book_mask; } +static const struct cpumask *cpu_drawer_mask(int cpu) +{ + return &per_cpu(cpu_topology, cpu).drawer_mask; +} + static int __init early_parse_topology(char *p) { return kstrtobool(p, &topology_enabled); @@ -452,6 +423,7 @@ static struct sched_domain_topology_level s390_topology[] = { { cpu_thread_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, { cpu_book_mask, SD_INIT_NAME(BOOK) }, + { cpu_drawer_mask, SD_INIT_NAME(DRAWER) }, { cpu_cpu_mask, SD_INIT_NAME(DIE) }, { NULL, }, }; @@ -487,6 +459,7 @@ static int __init s390_topology_init(void) printk(KERN_CONT " / %d\n", info->mnest); alloc_masks(info, &socket_info, 1); alloc_masks(info, &book_info, 2); + alloc_masks(info, &drawer_info, 3); set_sched_topology(s390_topology); return 0; } diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index f9c459586649..68145456fee2 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -1,5 +1,7 @@ # List of files in the vdso, has to be asm only for now +KCOV_INSTRUMENT := n + obj-vdso32 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o # Build rules diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index 058659c1b8cf..0b0fd22c869a 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -1,5 +1,7 @@ # List of files in the vdso, has to be asm only for now +KCOV_INSTRUMENT := n + obj-vdso64 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o # Build rules diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 0f41a8286378..429bfd111961 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -4,6 +4,16 @@ #include #include + +/* + * Put .bss..swapper_pg_dir as the first thing in .bss. This will + * make sure it has 16k alignment. + */ +#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) + +/* Handle ro_after_init data on our own. */ +#define RO_AFTER_INIT_DATA + #include OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") @@ -49,7 +59,14 @@ SECTIONS _eshared = .; /* End of shareable data */ _sdata = .; /* Start of data section */ - EXCEPTION_TABLE(16) :data + . = ALIGN(PAGE_SIZE); + __start_ro_after_init = .; + .data..ro_after_init : { + *(.data..ro_after_init) + } + EXCEPTION_TABLE(16) + . = ALIGN(PAGE_SIZE); + __end_ro_after_init = .; RW_DATA_SECTION(0x100, PAGE_SIZE, THREAD_SIZE) @@ -81,7 +98,7 @@ SECTIONS . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */ - BSS_SECTION(0, 2, 0) + BSS_SECTION(PAGE_SIZE, 4 * PAGE_SIZE, PAGE_SIZE) _end = . ; diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 43f2a2b80490..6f5c344cd785 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/lib/string.c b/arch/s390/lib/string.c index b647d5ff0ad9..e390bbb16443 100644 --- a/arch/s390/lib/string.c +++ b/arch/s390/lib/string.c @@ -236,6 +236,26 @@ char * strrchr(const char * s, int c) } EXPORT_SYMBOL(strrchr); +static inline int clcle(const char *s1, unsigned long l1, + const char *s2, unsigned long l2, + int *diff) +{ + register unsigned long r2 asm("2") = (unsigned long) s1; + register unsigned long r3 asm("3") = (unsigned long) l2; + register unsigned long r4 asm("4") = (unsigned long) s2; + register unsigned long r5 asm("5") = (unsigned long) l2; + int cc; + + asm volatile ("0: clcle %1,%3,0\n" + " jo 0b\n" + " ipm %0\n" + " srl %0,28" + : "=&d" (cc), "+a" (r2), "+a" (r3), + "+a" (r4), "+a" (r5) : : "cc"); + *diff = *(char *)r2 - *(char *)r4; + return cc; +} + /** * strstr - Find the first substring in a %NUL terminated string * @s1: The string to be searched @@ -250,18 +270,9 @@ char * strstr(const char * s1,const char * s2) return (char *) s1; l1 = __strend(s1) - s1; while (l1-- >= l2) { - register unsigned long r2 asm("2") = (unsigned long) s1; - register unsigned long r3 asm("3") = (unsigned long) l2; - register unsigned long r4 asm("4") = (unsigned long) s2; - register unsigned long r5 asm("5") = (unsigned long) l2; - int cc; - - asm volatile ("0: clcle %1,%3,0\n" - " jo 0b\n" - " ipm %0\n" - " srl %0,28" - : "=&d" (cc), "+a" (r2), "+a" (r3), - "+a" (r4), "+a" (r5) : : "cc" ); + int cc, dummy; + + cc = clcle(s1, l1, s2, l2, &dummy); if (!cc) return (char *) s1; s1++; @@ -302,20 +313,11 @@ EXPORT_SYMBOL(memchr); */ int memcmp(const void *cs, const void *ct, size_t n) { - register unsigned long r2 asm("2") = (unsigned long) cs; - register unsigned long r3 asm("3") = (unsigned long) n; - register unsigned long r4 asm("4") = (unsigned long) ct; - register unsigned long r5 asm("5") = (unsigned long) n; - int ret; + int ret, diff; - asm volatile ("0: clcle %1,%3,0\n" - " jo 0b\n" - " ipm %0\n" - " srl %0,28" - : "=&d" (ret), "+a" (r2), "+a" (r3), "+a" (r4), "+a" (r5) - : : "cc" ); + ret = clcle(cs, n, ct, n, &diff); if (ret) - ret = *(char *) r2 - *(char *) r4; + ret = diff; return ret; } EXPORT_SYMBOL(memcmp); diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index ae4de559e3a0..d96596128e9f 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -49,7 +49,7 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr " jnm 5b\n" " ex %4,0(%3)\n" " j 8f\n" - "7:slgr %0,%0\n" + "7: slgr %0,%0\n" "8:\n" EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) @@ -93,7 +93,7 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, " jnm 6b\n" " ex %4,0(%3)\n" " j 9f\n" - "8:slgr %0,%0\n" + "8: slgr %0,%0\n" "9: sacf 768\n" EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,5b) EX_TABLE(10b,3b) EX_TABLE(11b,3b) EX_TABLE(12b,5b) @@ -266,7 +266,7 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n" " slgr %0,%3\n" " j 5f\n" - "4:slgr %0,%0\n" + "4: slgr %0,%0\n" "5:\n" EX_TABLE(0b,2b) EX_TABLE(3b,5b) : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2) diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 8556d6be9b54..861880df12c7 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -157,7 +157,7 @@ static void walk_pud_level(struct seq_file *m, struct pg_state *st, pud = pud_offset(pgd, addr); if (!pud_none(*pud)) if (pud_large(*pud)) { - prot = pud_val(*pud) & _REGION3_ENTRY_RO; + prot = pud_val(*pud) & _REGION_ENTRY_PROTECT; note_page(m, st, prot, 2); } else walk_pmd_level(m, st, pud, addr); diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 19288c1b36d3..25783dc3c813 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -456,7 +456,7 @@ retry: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); /* No reason to continue if interrupted by SIGKILL. */ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { fault = VM_FAULT_SIGNAL; @@ -624,7 +624,7 @@ void pfault_fini(void) diag_stat_inc(DIAG_STAT_X258); asm volatile( " diag %0,0,0x258\n" - "0:\n" + "0: nopr %%r7\n" EX_TABLE(0b,0b) : : "a" (&refbk), "m" (refbk) : "cc"); } diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index cace818d86eb..063c721ec0dc 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -85,7 +85,7 @@ EXPORT_SYMBOL_GPL(gmap_alloc); static void gmap_flush_tlb(struct gmap *gmap) { if (MACHINE_HAS_IDTE) - __tlb_flush_asce(gmap->mm, gmap->asce); + __tlb_flush_idte(gmap->asce); else __tlb_flush_global(); } @@ -124,7 +124,7 @@ void gmap_free(struct gmap *gmap) /* Flush tlb. */ if (MACHINE_HAS_IDTE) - __tlb_flush_asce(gmap->mm, gmap->asce); + __tlb_flush_idte(gmap->asce); else __tlb_flush_global(); @@ -430,6 +430,9 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) VM_BUG_ON(pgd_none(*pgd)); pud = pud_offset(pgd, vmaddr); VM_BUG_ON(pud_none(*pud)); + /* large puds cannot yet be handled */ + if (pud_large(*pud)) + return -EFAULT; pmd = pmd_offset(pud, vmaddr); VM_BUG_ON(pmd_none(*pmd)); /* large pmds cannot yet be handled */ diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c index a8a6765f1a51..adb0c34bf431 100644 --- a/arch/s390/mm/gup.c +++ b/arch/s390/mm/gup.c @@ -128,6 +128,44 @@ static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, return 1; } +static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr, + unsigned long end, int write, struct page **pages, int *nr) +{ + struct page *head, *page; + unsigned long mask; + int refs; + + mask = (write ? _REGION_ENTRY_PROTECT : 0) | _REGION_ENTRY_INVALID; + if ((pud_val(pud) & mask) != 0) + return 0; + VM_BUG_ON(!pfn_valid(pud_pfn(pud))); + + refs = 0; + head = pud_page(pud); + page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT); + do { + VM_BUG_ON_PAGE(compound_head(page) != head, page); + pages[*nr] = page; + (*nr)++; + page++; + refs++; + } while (addr += PAGE_SIZE, addr != end); + + if (!page_cache_add_speculative(head, refs)) { + *nr -= refs; + return 0; + } + + if (unlikely(pud_val(pud) != pud_val(*pudp))) { + *nr -= refs; + while (refs--) + put_page(head); + return 0; + } + + return 1; +} + static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end, int write, struct page **pages, int *nr) { @@ -144,7 +182,12 @@ static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, next = pud_addr_end(addr, end); if (pud_none(pud)) return 0; - if (!gup_pmd_range(pudp, pud, addr, next, write, pages, nr)) + if (unlikely(pud_large(pud))) { + if (!gup_huge_pud(pudp, pud, addr, next, write, pages, + nr)) + return 0; + } else if (!gup_pmd_range(pudp, pud, addr, next, write, pages, + nr)) return 0; } while (pudp++, addr = next, addr != end); diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index 1b5e8983f4f3..e19d853883be 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -1,19 +1,22 @@ /* * IBM System z Huge TLB Page Support for Kernel. * - * Copyright IBM Corp. 2007 + * Copyright IBM Corp. 2007,2016 * Author(s): Gerald Schaefer */ +#define KMSG_COMPONENT "hugetlb" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + #include #include -static inline pmd_t __pte_to_pmd(pte_t pte) +static inline unsigned long __pte_to_rste(pte_t pte) { - pmd_t pmd; + unsigned long rste; /* - * Convert encoding pte bits pmd bits + * Convert encoding pte bits pmd / pud bits * lIR.uswrdy.p dy..R...I...wr * empty 010.000000.0 -> 00..0...1...00 * prot-none, clean, old 111.000000.1 -> 00..1...1...00 @@ -33,25 +36,31 @@ static inline pmd_t __pte_to_pmd(pte_t pte) * u unused, l large */ if (pte_present(pte)) { - pmd_val(pmd) = pte_val(pte) & PAGE_MASK; - pmd_val(pmd) |= (pte_val(pte) & _PAGE_READ) >> 4; - pmd_val(pmd) |= (pte_val(pte) & _PAGE_WRITE) >> 4; - pmd_val(pmd) |= (pte_val(pte) & _PAGE_INVALID) >> 5; - pmd_val(pmd) |= (pte_val(pte) & _PAGE_PROTECT); - pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10; - pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10; - pmd_val(pmd) |= (pte_val(pte) & _PAGE_SOFT_DIRTY) << 13; + rste = pte_val(pte) & PAGE_MASK; + rste |= (pte_val(pte) & _PAGE_READ) >> 4; + rste |= (pte_val(pte) & _PAGE_WRITE) >> 4; + rste |= (pte_val(pte) & _PAGE_INVALID) >> 5; + rste |= (pte_val(pte) & _PAGE_PROTECT); + rste |= (pte_val(pte) & _PAGE_DIRTY) << 10; + rste |= (pte_val(pte) & _PAGE_YOUNG) << 10; + rste |= (pte_val(pte) & _PAGE_SOFT_DIRTY) << 13; } else - pmd_val(pmd) = _SEGMENT_ENTRY_INVALID; - return pmd; + rste = _SEGMENT_ENTRY_INVALID; + return rste; } -static inline pte_t __pmd_to_pte(pmd_t pmd) +static inline pte_t __rste_to_pte(unsigned long rste) { + int present; pte_t pte; + if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + present = pud_present(__pud(rste)); + else + present = pmd_present(__pmd(rste)); + /* - * Convert encoding pmd bits pte bits + * Convert encoding pmd / pud bits pte bits * dy..R...I...wr lIR.uswrdy.p * empty 00..0...1...00 -> 010.000000.0 * prot-none, clean, old 00..1...1...00 -> 111.000000.1 @@ -70,16 +79,16 @@ static inline pte_t __pmd_to_pte(pmd_t pmd) * SW-bits: p present, y young, d dirty, r read, w write, s special, * u unused, l large */ - if (pmd_present(pmd)) { - pte_val(pte) = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN_LARGE; + if (present) { + pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE; pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT; - pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_READ) << 4; - pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) << 4; - pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) << 5; - pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT); - pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) >> 10; - pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) >> 10; - pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_SOFT_DIRTY) >> 13; + pte_val(pte) |= (rste & _SEGMENT_ENTRY_READ) << 4; + pte_val(pte) |= (rste & _SEGMENT_ENTRY_WRITE) << 4; + pte_val(pte) |= (rste & _SEGMENT_ENTRY_INVALID) << 5; + pte_val(pte) |= (rste & _SEGMENT_ENTRY_PROTECT); + pte_val(pte) |= (rste & _SEGMENT_ENTRY_DIRTY) >> 10; + pte_val(pte) |= (rste & _SEGMENT_ENTRY_YOUNG) >> 10; + pte_val(pte) |= (rste & _SEGMENT_ENTRY_SOFT_DIRTY) >> 13; } else pte_val(pte) = _PAGE_INVALID; return pte; @@ -88,27 +97,33 @@ static inline pte_t __pmd_to_pte(pmd_t pmd) void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { - pmd_t pmd = __pte_to_pmd(pte); - - pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE; - *(pmd_t *) ptep = pmd; + unsigned long rste = __pte_to_rste(pte); + + /* Set correct table type for 2G hugepages */ + if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + rste |= _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE; + else + rste |= _SEGMENT_ENTRY_LARGE; + pte_val(*ptep) = rste; } pte_t huge_ptep_get(pte_t *ptep) { - pmd_t pmd = *(pmd_t *) ptep; - - return __pmd_to_pte(pmd); + return __rste_to_pte(pte_val(*ptep)); } pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { + pte_t pte = huge_ptep_get(ptep); pmd_t *pmdp = (pmd_t *) ptep; - pmd_t old; + pud_t *pudp = (pud_t *) ptep; - old = pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); - return __pmd_to_pte(old); + if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY)); + else + pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); + return pte; } pte_t *huge_pte_alloc(struct mm_struct *mm, @@ -120,8 +135,12 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, pgdp = pgd_offset(mm, addr); pudp = pud_alloc(mm, pgdp, addr); - if (pudp) - pmdp = pmd_alloc(mm, pudp, addr); + if (pudp) { + if (sz == PUD_SIZE) + return (pte_t *) pudp; + else if (sz == PMD_SIZE) + pmdp = pmd_alloc(mm, pudp, addr); + } return (pte_t *) pmdp; } @@ -134,8 +153,11 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) pgdp = pgd_offset(mm, addr); if (pgd_present(*pgdp)) { pudp = pud_offset(pgdp, addr); - if (pud_present(*pudp)) + if (pud_present(*pudp)) { + if (pud_large(*pudp)) + return (pte_t *) pudp; pmdp = pmd_offset(pudp, addr); + } } return (pte_t *) pmdp; } @@ -147,5 +169,34 @@ int pmd_huge(pmd_t pmd) int pud_huge(pud_t pud) { - return 0; + return pud_large(pud); +} + +struct page * +follow_huge_pud(struct mm_struct *mm, unsigned long address, + pud_t *pud, int flags) +{ + if (flags & FOLL_GET) + return NULL; + + return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); +} + +static __init int setup_hugepagesz(char *opt) +{ + unsigned long size; + char *string = opt; + + size = memparse(opt, &opt); + if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) { + hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); + } else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) { + hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); + } else { + pr_err("hugepagesz= specifies an unsupported page size %s\n", + string); + return 0; + } + return 1; } +__setup("hugepagesz=", setup_hugepagesz); diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 2489b2e917c8..f56a39bd8ba6 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -40,7 +40,7 @@ #include #include -pgd_t swapper_pg_dir[PTRS_PER_PGD] __attribute__((__aligned__(PAGE_SIZE))); +pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(.bss..swapper_pg_dir); unsigned long empty_zero_page, zero_page_mask; EXPORT_SYMBOL(empty_zero_page); @@ -111,17 +111,16 @@ void __init paging_init(void) void mark_rodata_ro(void) { - /* Text and rodata are already protected. Nothing to do here. */ - pr_info("Write protecting the kernel read-only data: %luk\n", - ((unsigned long)&_eshared - (unsigned long)&_stext) >> 10); + unsigned long size = __end_ro_after_init - __start_ro_after_init; + + set_memory_ro((unsigned long)__start_ro_after_init, size >> PAGE_SHIFT); + pr_info("Write protected read-only-after-init data: %luk\n", size >> 10); } void __init mem_init(void) { - if (MACHINE_HAS_TLB_LC) - cpumask_set_cpu(0, &init_mm.context.cpu_attach_mask); + cpumask_set_cpu(0, &init_mm.context.cpu_attach_mask); cpumask_set_cpu(0, mm_cpumask(&init_mm)); - atomic_set(&init_mm.context.attach_count, 1); set_max_mapnr(max_low_pfn); high_memory = (void *) __va(max_low_pfn * PAGE_SIZE); diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index a90d45e9dfb0..3330ea124eec 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -34,20 +34,25 @@ static int __init cmma(char *str) } __setup("cmma=", cmma); -void __init cmma_init(void) +static inline int cmma_test_essa(void) { register unsigned long tmp asm("0") = 0; register int rc asm("1") = -EOPNOTSUPP; - if (!cmma_flag) - return; asm volatile( " .insn rrf,0xb9ab0000,%1,%1,0,0\n" "0: la %0,0\n" "1:\n" EX_TABLE(0b,1b) : "+&d" (rc), "+&d" (tmp)); - if (rc) + return rc; +} + +void __init cmma_init(void) +{ + if (!cmma_flag) + return; + if (cmma_test_essa()) cmma_flag = 0; } diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index f2a5c29a97e9..7104ffb5a67f 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -10,7 +10,6 @@ #include #include -#if PAGE_DEFAULT_KEY static inline unsigned long sske_frame(unsigned long addr, unsigned char skey) { asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0" @@ -22,6 +21,8 @@ void __storage_key_init_range(unsigned long start, unsigned long end) { unsigned long boundary, size; + if (!PAGE_DEFAULT_KEY) + return; while (start < end) { if (MACHINE_HAS_EDAT1) { /* set storage keys for a 1MB frame */ @@ -38,56 +39,254 @@ void __storage_key_init_range(unsigned long start, unsigned long end) start += PAGE_SIZE; } } -#endif -static pte_t *walk_page_table(unsigned long addr) +#ifdef CONFIG_PROC_FS +atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX]; + +void arch_report_meminfo(struct seq_file *m) { - pgd_t *pgdp; - pud_t *pudp; + seq_printf(m, "DirectMap4k: %8lu kB\n", + atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_4K]) << 2); + seq_printf(m, "DirectMap1M: %8lu kB\n", + atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_1M]) << 10); + seq_printf(m, "DirectMap2G: %8lu kB\n", + atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_2G]) << 21); +} +#endif /* CONFIG_PROC_FS */ + +static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, + unsigned long dtt) +{ + unsigned long table, mask; + + mask = 0; + if (MACHINE_HAS_EDAT2) { + switch (dtt) { + case CRDTE_DTT_REGION3: + mask = ~(PTRS_PER_PUD * sizeof(pud_t) - 1); + break; + case CRDTE_DTT_SEGMENT: + mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); + break; + case CRDTE_DTT_PAGE: + mask = ~(PTRS_PER_PTE * sizeof(pte_t) - 1); + break; + } + table = (unsigned long)old & mask; + crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce); + } else if (MACHINE_HAS_IDTE) { + cspg(old, *old, new); + } else { + csp((unsigned int *)old + 1, *old, new); + } +} + +struct cpa { + unsigned int set_ro : 1; + unsigned int clear_ro : 1; +}; + +static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, + struct cpa cpa) +{ + pte_t *ptep, new; + + ptep = pte_offset(pmdp, addr); + do { + if (pte_none(*ptep)) + return -EINVAL; + if (cpa.set_ro) + new = pte_wrprotect(*ptep); + else if (cpa.clear_ro) + new = pte_mkwrite(pte_mkdirty(*ptep)); + pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE); + ptep++; + addr += PAGE_SIZE; + cond_resched(); + } while (addr < end); + return 0; +} + +static int split_pmd_page(pmd_t *pmdp, unsigned long addr) +{ + unsigned long pte_addr, prot; + pte_t *pt_dir, *ptep; + pmd_t new; + int i, ro; + + pt_dir = vmem_pte_alloc(); + if (!pt_dir) + return -ENOMEM; + pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; + ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); + prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); + ptep = pt_dir; + for (i = 0; i < PTRS_PER_PTE; i++) { + pte_val(*ptep) = pte_addr | prot; + pte_addr += PAGE_SIZE; + ptep++; + } + pmd_val(new) = __pa(pt_dir) | _SEGMENT_ENTRY; + pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); + update_page_count(PG_DIRECT_MAP_4K, PTRS_PER_PTE); + update_page_count(PG_DIRECT_MAP_1M, -1); + return 0; +} + +static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, struct cpa cpa) +{ + pmd_t new; + + if (cpa.set_ro) + new = pmd_wrprotect(*pmdp); + else if (cpa.clear_ro) + new = pmd_mkwrite(pmd_mkdirty(*pmdp)); + pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); +} + +static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, + struct cpa cpa) +{ + unsigned long next; pmd_t *pmdp; - pte_t *ptep; + int rc = 0; - pgdp = pgd_offset_k(addr); - if (pgd_none(*pgdp)) - return NULL; - pudp = pud_offset(pgdp, addr); - if (pud_none(*pudp) || pud_large(*pudp)) - return NULL; pmdp = pmd_offset(pudp, addr); - if (pmd_none(*pmdp) || pmd_large(*pmdp)) - return NULL; - ptep = pte_offset_kernel(pmdp, addr); - if (pte_none(*ptep)) - return NULL; - return ptep; + do { + if (pmd_none(*pmdp)) + return -EINVAL; + next = pmd_addr_end(addr, end); + if (pmd_large(*pmdp)) { + if (addr & ~PMD_MASK || addr + PMD_SIZE > next) { + rc = split_pmd_page(pmdp, addr); + if (rc) + return rc; + continue; + } + modify_pmd_page(pmdp, addr, cpa); + } else { + rc = walk_pte_level(pmdp, addr, next, cpa); + if (rc) + return rc; + } + pmdp++; + addr = next; + cond_resched(); + } while (addr < end); + return rc; } -static void change_page_attr(unsigned long addr, int numpages, - pte_t (*set) (pte_t)) +static int split_pud_page(pud_t *pudp, unsigned long addr) { - pte_t *ptep; - int i; + unsigned long pmd_addr, prot; + pmd_t *pm_dir, *pmdp; + pud_t new; + int i, ro; - for (i = 0; i < numpages; i++) { - ptep = walk_page_table(addr); - if (WARN_ON_ONCE(!ptep)) - break; - *ptep = set(*ptep); - addr += PAGE_SIZE; + pm_dir = vmem_pmd_alloc(); + if (!pm_dir) + return -ENOMEM; + pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; + ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); + prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL); + pmdp = pm_dir; + for (i = 0; i < PTRS_PER_PMD; i++) { + pmd_val(*pmdp) = pmd_addr | prot; + pmd_addr += PMD_SIZE; + pmdp++; } - __tlb_flush_kernel(); + pud_val(new) = __pa(pm_dir) | _REGION3_ENTRY; + pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); + update_page_count(PG_DIRECT_MAP_1M, PTRS_PER_PMD); + update_page_count(PG_DIRECT_MAP_2G, -1); + return 0; +} + +static void modify_pud_page(pud_t *pudp, unsigned long addr, struct cpa cpa) +{ + pud_t new; + + if (cpa.set_ro) + new = pud_wrprotect(*pudp); + else if (cpa.clear_ro) + new = pud_mkwrite(pud_mkdirty(*pudp)); + pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); +} + +static int walk_pud_level(pgd_t *pgd, unsigned long addr, unsigned long end, + struct cpa cpa) +{ + unsigned long next; + pud_t *pudp; + int rc = 0; + + pudp = pud_offset(pgd, addr); + do { + if (pud_none(*pudp)) + return -EINVAL; + next = pud_addr_end(addr, end); + if (pud_large(*pudp)) { + if (addr & ~PUD_MASK || addr + PUD_SIZE > next) { + rc = split_pud_page(pudp, addr); + if (rc) + break; + continue; + } + modify_pud_page(pudp, addr, cpa); + } else { + rc = walk_pmd_level(pudp, addr, next, cpa); + } + pudp++; + addr = next; + cond_resched(); + } while (addr < end && !rc); + return rc; +} + +static DEFINE_MUTEX(cpa_mutex); + +static int change_page_attr(unsigned long addr, unsigned long end, + struct cpa cpa) +{ + unsigned long next; + int rc = -EINVAL; + pgd_t *pgdp; + + if (end >= MODULES_END) + return -EINVAL; + mutex_lock(&cpa_mutex); + pgdp = pgd_offset_k(addr); + do { + if (pgd_none(*pgdp)) + break; + next = pgd_addr_end(addr, end); + rc = walk_pud_level(pgdp, addr, next, cpa); + if (rc) + break; + cond_resched(); + } while (pgdp++, addr = next, addr < end && !rc); + mutex_unlock(&cpa_mutex); + return rc; } int set_memory_ro(unsigned long addr, int numpages) { - change_page_attr(addr, numpages, pte_wrprotect); - return 0; + struct cpa cpa = { + .set_ro = 1, + }; + + addr &= PAGE_MASK; + return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa); } int set_memory_rw(unsigned long addr, int numpages) { - change_page_attr(addr, numpages, pte_mkwrite); - return 0; + struct cpa cpa = { + .clear_ro = 1, + }; + + addr &= PAGE_MASK; + return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa); } /* not possible */ @@ -138,7 +337,7 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) nr = min(numpages - i, nr); if (enable) { for (j = 0; j < nr; j++) { - pte_val(*pte) = __pa(address); + pte_val(*pte) = address | pgprot_val(PAGE_KERNEL); address += PAGE_SIZE; pte++; } diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index e8b5962ac12a..e2565d2d0c32 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -169,7 +169,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm) return table; } /* Allocate a fresh page */ - page = alloc_page(GFP_KERNEL|__GFP_REPEAT); + page = alloc_page(GFP_KERNEL); if (!page) return NULL; if (!pgtable_page_ctor(page)) { diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 4324b87f9398..b98d1a152d46 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -27,40 +27,37 @@ static inline pte_t ptep_flush_direct(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - int active, count; pte_t old; old = *ptep; if (unlikely(pte_val(old) & _PAGE_INVALID)) return old; - active = (mm == current->active_mm) ? 1 : 0; - count = atomic_add_return(0x10000, &mm->context.attach_count); - if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active && + atomic_inc(&mm->context.flush_count); + if (MACHINE_HAS_TLB_LC && cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) __ptep_ipte_local(addr, ptep); else __ptep_ipte(addr, ptep); - atomic_sub(0x10000, &mm->context.attach_count); + atomic_dec(&mm->context.flush_count); return old; } static inline pte_t ptep_flush_lazy(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - int active, count; pte_t old; old = *ptep; if (unlikely(pte_val(old) & _PAGE_INVALID)) return old; - active = (mm == current->active_mm) ? 1 : 0; - count = atomic_add_return(0x10000, &mm->context.attach_count); - if ((count & 0xffff) <= active) { + atomic_inc(&mm->context.flush_count); + if (cpumask_equal(&mm->context.cpu_attach_mask, + cpumask_of(smp_processor_id()))) { pte_val(*ptep) |= _PAGE_INVALID; mm->context.flush_mm = 1; } else __ptep_ipte(addr, ptep); - atomic_sub(0x10000, &mm->context.attach_count); + atomic_dec(&mm->context.flush_count); return old; } @@ -70,7 +67,6 @@ static inline pgste_t pgste_get_lock(pte_t *ptep) #ifdef CONFIG_PGSTE unsigned long old; - preempt_disable(); asm( " lg %0,%2\n" "0: lgr %1,%0\n" @@ -93,7 +89,6 @@ static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) : "=Q" (ptep[PTRS_PER_PTE]) : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); - preempt_enable(); #endif } @@ -230,9 +225,11 @@ pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, pgste_t pgste; pte_t old; + preempt_disable(); pgste = ptep_xchg_start(mm, addr, ptep); old = ptep_flush_direct(mm, addr, ptep); ptep_xchg_commit(mm, addr, ptep, pgste, old, new); + preempt_enable(); return old; } EXPORT_SYMBOL(ptep_xchg_direct); @@ -243,9 +240,11 @@ pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, pgste_t pgste; pte_t old; + preempt_disable(); pgste = ptep_xchg_start(mm, addr, ptep); old = ptep_flush_lazy(mm, addr, ptep); ptep_xchg_commit(mm, addr, ptep, pgste, old, new); + preempt_enable(); return old; } EXPORT_SYMBOL(ptep_xchg_lazy); @@ -256,6 +255,7 @@ pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pgste_t pgste; pte_t old; + preempt_disable(); pgste = ptep_xchg_start(mm, addr, ptep); old = ptep_flush_lazy(mm, addr, ptep); if (mm_has_pgste(mm)) { @@ -279,13 +279,13 @@ void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, } else { *ptep = pte; } + preempt_enable(); } EXPORT_SYMBOL(ptep_modify_prot_commit); static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { - int active, count; pmd_t old; old = *pmdp; @@ -295,36 +295,34 @@ static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, __pmdp_csp(pmdp); return old; } - active = (mm == current->active_mm) ? 1 : 0; - count = atomic_add_return(0x10000, &mm->context.attach_count); - if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active && + atomic_inc(&mm->context.flush_count); + if (MACHINE_HAS_TLB_LC && cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) __pmdp_idte_local(addr, pmdp); else __pmdp_idte(addr, pmdp); - atomic_sub(0x10000, &mm->context.attach_count); + atomic_dec(&mm->context.flush_count); return old; } static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { - int active, count; pmd_t old; old = *pmdp; if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) return old; - active = (mm == current->active_mm) ? 1 : 0; - count = atomic_add_return(0x10000, &mm->context.attach_count); - if ((count & 0xffff) <= active) { + atomic_inc(&mm->context.flush_count); + if (cpumask_equal(&mm->context.cpu_attach_mask, + cpumask_of(smp_processor_id()))) { pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; mm->context.flush_mm = 1; } else if (MACHINE_HAS_IDTE) __pmdp_idte(addr, pmdp); else __pmdp_csp(pmdp); - atomic_sub(0x10000, &mm->context.attach_count); + atomic_dec(&mm->context.flush_count); return old; } @@ -333,8 +331,10 @@ pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, { pmd_t old; + preempt_disable(); old = pmdp_flush_direct(mm, addr, pmdp); *pmdp = new; + preempt_enable(); return old; } EXPORT_SYMBOL(pmdp_xchg_direct); @@ -344,12 +344,53 @@ pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, { pmd_t old; + preempt_disable(); old = pmdp_flush_lazy(mm, addr, pmdp); *pmdp = new; + preempt_enable(); return old; } EXPORT_SYMBOL(pmdp_xchg_lazy); +static inline pud_t pudp_flush_direct(struct mm_struct *mm, + unsigned long addr, pud_t *pudp) +{ + pud_t old; + + old = *pudp; + if (pud_val(old) & _REGION_ENTRY_INVALID) + return old; + if (!MACHINE_HAS_IDTE) { + /* + * Invalid bit position is the same for pmd and pud, so we can + * re-use _pmd_csp() here + */ + __pmdp_csp((pmd_t *) pudp); + return old; + } + atomic_inc(&mm->context.flush_count); + if (MACHINE_HAS_TLB_LC && + cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) + __pudp_idte_local(addr, pudp); + else + __pudp_idte(addr, pudp); + atomic_dec(&mm->context.flush_count); + return old; +} + +pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, + pud_t *pudp, pud_t new) +{ + pud_t old; + + preempt_disable(); + old = pudp_flush_direct(mm, addr, pudp); + *pudp = new; + preempt_enable(); + return old; +} +EXPORT_SYMBOL(pudp_xchg_direct); + #ifdef CONFIG_TRANSPARENT_HUGEPAGE void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) @@ -398,20 +439,24 @@ void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, pgste_t pgste; /* the mm_has_pgste() check is done in set_pte_at() */ + preempt_disable(); pgste = pgste_get_lock(ptep); pgste_val(pgste) &= ~_PGSTE_GPS_ZERO; pgste_set_key(ptep, pgste, entry, mm); pgste = pgste_set_pte(ptep, pgste, entry); pgste_set_unlock(ptep, pgste); + preempt_enable(); } void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pgste_t pgste; + preempt_disable(); pgste = pgste_get_lock(ptep); pgste_val(pgste) |= PGSTE_IN_BIT; pgste_set_unlock(ptep, pgste); + preempt_enable(); } static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) @@ -434,10 +479,11 @@ void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, pte_t pte; /* Zap unused and logically-zero pages */ + preempt_disable(); pgste = pgste_get_lock(ptep); pgstev = pgste_val(pgste); pte = *ptep; - if (pte_swap(pte) && + if (!reset && pte_swap(pte) && ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || (pgstev & _PGSTE_GPS_ZERO))) { ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); @@ -446,6 +492,7 @@ void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, if (reset) pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; pgste_set_unlock(ptep, pgste); + preempt_enable(); } void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) @@ -454,6 +501,7 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) pgste_t pgste; /* Clear storage key */ + preempt_disable(); pgste = pgste_get_lock(ptep); pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT); @@ -461,6 +509,7 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); pgste_set_unlock(ptep, pgste); + preempt_enable(); } /* diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index d48cf25cfe99..1848292766ef 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -29,9 +30,11 @@ static LIST_HEAD(mem_segs); static void __ref *vmem_alloc_pages(unsigned int order) { + unsigned long size = PAGE_SIZE << order; + if (slab_is_available()) return (void *)__get_free_pages(GFP_KERNEL, order); - return alloc_bootmem_pages((1 << order) * PAGE_SIZE); + return alloc_bootmem_align(size, size); } static inline pud_t *vmem_pud_alloc(void) @@ -45,7 +48,7 @@ static inline pud_t *vmem_pud_alloc(void) return pud; } -static inline pmd_t *vmem_pmd_alloc(void) +pmd_t *vmem_pmd_alloc(void) { pmd_t *pmd = NULL; @@ -56,7 +59,7 @@ static inline pmd_t *vmem_pmd_alloc(void) return pmd; } -static pte_t __ref *vmem_pte_alloc(void) +pte_t __ref *vmem_pte_alloc(void) { pte_t *pte; @@ -75,8 +78,9 @@ static pte_t __ref *vmem_pte_alloc(void) /* * Add a physical memory range to the 1:1 mapping. */ -static int vmem_add_mem(unsigned long start, unsigned long size, int ro) +static int vmem_add_mem(unsigned long start, unsigned long size) { + unsigned long pages4k, pages1m, pages2g; unsigned long end = start + size; unsigned long address = start; pgd_t *pg_dir; @@ -85,6 +89,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) pte_t *pt_dir; int ret = -ENOMEM; + pages4k = pages1m = pages2g = 0; while (address < end) { pg_dir = pgd_offset_k(address); if (pgd_none(*pg_dir)) { @@ -97,10 +102,9 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address && !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) && !debug_pagealloc_enabled()) { - pud_val(*pu_dir) = __pa(address) | - _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE | - (ro ? _REGION_ENTRY_PROTECT : 0); + pud_val(*pu_dir) = address | pgprot_val(REGION3_KERNEL); address += PUD_SIZE; + pages2g++; continue; } if (pud_none(*pu_dir)) { @@ -113,11 +117,9 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address && !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) && !debug_pagealloc_enabled()) { - pmd_val(*pm_dir) = __pa(address) | - _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE | - _SEGMENT_ENTRY_YOUNG | - (ro ? _SEGMENT_ENTRY_PROTECT : 0); + pmd_val(*pm_dir) = address | pgprot_val(SEGMENT_KERNEL); address += PMD_SIZE; + pages1m++; continue; } if (pmd_none(*pm_dir)) { @@ -128,12 +130,15 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) } pt_dir = pte_offset_kernel(pm_dir, address); - pte_val(*pt_dir) = __pa(address) | - pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); + pte_val(*pt_dir) = address | pgprot_val(PAGE_KERNEL); address += PAGE_SIZE; + pages4k++; } ret = 0; out: + update_page_count(PG_DIRECT_MAP_4K, pages4k); + update_page_count(PG_DIRECT_MAP_1M, pages1m); + update_page_count(PG_DIRECT_MAP_2G, pages2g); return ret; } @@ -143,15 +148,15 @@ out: */ static void vmem_remove_range(unsigned long start, unsigned long size) { + unsigned long pages4k, pages1m, pages2g; unsigned long end = start + size; unsigned long address = start; pgd_t *pg_dir; pud_t *pu_dir; pmd_t *pm_dir; pte_t *pt_dir; - pte_t pte; - pte_val(pte) = _PAGE_INVALID; + pages4k = pages1m = pages2g = 0; while (address < end) { pg_dir = pgd_offset_k(address); if (pgd_none(*pg_dir)) { @@ -166,6 +171,7 @@ static void vmem_remove_range(unsigned long start, unsigned long size) if (pud_large(*pu_dir)) { pud_clear(pu_dir); address += PUD_SIZE; + pages2g++; continue; } pm_dir = pmd_offset(pu_dir, address); @@ -176,13 +182,18 @@ static void vmem_remove_range(unsigned long start, unsigned long size) if (pmd_large(*pm_dir)) { pmd_clear(pm_dir); address += PMD_SIZE; + pages1m++; continue; } pt_dir = pte_offset_kernel(pm_dir, address); - *pt_dir = pte; + pte_clear(&init_mm, address, pt_dir); address += PAGE_SIZE; + pages4k++; } flush_tlb_kernel_range(start, end); + update_page_count(PG_DIRECT_MAP_4K, -pages4k); + update_page_count(PG_DIRECT_MAP_1M, -pages1m); + update_page_count(PG_DIRECT_MAP_2G, -pages2g); } /* @@ -341,7 +352,7 @@ int vmem_add_mapping(unsigned long start, unsigned long size) if (ret) goto out_free; - ret = vmem_add_mem(start, size, 0); + ret = vmem_add_mem(start, size); if (ret) goto out_remove; goto out; @@ -362,31 +373,13 @@ out: */ void __init vmem_map_init(void) { - unsigned long ro_start, ro_end; + unsigned long size = _eshared - _stext; struct memblock_region *reg; - phys_addr_t start, end; - ro_start = PFN_ALIGN((unsigned long)&_stext); - ro_end = (unsigned long)&_eshared & PAGE_MASK; - for_each_memblock(memory, reg) { - start = reg->base; - end = reg->base + reg->size; - if (start >= ro_end || end <= ro_start) - vmem_add_mem(start, end - start, 0); - else if (start >= ro_start && end <= ro_end) - vmem_add_mem(start, end - start, 1); - else if (start >= ro_start) { - vmem_add_mem(start, ro_end - start, 1); - vmem_add_mem(ro_end, end - ro_end, 0); - } else if (end < ro_end) { - vmem_add_mem(start, ro_start - start, 0); - vmem_add_mem(ro_start, end - ro_start, 1); - } else { - vmem_add_mem(start, ro_start - start, 0); - vmem_add_mem(ro_start, ro_end - ro_start, 1); - vmem_add_mem(ro_end, end - ro_end, 0); - } - } + for_each_memblock(memory, reg) + vmem_add_mem(reg->base, reg->size); + set_memory_ro((unsigned long)_stext, size >> PAGE_SHIFT); + pr_info("Write protected kernel read-only data: %luk\n", size >> 10); } /* diff --git a/arch/s390/numa/mode_emu.c b/arch/s390/numa/mode_emu.c index 828d0695d0d4..fbc394e16b2c 100644 --- a/arch/s390/numa/mode_emu.c +++ b/arch/s390/numa/mode_emu.c @@ -34,7 +34,8 @@ #define DIST_CORE 1 #define DIST_MC 2 #define DIST_BOOK 3 -#define DIST_MAX 4 +#define DIST_DRAWER 4 +#define DIST_MAX 5 /* Node distance reported to common code */ #define EMU_NODE_DIST 10 @@ -43,7 +44,7 @@ #define NODE_ID_FREE -1 /* Different levels of toptree */ -enum toptree_level {CORE, MC, BOOK, NODE, TOPOLOGY}; +enum toptree_level {CORE, MC, BOOK, DRAWER, NODE, TOPOLOGY}; /* The two toptree IDs */ enum {TOPTREE_ID_PHYS, TOPTREE_ID_NUMA}; @@ -113,6 +114,14 @@ static int cores_free(struct toptree *tree) * Return node of core */ static struct toptree *core_node(struct toptree *core) +{ + return core->parent->parent->parent->parent; +} + +/* + * Return drawer of core + */ +static struct toptree *core_drawer(struct toptree *core) { return core->parent->parent->parent; } @@ -138,6 +147,8 @@ static struct toptree *core_mc(struct toptree *core) */ static int dist_core_to_core(struct toptree *core1, struct toptree *core2) { + if (core_drawer(core1)->id != core_drawer(core2)->id) + return DIST_DRAWER; if (core_book(core1)->id != core_book(core2)->id) return DIST_BOOK; if (core_mc(core1)->id != core_mc(core2)->id) @@ -262,6 +273,8 @@ static void toptree_to_numa_first(struct toptree *numa, struct toptree *phys) struct toptree *core; /* Always try to move perfectly fitting structures first */ + move_level_to_numa(numa, phys, DRAWER, true); + move_level_to_numa(numa, phys, DRAWER, false); move_level_to_numa(numa, phys, BOOK, true); move_level_to_numa(numa, phys, BOOK, false); move_level_to_numa(numa, phys, MC, true); @@ -335,7 +348,7 @@ static struct toptree *toptree_to_numa(struct toptree *phys) */ static struct toptree *toptree_from_topology(void) { - struct toptree *phys, *node, *book, *mc, *core; + struct toptree *phys, *node, *drawer, *book, *mc, *core; struct cpu_topology_s390 *top; int cpu; @@ -344,10 +357,11 @@ static struct toptree *toptree_from_topology(void) for_each_online_cpu(cpu) { top = &per_cpu(cpu_topology, cpu); node = toptree_get_child(phys, 0); - book = toptree_get_child(node, top->book_id); + drawer = toptree_get_child(node, top->drawer_id); + book = toptree_get_child(drawer, top->book_id); mc = toptree_get_child(book, top->socket_id); core = toptree_get_child(mc, top->core_id); - if (!book || !mc || !core) + if (!drawer || !book || !mc || !core) panic("NUMA emulation could not allocate memory"); cpumask_set_cpu(cpu, &core->mask); toptree_update_mask(mc); @@ -368,6 +382,7 @@ static void topology_add_core(struct toptree *core) cpumask_copy(&top->thread_mask, &core->mask); cpumask_copy(&top->core_mask, &core_mc(core)->mask); cpumask_copy(&top->book_mask, &core_book(core)->mask); + cpumask_copy(&top->drawer_mask, &core_drawer(core)->mask); cpumask_set_cpu(cpu, &node_to_cpumask_map[core_node(core)->id]); top->node_id = core_node(core)->id; } diff --git a/arch/s390/oprofile/Makefile b/arch/s390/oprofile/Makefile index 496e4a7ee00e..e9dd41b0b8d3 100644 --- a/arch/s390/oprofile/Makefile +++ b/arch/s390/oprofile/Makefile @@ -7,4 +7,3 @@ DRIVER_OBJS = $(addprefix ../../../drivers/oprofile/, \ timer_int.o ) oprofile-y := $(DRIVER_OBJS) init.o -oprofile-y += hwsampler.o diff --git a/arch/s390/oprofile/hwsampler.c b/arch/s390/oprofile/hwsampler.c deleted file mode 100644 index ff9b4eb34589..000000000000 --- a/arch/s390/oprofile/hwsampler.c +++ /dev/null @@ -1,1178 +0,0 @@ -/* - * Copyright IBM Corp. 2010 - * Author: Heinz Graalfs - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "hwsampler.h" -#include "op_counter.h" - -#define MAX_NUM_SDB 511 -#define MIN_NUM_SDB 1 - -DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer); - -struct hws_execute_parms { - void *buffer; - signed int rc; -}; - -DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer); -EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer); - -static DEFINE_MUTEX(hws_sem); -static DEFINE_MUTEX(hws_sem_oom); - -static unsigned char hws_flush_all; -static unsigned int hws_oom; -static unsigned int hws_alert; -static struct workqueue_struct *hws_wq; - -static unsigned int hws_state; -enum { - HWS_INIT = 1, - HWS_DEALLOCATED, - HWS_STOPPED, - HWS_STARTED, - HWS_STOPPING }; - -/* set to 1 if called by kernel during memory allocation */ -static unsigned char oom_killer_was_active; -/* size of SDBT and SDB as of allocate API */ -static unsigned long num_sdbt = 100; -static unsigned long num_sdb = 511; -/* sampling interval (machine cycles) */ -static unsigned long interval; - -static unsigned long min_sampler_rate; -static unsigned long max_sampler_rate; - -static void execute_qsi(void *parms) -{ - struct hws_execute_parms *ep = parms; - - ep->rc = qsi(ep->buffer); -} - -static void execute_ssctl(void *parms) -{ - struct hws_execute_parms *ep = parms; - - ep->rc = lsctl(ep->buffer); -} - -static int smp_ctl_ssctl_stop(int cpu) -{ - int rc; - struct hws_execute_parms ep; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - cb->ssctl.es = 0; - cb->ssctl.cs = 0; - - ep.buffer = &cb->ssctl; - smp_call_function_single(cpu, execute_ssctl, &ep, 1); - rc = ep.rc; - if (rc) { - printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu); - dump_stack(); - } - - ep.buffer = &cb->qsi; - smp_call_function_single(cpu, execute_qsi, &ep, 1); - - if (cb->qsi.es || cb->qsi.cs) { - printk(KERN_EMERG "CPUMF sampling did not stop properly.\n"); - dump_stack(); - } - - return rc; -} - -static int smp_ctl_ssctl_deactivate(int cpu) -{ - int rc; - struct hws_execute_parms ep; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - cb->ssctl.es = 1; - cb->ssctl.cs = 0; - - ep.buffer = &cb->ssctl; - smp_call_function_single(cpu, execute_ssctl, &ep, 1); - rc = ep.rc; - if (rc) - printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu); - - ep.buffer = &cb->qsi; - smp_call_function_single(cpu, execute_qsi, &ep, 1); - - if (cb->qsi.cs) - printk(KERN_EMERG "CPUMF sampling was not set inactive.\n"); - - return rc; -} - -static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval) -{ - int rc; - struct hws_execute_parms ep; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - cb->ssctl.h = 1; - cb->ssctl.tear = cb->first_sdbt; - cb->ssctl.dear = *(unsigned long *) cb->first_sdbt; - cb->ssctl.interval = interval; - cb->ssctl.es = 1; - cb->ssctl.cs = 1; - - ep.buffer = &cb->ssctl; - smp_call_function_single(cpu, execute_ssctl, &ep, 1); - rc = ep.rc; - if (rc) - printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu); - - ep.buffer = &cb->qsi; - smp_call_function_single(cpu, execute_qsi, &ep, 1); - if (ep.rc) - printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu); - - return rc; -} - -static int smp_ctl_qsi(int cpu) -{ - struct hws_execute_parms ep; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - ep.buffer = &cb->qsi; - smp_call_function_single(cpu, execute_qsi, &ep, 1); - - return ep.rc; -} - -static void hws_ext_handler(struct ext_code ext_code, - unsigned int param32, unsigned long param64) -{ - struct hws_cpu_buffer *cb = this_cpu_ptr(&sampler_cpu_buffer); - - if (!(param32 & CPU_MF_INT_SF_MASK)) - return; - - if (!hws_alert) - return; - - inc_irq_stat(IRQEXT_CMS); - atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32); - - if (hws_wq) - queue_work(hws_wq, &cb->worker); -} - -static void worker(struct work_struct *work); - -static void add_samples_to_oprofile(unsigned cpu, unsigned long *, - unsigned long *dear); - -static void init_all_cpu_buffers(void) -{ - int cpu; - struct hws_cpu_buffer *cb; - - for_each_online_cpu(cpu) { - cb = &per_cpu(sampler_cpu_buffer, cpu); - memset(cb, 0, sizeof(struct hws_cpu_buffer)); - } -} - -static void prepare_cpu_buffers(void) -{ - struct hws_cpu_buffer *cb; - int cpu; - - for_each_online_cpu(cpu) { - cb = &per_cpu(sampler_cpu_buffer, cpu); - atomic_set(&cb->ext_params, 0); - cb->worker_entry = 0; - cb->sample_overflow = 0; - cb->req_alert = 0; - cb->incorrect_sdbt_entry = 0; - cb->invalid_entry_address = 0; - cb->loss_of_sample_data = 0; - cb->sample_auth_change_alert = 0; - cb->finish = 0; - cb->oom = 0; - cb->stop_mode = 0; - } -} - -/* - * allocate_sdbt() - allocate sampler memory - * @cpu: the cpu for which sampler memory is allocated - * - * A 4K page is allocated for each requested SDBT. - * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs. - * Set ALERT_REQ mask in each SDBs trailer. - * Returns zero if successful, <0 otherwise. - */ -static int allocate_sdbt(int cpu) -{ - int j, k, rc; - unsigned long *sdbt; - unsigned long sdb; - unsigned long *tail; - unsigned long *trailer; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - if (cb->first_sdbt) - return -EINVAL; - - sdbt = NULL; - tail = sdbt; - - for (j = 0; j < num_sdbt; j++) { - sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); - - mutex_lock(&hws_sem_oom); - /* OOM killer might have been activated */ - barrier(); - if (oom_killer_was_active || !sdbt) { - if (sdbt) - free_page((unsigned long)sdbt); - - goto allocate_sdbt_error; - } - if (cb->first_sdbt == 0) - cb->first_sdbt = (unsigned long)sdbt; - - /* link current page to tail of chain */ - if (tail) - *tail = (unsigned long)(void *)sdbt + 1; - - mutex_unlock(&hws_sem_oom); - - for (k = 0; k < num_sdb; k++) { - /* get and set SDB page */ - sdb = get_zeroed_page(GFP_KERNEL); - - mutex_lock(&hws_sem_oom); - /* OOM killer might have been activated */ - barrier(); - if (oom_killer_was_active || !sdb) { - if (sdb) - free_page(sdb); - - goto allocate_sdbt_error; - } - *sdbt = sdb; - trailer = trailer_entry_ptr(*sdbt); - *trailer = SDB_TE_ALERT_REQ_MASK; - sdbt++; - mutex_unlock(&hws_sem_oom); - } - tail = sdbt; - } - mutex_lock(&hws_sem_oom); - if (oom_killer_was_active) - goto allocate_sdbt_error; - - rc = 0; - if (tail) - *tail = (unsigned long) - ((void *)cb->first_sdbt) + 1; - -allocate_sdbt_exit: - mutex_unlock(&hws_sem_oom); - return rc; - -allocate_sdbt_error: - rc = -ENOMEM; - goto allocate_sdbt_exit; -} - -/* - * deallocate_sdbt() - deallocate all sampler memory - * - * For each online CPU all SDBT trees are deallocated. - * Returns the number of freed pages. - */ -static int deallocate_sdbt(void) -{ - int cpu; - int counter; - - counter = 0; - - for_each_online_cpu(cpu) { - unsigned long start; - unsigned long sdbt; - unsigned long *curr; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - if (!cb->first_sdbt) - continue; - - sdbt = cb->first_sdbt; - curr = (unsigned long *) sdbt; - start = sdbt; - - /* we'll free the SDBT after all SDBs are processed... */ - while (1) { - if (!*curr || !sdbt) - break; - - /* watch for link entry reset if found */ - if (is_link_entry(curr)) { - curr = get_next_sdbt(curr); - if (sdbt) - free_page(sdbt); - - /* we are done if we reach the start */ - if ((unsigned long) curr == start) - break; - else - sdbt = (unsigned long) curr; - } else { - /* process SDB pointer */ - if (*curr) { - free_page(*curr); - curr++; - } - } - counter++; - } - cb->first_sdbt = 0; - } - return counter; -} - -static int start_sampling(int cpu) -{ - int rc; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - rc = smp_ctl_ssctl_enable_activate(cpu, interval); - if (rc) { - printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu); - goto start_exit; - } - - rc = -EINVAL; - if (!cb->qsi.es) { - printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu); - goto start_exit; - } - - if (!cb->qsi.cs) { - printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu); - goto start_exit; - } - - printk(KERN_INFO - "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n", - cpu, interval); - - rc = 0; - -start_exit: - return rc; -} - -static int stop_sampling(int cpu) -{ - unsigned long v; - int rc; - struct hws_cpu_buffer *cb; - - rc = smp_ctl_qsi(cpu); - WARN_ON(rc); - - cb = &per_cpu(sampler_cpu_buffer, cpu); - if (!rc && !cb->qsi.es) - printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu); - - rc = smp_ctl_ssctl_stop(cpu); - if (rc) { - printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n", - cpu, rc); - goto stop_exit; - } - - printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu); - -stop_exit: - v = cb->req_alert; - if (v) - printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert," - " count=%lu.\n", cpu, v); - - v = cb->loss_of_sample_data; - if (v) - printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data," - " count=%lu.\n", cpu, v); - - v = cb->invalid_entry_address; - if (v) - printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address," - " count=%lu.\n", cpu, v); - - v = cb->incorrect_sdbt_entry; - if (v) - printk(KERN_ERR - "hwsampler: CPU %d CPUMF Incorrect SDBT address," - " count=%lu.\n", cpu, v); - - v = cb->sample_auth_change_alert; - if (v) - printk(KERN_ERR - "hwsampler: CPU %d CPUMF Sample authorization change," - " count=%lu.\n", cpu, v); - - return rc; -} - -static int check_hardware_prerequisites(void) -{ - if (!test_facility(68)) - return -EOPNOTSUPP; - return 0; -} -/* - * hws_oom_callback() - the OOM callback function - * - * In case the callback is invoked during memory allocation for the - * hw sampler, all obtained memory is deallocated and a flag is set - * so main sampler memory allocation can exit with a failure code. - * In case the callback is invoked during sampling the hw sampler - * is deactivated for all CPUs. - */ -static int hws_oom_callback(struct notifier_block *nfb, - unsigned long dummy, void *parm) -{ - unsigned long *freed; - int cpu; - struct hws_cpu_buffer *cb; - - freed = parm; - - mutex_lock(&hws_sem_oom); - - if (hws_state == HWS_DEALLOCATED) { - /* during memory allocation */ - if (oom_killer_was_active == 0) { - oom_killer_was_active = 1; - *freed += deallocate_sdbt(); - } - } else { - int i; - cpu = get_cpu(); - cb = &per_cpu(sampler_cpu_buffer, cpu); - - if (!cb->oom) { - for_each_online_cpu(i) { - smp_ctl_ssctl_deactivate(i); - cb->oom = 1; - } - cb->finish = 1; - - printk(KERN_INFO - "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n", - cpu); - } - } - - mutex_unlock(&hws_sem_oom); - - return NOTIFY_OK; -} - -static struct notifier_block hws_oom_notifier = { - .notifier_call = hws_oom_callback -}; - -static int hws_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - /* We do not have sampler space available for all possible CPUs. - All CPUs should be online when hw sampling is activated. */ - return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD; -} - -static struct notifier_block hws_cpu_notifier = { - .notifier_call = hws_cpu_callback -}; - -/** - * hwsampler_deactivate() - set hardware sampling temporarily inactive - * @cpu: specifies the CPU to be set inactive. - * - * Returns 0 on success, !0 on failure. - */ -int hwsampler_deactivate(unsigned int cpu) -{ - /* - * Deactivate hw sampling temporarily and flush the buffer - * by pushing all the pending samples to oprofile buffer. - * - * This function can be called under one of the following conditions: - * Memory unmap, task is exiting. - */ - int rc; - struct hws_cpu_buffer *cb; - - rc = 0; - mutex_lock(&hws_sem); - - cb = &per_cpu(sampler_cpu_buffer, cpu); - if (hws_state == HWS_STARTED) { - rc = smp_ctl_qsi(cpu); - WARN_ON(rc); - if (cb->qsi.cs) { - rc = smp_ctl_ssctl_deactivate(cpu); - if (rc) { - printk(KERN_INFO - "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu); - cb->finish = 1; - hws_state = HWS_STOPPING; - } else { - hws_flush_all = 1; - /* Add work to queue to read pending samples.*/ - queue_work_on(cpu, hws_wq, &cb->worker); - } - } - } - mutex_unlock(&hws_sem); - - if (hws_wq) - flush_workqueue(hws_wq); - - return rc; -} - -/** - * hwsampler_activate() - activate/resume hardware sampling which was deactivated - * @cpu: specifies the CPU to be set active. - * - * Returns 0 on success, !0 on failure. - */ -int hwsampler_activate(unsigned int cpu) -{ - /* - * Re-activate hw sampling. This should be called in pair with - * hwsampler_deactivate(). - */ - int rc; - struct hws_cpu_buffer *cb; - - rc = 0; - mutex_lock(&hws_sem); - - cb = &per_cpu(sampler_cpu_buffer, cpu); - if (hws_state == HWS_STARTED) { - rc = smp_ctl_qsi(cpu); - WARN_ON(rc); - if (!cb->qsi.cs) { - hws_flush_all = 0; - rc = smp_ctl_ssctl_enable_activate(cpu, interval); - if (rc) { - printk(KERN_ERR - "CPU %d, CPUMF activate sampling failed.\n", - cpu); - } - } - } - - mutex_unlock(&hws_sem); - - return rc; -} - -static int check_qsi_on_setup(void) -{ - int rc; - unsigned int cpu; - struct hws_cpu_buffer *cb; - - for_each_online_cpu(cpu) { - cb = &per_cpu(sampler_cpu_buffer, cpu); - rc = smp_ctl_qsi(cpu); - WARN_ON(rc); - if (rc) - return -EOPNOTSUPP; - - if (!cb->qsi.as) { - printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n"); - return -EINVAL; - } - - if (cb->qsi.es) { - printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n"); - rc = smp_ctl_ssctl_stop(cpu); - if (rc) - return -EINVAL; - - printk(KERN_INFO - "CPU %d, CPUMF Sampling stopped now.\n", cpu); - } - } - return 0; -} - -static int check_qsi_on_start(void) -{ - unsigned int cpu; - int rc; - struct hws_cpu_buffer *cb; - - for_each_online_cpu(cpu) { - cb = &per_cpu(sampler_cpu_buffer, cpu); - rc = smp_ctl_qsi(cpu); - WARN_ON(rc); - - if (!cb->qsi.as) - return -EINVAL; - - if (cb->qsi.es) - return -EINVAL; - - if (cb->qsi.cs) - return -EINVAL; - } - return 0; -} - -static void worker_on_start(unsigned int cpu) -{ - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - cb->worker_entry = cb->first_sdbt; -} - -static int worker_check_error(unsigned int cpu, int ext_params) -{ - int rc; - unsigned long *sdbt; - struct hws_cpu_buffer *cb; - - rc = 0; - cb = &per_cpu(sampler_cpu_buffer, cpu); - sdbt = (unsigned long *) cb->worker_entry; - - if (!sdbt || !*sdbt) - return -EINVAL; - - if (ext_params & CPU_MF_INT_SF_PRA) - cb->req_alert++; - - if (ext_params & CPU_MF_INT_SF_LSDA) - cb->loss_of_sample_data++; - - if (ext_params & CPU_MF_INT_SF_IAE) { - cb->invalid_entry_address++; - rc = -EINVAL; - } - - if (ext_params & CPU_MF_INT_SF_ISE) { - cb->incorrect_sdbt_entry++; - rc = -EINVAL; - } - - if (ext_params & CPU_MF_INT_SF_SACA) { - cb->sample_auth_change_alert++; - rc = -EINVAL; - } - - return rc; -} - -static void worker_on_finish(unsigned int cpu) -{ - int rc, i; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - if (cb->finish) { - rc = smp_ctl_qsi(cpu); - WARN_ON(rc); - if (cb->qsi.es) { - printk(KERN_INFO - "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n", - cpu); - rc = smp_ctl_ssctl_stop(cpu); - if (rc) - printk(KERN_INFO - "hwsampler: CPU %d, CPUMF Deactivation failed.\n", - cpu); - - for_each_online_cpu(i) { - if (i == cpu) - continue; - if (!cb->finish) { - cb->finish = 1; - queue_work_on(i, hws_wq, - &cb->worker); - } - } - } - } -} - -static void worker_on_interrupt(unsigned int cpu) -{ - unsigned long *sdbt; - unsigned char done; - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - sdbt = (unsigned long *) cb->worker_entry; - - done = 0; - /* do not proceed if stop was entered, - * forget the buffers not yet processed */ - while (!done && !cb->stop_mode) { - unsigned long *trailer; - struct hws_trailer_entry *te; - unsigned long *dear = 0; - - trailer = trailer_entry_ptr(*sdbt); - /* leave loop if no more work to do */ - if (!(*trailer & SDB_TE_BUFFER_FULL_MASK)) { - done = 1; - if (!hws_flush_all) - continue; - } - - te = (struct hws_trailer_entry *)trailer; - cb->sample_overflow += te->overflow; - - add_samples_to_oprofile(cpu, sdbt, dear); - - /* reset trailer */ - xchg((unsigned char *) te, 0x40); - - /* advance to next sdb slot in current sdbt */ - sdbt++; - /* in case link bit is set use address w/o link bit */ - if (is_link_entry(sdbt)) - sdbt = get_next_sdbt(sdbt); - - cb->worker_entry = (unsigned long)sdbt; - } -} - -static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt, - unsigned long *dear) -{ - struct hws_basic_entry *sample_data_ptr; - unsigned long *trailer; - - trailer = trailer_entry_ptr(*sdbt); - if (dear) { - if (dear > trailer) - return; - trailer = dear; - } - - sample_data_ptr = (struct hws_basic_entry *)(*sdbt); - - while ((unsigned long *)sample_data_ptr < trailer) { - struct pt_regs *regs = NULL; - struct task_struct *tsk = NULL; - - /* - * Check sampling mode, 1 indicates basic (=customer) sampling - * mode. - */ - if (sample_data_ptr->def != 1) { - /* sample slot is not yet written */ - break; - } else { - /* make sure we don't use it twice, - * the next time the sampler will set it again */ - sample_data_ptr->def = 0; - } - - /* Get pt_regs. */ - if (sample_data_ptr->P == 1) { - /* userspace sample */ - unsigned int pid = sample_data_ptr->prim_asn; - if (!counter_config.user) - goto skip_sample; - rcu_read_lock(); - tsk = pid_task(find_vpid(pid), PIDTYPE_PID); - if (tsk) - regs = task_pt_regs(tsk); - rcu_read_unlock(); - } else { - /* kernelspace sample */ - if (!counter_config.kernel) - goto skip_sample; - regs = task_pt_regs(current); - } - - mutex_lock(&hws_sem); - oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0, - !sample_data_ptr->P, tsk); - mutex_unlock(&hws_sem); - skip_sample: - sample_data_ptr++; - } -} - -static void worker(struct work_struct *work) -{ - unsigned int cpu; - int ext_params; - struct hws_cpu_buffer *cb; - - cb = container_of(work, struct hws_cpu_buffer, worker); - cpu = smp_processor_id(); - ext_params = atomic_xchg(&cb->ext_params, 0); - - if (!cb->worker_entry) - worker_on_start(cpu); - - if (worker_check_error(cpu, ext_params)) - return; - - if (!cb->finish) - worker_on_interrupt(cpu); - - if (cb->finish) - worker_on_finish(cpu); -} - -/** - * hwsampler_allocate() - allocate memory for the hardware sampler - * @sdbt: number of SDBTs per online CPU (must be > 0) - * @sdb: number of SDBs per SDBT (minimum 1, maximum 511) - * - * Returns 0 on success, !0 on failure. - */ -int hwsampler_allocate(unsigned long sdbt, unsigned long sdb) -{ - int cpu, rc; - mutex_lock(&hws_sem); - - rc = -EINVAL; - if (hws_state != HWS_DEALLOCATED) - goto allocate_exit; - - if (sdbt < 1) - goto allocate_exit; - - if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB) - goto allocate_exit; - - num_sdbt = sdbt; - num_sdb = sdb; - - oom_killer_was_active = 0; - register_oom_notifier(&hws_oom_notifier); - - for_each_online_cpu(cpu) { - if (allocate_sdbt(cpu)) { - unregister_oom_notifier(&hws_oom_notifier); - goto allocate_error; - } - } - unregister_oom_notifier(&hws_oom_notifier); - if (oom_killer_was_active) - goto allocate_error; - - hws_state = HWS_STOPPED; - rc = 0; - -allocate_exit: - mutex_unlock(&hws_sem); - return rc; - -allocate_error: - rc = -ENOMEM; - printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n"); - goto allocate_exit; -} - -/** - * hwsampler_deallocate() - deallocate hardware sampler memory - * - * Returns 0 on success, !0 on failure. - */ -int hwsampler_deallocate(void) -{ - int rc; - - mutex_lock(&hws_sem); - - rc = -EINVAL; - if (hws_state != HWS_STOPPED) - goto deallocate_exit; - - irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); - hws_alert = 0; - deallocate_sdbt(); - - hws_state = HWS_DEALLOCATED; - rc = 0; - -deallocate_exit: - mutex_unlock(&hws_sem); - - return rc; -} - -unsigned long hwsampler_query_min_interval(void) -{ - return min_sampler_rate; -} - -unsigned long hwsampler_query_max_interval(void) -{ - return max_sampler_rate; -} - -unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu) -{ - struct hws_cpu_buffer *cb; - - cb = &per_cpu(sampler_cpu_buffer, cpu); - - return cb->sample_overflow; -} - -int hwsampler_setup(void) -{ - int rc; - int cpu; - struct hws_cpu_buffer *cb; - - mutex_lock(&hws_sem); - - rc = -EINVAL; - if (hws_state) - goto setup_exit; - - hws_state = HWS_INIT; - - init_all_cpu_buffers(); - - rc = check_hardware_prerequisites(); - if (rc) - goto setup_exit; - - rc = check_qsi_on_setup(); - if (rc) - goto setup_exit; - - rc = -EINVAL; - hws_wq = create_workqueue("hwsampler"); - if (!hws_wq) - goto setup_exit; - - register_cpu_notifier(&hws_cpu_notifier); - - for_each_online_cpu(cpu) { - cb = &per_cpu(sampler_cpu_buffer, cpu); - INIT_WORK(&cb->worker, worker); - rc = smp_ctl_qsi(cpu); - WARN_ON(rc); - if (min_sampler_rate != cb->qsi.min_sampl_rate) { - if (min_sampler_rate) { - printk(KERN_WARNING - "hwsampler: different min sampler rate values.\n"); - if (min_sampler_rate < cb->qsi.min_sampl_rate) - min_sampler_rate = - cb->qsi.min_sampl_rate; - } else - min_sampler_rate = cb->qsi.min_sampl_rate; - } - if (max_sampler_rate != cb->qsi.max_sampl_rate) { - if (max_sampler_rate) { - printk(KERN_WARNING - "hwsampler: different max sampler rate values.\n"); - if (max_sampler_rate > cb->qsi.max_sampl_rate) - max_sampler_rate = - cb->qsi.max_sampl_rate; - } else - max_sampler_rate = cb->qsi.max_sampl_rate; - } - } - register_external_irq(EXT_IRQ_MEASURE_ALERT, hws_ext_handler); - - hws_state = HWS_DEALLOCATED; - rc = 0; - -setup_exit: - mutex_unlock(&hws_sem); - return rc; -} - -int hwsampler_shutdown(void) -{ - int rc; - - mutex_lock(&hws_sem); - - rc = -EINVAL; - if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) { - mutex_unlock(&hws_sem); - - if (hws_wq) - flush_workqueue(hws_wq); - - mutex_lock(&hws_sem); - - if (hws_state == HWS_STOPPED) { - irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); - hws_alert = 0; - deallocate_sdbt(); - } - if (hws_wq) { - destroy_workqueue(hws_wq); - hws_wq = NULL; - } - - unregister_external_irq(EXT_IRQ_MEASURE_ALERT, hws_ext_handler); - hws_state = HWS_INIT; - rc = 0; - } - mutex_unlock(&hws_sem); - - unregister_cpu_notifier(&hws_cpu_notifier); - - return rc; -} - -/** - * hwsampler_start_all() - start hardware sampling on all online CPUs - * @rate: specifies the used interval when samples are taken - * - * Returns 0 on success, !0 on failure. - */ -int hwsampler_start_all(unsigned long rate) -{ - int rc, cpu; - - mutex_lock(&hws_sem); - - hws_oom = 0; - - rc = -EINVAL; - if (hws_state != HWS_STOPPED) - goto start_all_exit; - - interval = rate; - - /* fail if rate is not valid */ - if (interval < min_sampler_rate || interval > max_sampler_rate) - goto start_all_exit; - - rc = check_qsi_on_start(); - if (rc) - goto start_all_exit; - - prepare_cpu_buffers(); - - for_each_online_cpu(cpu) { - rc = start_sampling(cpu); - if (rc) - break; - } - if (rc) { - for_each_online_cpu(cpu) { - stop_sampling(cpu); - } - goto start_all_exit; - } - hws_state = HWS_STARTED; - rc = 0; - -start_all_exit: - mutex_unlock(&hws_sem); - - if (rc) - return rc; - - register_oom_notifier(&hws_oom_notifier); - hws_oom = 1; - hws_flush_all = 0; - /* now let them in, 1407 CPUMF external interrupts */ - hws_alert = 1; - irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); - - return 0; -} - -/** - * hwsampler_stop_all() - stop hardware sampling on all online CPUs - * - * Returns 0 on success, !0 on failure. - */ -int hwsampler_stop_all(void) -{ - int tmp_rc, rc, cpu; - struct hws_cpu_buffer *cb; - - mutex_lock(&hws_sem); - - rc = 0; - if (hws_state == HWS_INIT) { - mutex_unlock(&hws_sem); - return 0; - } - hws_state = HWS_STOPPING; - mutex_unlock(&hws_sem); - - for_each_online_cpu(cpu) { - cb = &per_cpu(sampler_cpu_buffer, cpu); - cb->stop_mode = 1; - tmp_rc = stop_sampling(cpu); - if (tmp_rc) - rc = tmp_rc; - } - - if (hws_wq) - flush_workqueue(hws_wq); - - mutex_lock(&hws_sem); - if (hws_oom) { - unregister_oom_notifier(&hws_oom_notifier); - hws_oom = 0; - } - hws_state = HWS_STOPPED; - mutex_unlock(&hws_sem); - - return rc; -} diff --git a/arch/s390/oprofile/hwsampler.h b/arch/s390/oprofile/hwsampler.h deleted file mode 100644 index a483d06f2fa7..000000000000 --- a/arch/s390/oprofile/hwsampler.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * CPUMF HW sampler functions and internal structures - * - * Copyright IBM Corp. 2010 - * Author(s): Heinz Graalfs - */ - -#ifndef HWSAMPLER_H_ -#define HWSAMPLER_H_ - -#include -#include - -struct hws_ssctl_request_block /* SET SAMPLING CONTROLS req block */ -{ /* bytes 0 - 7 Bit(s) */ - unsigned int s:1; /* 0: maximum buffer indicator */ - unsigned int h:1; /* 1: part. level reserved for VM use*/ - unsigned long b2_53:52; /* 2-53: zeros */ - unsigned int es:1; /* 54: sampling enable control */ - unsigned int b55_61:7; /* 55-61: - zeros */ - unsigned int cs:1; /* 62: sampling activation control */ - unsigned int b63:1; /* 63: zero */ - unsigned long interval; /* 8-15: sampling interval */ - unsigned long tear; /* 16-23: TEAR contents */ - unsigned long dear; /* 24-31: DEAR contents */ - /* 32-63: */ - unsigned long rsvrd1; /* reserved */ - unsigned long rsvrd2; /* reserved */ - unsigned long rsvrd3; /* reserved */ - unsigned long rsvrd4; /* reserved */ -}; - -struct hws_cpu_buffer { - unsigned long first_sdbt; /* @ of 1st SDB-Table for this CP*/ - unsigned long worker_entry; - unsigned long sample_overflow; /* taken from SDB ... */ - struct hws_qsi_info_block qsi; - struct hws_ssctl_request_block ssctl; - struct work_struct worker; - atomic_t ext_params; - unsigned long req_alert; - unsigned long loss_of_sample_data; - unsigned long invalid_entry_address; - unsigned long incorrect_sdbt_entry; - unsigned long sample_auth_change_alert; - unsigned int finish:1; - unsigned int oom:1; - unsigned int stop_mode:1; -}; - -int hwsampler_setup(void); -int hwsampler_shutdown(void); -int hwsampler_allocate(unsigned long sdbt, unsigned long sdb); -int hwsampler_deallocate(void); -unsigned long hwsampler_query_min_interval(void); -unsigned long hwsampler_query_max_interval(void); -int hwsampler_start_all(unsigned long interval); -int hwsampler_stop_all(void); -int hwsampler_deactivate(unsigned int cpu); -int hwsampler_activate(unsigned int cpu); -unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu); - -#endif /*HWSAMPLER_H_*/ diff --git a/arch/s390/oprofile/init.c b/arch/s390/oprofile/init.c index 791935a65800..16f4c3960b87 100644 --- a/arch/s390/oprofile/init.c +++ b/arch/s390/oprofile/init.c @@ -10,488 +10,8 @@ */ #include -#include #include -#include -#include -#include #include -#include - -#include "../../../drivers/oprofile/oprof.h" - -#include "hwsampler.h" -#include "op_counter.h" - -#define DEFAULT_INTERVAL 4127518 - -#define DEFAULT_SDBT_BLOCKS 1 -#define DEFAULT_SDB_BLOCKS 511 - -static unsigned long oprofile_hw_interval = DEFAULT_INTERVAL; -static unsigned long oprofile_min_interval; -static unsigned long oprofile_max_interval; - -static unsigned long oprofile_sdbt_blocks = DEFAULT_SDBT_BLOCKS; -static unsigned long oprofile_sdb_blocks = DEFAULT_SDB_BLOCKS; - -static int hwsampler_enabled; -static int hwsampler_running; /* start_mutex must be held to change */ -static int hwsampler_available; - -static struct oprofile_operations timer_ops; - -struct op_counter_config counter_config; - -enum __force_cpu_type { - reserved = 0, /* do not force */ - timer, -}; -static int force_cpu_type; - -static int set_cpu_type(const char *str, struct kernel_param *kp) -{ - if (!strcmp(str, "timer")) { - force_cpu_type = timer; - printk(KERN_INFO "oprofile: forcing timer to be returned " - "as cpu type\n"); - } else { - force_cpu_type = 0; - } - - return 0; -} -module_param_call(cpu_type, set_cpu_type, NULL, NULL, 0); -MODULE_PARM_DESC(cpu_type, "Force legacy basic mode sampling" - "(report cpu_type \"timer\""); - -static int __oprofile_hwsampler_start(void) -{ - int retval; - - retval = hwsampler_allocate(oprofile_sdbt_blocks, oprofile_sdb_blocks); - if (retval) - return retval; - - retval = hwsampler_start_all(oprofile_hw_interval); - if (retval) - hwsampler_deallocate(); - - return retval; -} - -static int oprofile_hwsampler_start(void) -{ - int retval; - - hwsampler_running = hwsampler_enabled; - - if (!hwsampler_running) - return timer_ops.start(); - - retval = perf_reserve_sampling(); - if (retval) - return retval; - - retval = __oprofile_hwsampler_start(); - if (retval) - perf_release_sampling(); - - return retval; -} - -static void oprofile_hwsampler_stop(void) -{ - if (!hwsampler_running) { - timer_ops.stop(); - return; - } - - hwsampler_stop_all(); - hwsampler_deallocate(); - perf_release_sampling(); - return; -} - -/* - * File ops used for: - * /dev/oprofile/0/enabled - * /dev/oprofile/hwsampling/hwsampler (cpu_type = timer) - */ - -static ssize_t hwsampler_read(struct file *file, char __user *buf, - size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(hwsampler_enabled, buf, count, offset); -} - -static ssize_t hwsampler_write(struct file *file, char const __user *buf, - size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - - if (val != 0 && val != 1) - return -EINVAL; - - if (oprofile_started) - /* - * save to do without locking as we set - * hwsampler_running in start() when start_mutex is - * held - */ - return -EBUSY; - - hwsampler_enabled = val; - - return count; -} - -static const struct file_operations hwsampler_fops = { - .read = hwsampler_read, - .write = hwsampler_write, -}; - -/* - * File ops used for: - * /dev/oprofile/0/count - * /dev/oprofile/hwsampling/hw_interval (cpu_type = timer) - * - * Make sure that the value is within the hardware range. - */ - -static ssize_t hw_interval_read(struct file *file, char __user *buf, - size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(oprofile_hw_interval, buf, - count, offset); -} - -static ssize_t hw_interval_write(struct file *file, char const __user *buf, - size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - if (val < oprofile_min_interval) - oprofile_hw_interval = oprofile_min_interval; - else if (val > oprofile_max_interval) - oprofile_hw_interval = oprofile_max_interval; - else - oprofile_hw_interval = val; - - return count; -} - -static const struct file_operations hw_interval_fops = { - .read = hw_interval_read, - .write = hw_interval_write, -}; - -/* - * File ops used for: - * /dev/oprofile/0/event - * Only a single event with number 0 is supported with this counter. - * - * /dev/oprofile/0/unit_mask - * This is a dummy file needed by the user space tools. - * No value other than 0 is accepted or returned. - */ - -static ssize_t hwsampler_zero_read(struct file *file, char __user *buf, - size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(0, buf, count, offset); -} - -static ssize_t hwsampler_zero_write(struct file *file, char const __user *buf, - size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - if (val != 0) - return -EINVAL; - return count; -} - -static const struct file_operations zero_fops = { - .read = hwsampler_zero_read, - .write = hwsampler_zero_write, -}; - -/* /dev/oprofile/0/kernel file ops. */ - -static ssize_t hwsampler_kernel_read(struct file *file, char __user *buf, - size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(counter_config.kernel, - buf, count, offset); -} - -static ssize_t hwsampler_kernel_write(struct file *file, char const __user *buf, - size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - - if (val != 0 && val != 1) - return -EINVAL; - - counter_config.kernel = val; - - return count; -} - -static const struct file_operations kernel_fops = { - .read = hwsampler_kernel_read, - .write = hwsampler_kernel_write, -}; - -/* /dev/oprofile/0/user file ops. */ - -static ssize_t hwsampler_user_read(struct file *file, char __user *buf, - size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(counter_config.user, - buf, count, offset); -} - -static ssize_t hwsampler_user_write(struct file *file, char const __user *buf, - size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - - if (val != 0 && val != 1) - return -EINVAL; - - counter_config.user = val; - - return count; -} - -static const struct file_operations user_fops = { - .read = hwsampler_user_read, - .write = hwsampler_user_write, -}; - - -/* - * File ops used for: /dev/oprofile/timer/enabled - * The value always has to be the inverted value of hwsampler_enabled. So - * no separate variable is created. That way we do not need locking. - */ - -static ssize_t timer_enabled_read(struct file *file, char __user *buf, - size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(!hwsampler_enabled, buf, count, offset); -} - -static ssize_t timer_enabled_write(struct file *file, char const __user *buf, - size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - - if (val != 0 && val != 1) - return -EINVAL; - - /* Timer cannot be disabled without having hardware sampling. */ - if (val == 0 && !hwsampler_available) - return -EINVAL; - - if (oprofile_started) - /* - * save to do without locking as we set - * hwsampler_running in start() when start_mutex is - * held - */ - return -EBUSY; - - hwsampler_enabled = !val; - - return count; -} - -static const struct file_operations timer_enabled_fops = { - .read = timer_enabled_read, - .write = timer_enabled_write, -}; - - -static int oprofile_create_hwsampling_files(struct dentry *root) -{ - struct dentry *dir; - - dir = oprofilefs_mkdir(root, "timer"); - if (!dir) - return -EINVAL; - - oprofilefs_create_file(dir, "enabled", &timer_enabled_fops); - - if (!hwsampler_available) - return 0; - - /* reinitialize default values */ - hwsampler_enabled = 1; - counter_config.kernel = 1; - counter_config.user = 1; - - if (!force_cpu_type) { - /* - * Create the counter file system. A single virtual - * counter is created which can be used to - * enable/disable hardware sampling dynamically from - * user space. The user space will configure a single - * counter with a single event. The value of 'event' - * and 'unit_mask' are not evaluated by the kernel code - * and can only be set to 0. - */ - - dir = oprofilefs_mkdir(root, "0"); - if (!dir) - return -EINVAL; - - oprofilefs_create_file(dir, "enabled", &hwsampler_fops); - oprofilefs_create_file(dir, "event", &zero_fops); - oprofilefs_create_file(dir, "count", &hw_interval_fops); - oprofilefs_create_file(dir, "unit_mask", &zero_fops); - oprofilefs_create_file(dir, "kernel", &kernel_fops); - oprofilefs_create_file(dir, "user", &user_fops); - oprofilefs_create_ulong(dir, "hw_sdbt_blocks", - &oprofile_sdbt_blocks); - - } else { - /* - * Hardware sampling can be used but the cpu_type is - * forced to timer in order to deal with legacy user - * space tools. The /dev/oprofile/hwsampling fs is - * provided in that case. - */ - dir = oprofilefs_mkdir(root, "hwsampling"); - if (!dir) - return -EINVAL; - - oprofilefs_create_file(dir, "hwsampler", - &hwsampler_fops); - oprofilefs_create_file(dir, "hw_interval", - &hw_interval_fops); - oprofilefs_create_ro_ulong(dir, "hw_min_interval", - &oprofile_min_interval); - oprofilefs_create_ro_ulong(dir, "hw_max_interval", - &oprofile_max_interval); - oprofilefs_create_ulong(dir, "hw_sdbt_blocks", - &oprofile_sdbt_blocks); - } - return 0; -} - -static int oprofile_hwsampler_init(struct oprofile_operations *ops) -{ - /* - * Initialize the timer mode infrastructure as well in order - * to be able to switch back dynamically. oprofile_timer_init - * is not supposed to fail. - */ - if (oprofile_timer_init(ops)) - BUG(); - - memcpy(&timer_ops, ops, sizeof(timer_ops)); - ops->create_files = oprofile_create_hwsampling_files; - - /* - * If the user space tools do not support newer cpu types, - * the force_cpu_type module parameter - * can be used to always return \"timer\" as cpu type. - */ - if (force_cpu_type != timer) { - struct cpuid id; - - get_cpu_id (&id); - - switch (id.machine) { - case 0x2097: case 0x2098: ops->cpu_type = "s390/z10"; break; - case 0x2817: case 0x2818: ops->cpu_type = "s390/z196"; break; - case 0x2827: case 0x2828: ops->cpu_type = "s390/zEC12"; break; - case 0x2964: case 0x2965: ops->cpu_type = "s390/z13"; break; - default: return -ENODEV; - } - } - - if (hwsampler_setup()) - return -ENODEV; - - /* - * Query the range for the sampling interval from the - * hardware. - */ - oprofile_min_interval = hwsampler_query_min_interval(); - if (oprofile_min_interval == 0) - return -ENODEV; - oprofile_max_interval = hwsampler_query_max_interval(); - if (oprofile_max_interval == 0) - return -ENODEV; - - /* The initial value should be sane */ - if (oprofile_hw_interval < oprofile_min_interval) - oprofile_hw_interval = oprofile_min_interval; - if (oprofile_hw_interval > oprofile_max_interval) - oprofile_hw_interval = oprofile_max_interval; - - printk(KERN_INFO "oprofile: System z hardware sampling " - "facility found.\n"); - - ops->start = oprofile_hwsampler_start; - ops->stop = oprofile_hwsampler_stop; - - return 0; -} - -static void oprofile_hwsampler_exit(void) -{ - hwsampler_shutdown(); -} static int __s390_backtrace(void *data, unsigned long address) { @@ -514,18 +34,9 @@ static void s390_backtrace(struct pt_regs *regs, unsigned int depth) int __init oprofile_arch_init(struct oprofile_operations *ops) { ops->backtrace = s390_backtrace; - - /* - * -ENODEV is not reported to the caller. The module itself - * will use the timer mode sampling as fallback and this is - * always available. - */ - hwsampler_available = oprofile_hwsampler_init(ops) == 0; - return 0; } void oprofile_arch_exit(void) { - oprofile_hwsampler_exit(); } diff --git a/arch/s390/oprofile/op_counter.h b/arch/s390/oprofile/op_counter.h deleted file mode 100644 index 61b2531eef17..000000000000 --- a/arch/s390/oprofile/op_counter.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright IBM Corp. 2011 - * Author(s): Andreas Krebbel (krebbel@linux.vnet.ibm.com) - * - * @remark Copyright 2011 OProfile authors - */ - -#ifndef OP_COUNTER_H -#define OP_COUNTER_H - -struct op_counter_config { - /* `enabled' maps to the hwsampler_file variable. */ - /* `count' maps to the oprofile_hw_interval variable. */ - /* `event' and `unit_mask' are unused. */ - unsigned long kernel; - unsigned long user; -}; - -extern struct op_counter_config counter_config; - -#endif /* OP_COUNTER_H */ diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index 1ea8c07eab84..070f1ae5cfad 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -226,7 +226,8 @@ static unsigned long __dma_alloc_iommu(struct device *dev, boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, PAGE_SIZE) >> PAGE_SHIFT; return iommu_area_alloc(zdev->iommu_bitmap, zdev->iommu_pages, - start, size, 0, boundary_size, 0); + start, size, zdev->start_dma >> PAGE_SHIFT, + boundary_size, 0); } static unsigned long dma_alloc_iommu(struct device *dev, int size) @@ -469,6 +470,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev) * Also set zdev->end_dma to the actual end address of the usable * range, instead of the theoretical maximum as reported by hardware. */ + zdev->start_dma = PAGE_ALIGN(zdev->start_dma); zdev->iommu_size = min3((u64) high_memory, ZPCI_TABLE_SIZE_RT - zdev->start_dma, zdev->end_dma - zdev->start_dma + 1); diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index fb2a9a560fdc..c2b27ad8e94d 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -145,8 +145,7 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) default: break; } - if (pdev) - pci_dev_put(pdev); + pci_dev_put(pdev); } void zpci_event_availability(void *data) diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index 10ca15dcab11..fa8d7d4b9751 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -99,7 +99,7 @@ void zpci_set_irq_ctrl(u16 ctl, char *unused, u8 isc) } /* PCI Load */ -static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) +static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status) { register u64 __req asm("2") = req; register u64 __offset asm("3") = offset; @@ -116,6 +116,16 @@ static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) : "d" (__offset) : "cc"); *status = __req >> 24 & 0xff; + *data = __data; + return cc; +} + +static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) +{ + u64 __data; + int cc; + + cc = ____pcilg(&__data, req, offset, status); if (!cc) *data = __data; diff --git a/arch/score/include/asm/pgalloc.h b/arch/score/include/asm/pgalloc.h index 2e067657db98..49b012d78c1a 100644 --- a/arch/score/include/asm/pgalloc.h +++ b/arch/score/include/asm/pgalloc.h @@ -42,8 +42,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, { pte_t *pte; - pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, - PTE_ORDER); + pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_ZERO, PTE_ORDER); return pte; } @@ -53,7 +52,7 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, { struct page *pte; - pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER); + pte = alloc_pages(GFP_KERNEL, PTE_ORDER); if (!pte) return NULL; clear_highpage(pte); diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c index 37a6c2e0e969..995b71e4db4b 100644 --- a/arch/score/mm/fault.c +++ b/arch/score/mm/fault.c @@ -111,7 +111,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index e803a836cb7c..0d5f3a9bb315 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -264,7 +264,6 @@ config CPU_SUBTYPE_SH7203 select CPU_HAS_FPU select SYS_SUPPORTS_SH_CMT select SYS_SUPPORTS_SH_MTU2 - select ARCH_WANT_OPTIONAL_GPIOLIB select PINCTRL config CPU_SUBTYPE_SH7206 @@ -353,7 +352,6 @@ config CPU_SUBTYPE_SH7720 select CPU_SH3 select CPU_HAS_DSP select SYS_SUPPORTS_SH_CMT - select ARCH_WANT_OPTIONAL_GPIOLIB select USB_OHCI_SH if USB_OHCI_HCD select PINCTRL help @@ -419,7 +417,6 @@ config CPU_SUBTYPE_SH7723 select ARCH_SHMOBILE select ARCH_SPARSEMEM_ENABLE select SYS_SUPPORTS_SH_CMT - select ARCH_WANT_OPTIONAL_GPIOLIB select PINCTRL help Select SH7723 if you have an SH-MobileR2 CPU. @@ -431,7 +428,6 @@ config CPU_SUBTYPE_SH7724 select ARCH_SHMOBILE select ARCH_SPARSEMEM_ENABLE select SYS_SUPPORTS_SH_CMT - select ARCH_WANT_OPTIONAL_GPIOLIB select PINCTRL help Select SH7724 if you have an SH-MobileR2R CPU. @@ -440,7 +436,6 @@ config CPU_SUBTYPE_SH7734 bool "Support SH7734 processor" select CPU_SH4A select CPU_SHX2 - select ARCH_WANT_OPTIONAL_GPIOLIB select PINCTRL help Select SH7734 if you have a SH4A SH7734 CPU. @@ -449,7 +444,6 @@ config CPU_SUBTYPE_SH7757 bool "Support SH7757 processor" select CPU_SH4A select CPU_SHX2 - select ARCH_WANT_OPTIONAL_GPIOLIB select PINCTRL help Select SH7757 if you have a SH4A SH7757 CPU. @@ -475,7 +469,6 @@ config CPU_SUBTYPE_SH7785 select CPU_SHX2 select ARCH_SPARSEMEM_ENABLE select SYS_SUPPORTS_NUMA - select ARCH_WANT_OPTIONAL_GPIOLIB select PINCTRL config CPU_SUBTYPE_SH7786 @@ -484,7 +477,6 @@ config CPU_SUBTYPE_SH7786 select CPU_SHX3 select CPU_HAS_PTEAEX select GENERIC_CLOCKEVENTS_BROADCAST if SMP - select ARCH_WANT_OPTIONAL_GPIOLIB select USB_OHCI_SH if USB_OHCI_HCD select USB_EHCI_SH if USB_EHCI_HCD select PINCTRL @@ -494,7 +486,7 @@ config CPU_SUBTYPE_SHX3 select CPU_SH4A select CPU_SHX3 select GENERIC_CLOCKEVENTS_BROADCAST if SMP - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select PINCTRL # SH4AL-DSP Processor Support @@ -513,7 +505,6 @@ config CPU_SUBTYPE_SH7722 select ARCH_SPARSEMEM_ENABLE select SYS_SUPPORTS_NUMA select SYS_SUPPORTS_SH_CMT - select ARCH_WANT_OPTIONAL_GPIOLIB select PINCTRL config CPU_SUBTYPE_SH7366 diff --git a/arch/sh/boards/Kconfig b/arch/sh/boards/Kconfig index 5e52d5362292..e0db04664e2e 100644 --- a/arch/sh/boards/Kconfig +++ b/arch/sh/boards/Kconfig @@ -70,7 +70,7 @@ config SH_7724_SOLUTION_ENGINE bool "SolutionEngine7724" select SOLUTION_ENGINE depends on CPU_SUBTYPE_SH7724 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select SND_SOC_AK4642 if SND_SIMPLE_CARD select REGULATOR_FIXED_VOLTAGE if REGULATOR help @@ -174,7 +174,6 @@ config SH_SDK7786 depends on CPU_SUBTYPE_SH7786 select SYS_SUPPORTS_PCI select NO_IOPORT_MAP if !PCI - select ARCH_WANT_OPTIONAL_GPIOLIB select HAVE_SRAM_POOL select REGULATOR_FIXED_VOLTAGE if REGULATOR help @@ -190,7 +189,7 @@ config SH_HIGHLANDER config SH_SH7757LCR bool "SH7757LCR" depends on CPU_SUBTYPE_SH7757 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select REGULATOR_FIXED_VOLTAGE if REGULATOR config SH_SH7785LCR @@ -217,14 +216,14 @@ config SH_SH7785LCR_PT config SH_URQUELL bool "Urquell" depends on CPU_SUBTYPE_SH7786 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select SYS_SUPPORTS_PCI select NO_IOPORT_MAP if !PCI config SH_MIGOR bool "Migo-R" depends on CPU_SUBTYPE_SH7722 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select REGULATOR_FIXED_VOLTAGE if REGULATOR help Select Migo-R if configuring for the SH7722 Migo-R platform @@ -233,7 +232,7 @@ config SH_MIGOR config SH_AP325RXA bool "AP-325RXA" depends on CPU_SUBTYPE_SH7723 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select REGULATOR_FIXED_VOLTAGE if REGULATOR help Renesas "AP-325RXA" support. @@ -242,7 +241,7 @@ config SH_AP325RXA config SH_KFR2R09 bool "KFR2R09" depends on CPU_SUBTYPE_SH7724 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select REGULATOR_FIXED_VOLTAGE if REGULATOR help "Kit For R2R for 2009" support. @@ -250,7 +249,7 @@ config SH_KFR2R09 config SH_ECOVEC bool "EcoVec" depends on CPU_SUBTYPE_SH7724 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select SND_SOC_DA7210 if SND_SIMPLE_CARD select REGULATOR_FIXED_VOLTAGE if REGULATOR help @@ -327,7 +326,7 @@ config SH_X3PROTO config SH_MAGIC_PANEL_R2 bool "Magic Panel R2" depends on CPU_SUBTYPE_SH7720 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB select REGULATOR_FIXED_VOLTAGE if REGULATOR help Select Magic Panel R2 if configuring for Magic Panel R2. diff --git a/arch/sh/boards/mach-highlander/Kconfig b/arch/sh/boards/mach-highlander/Kconfig index def49cc0a7b9..42f5589b4bf3 100644 --- a/arch/sh/boards/mach-highlander/Kconfig +++ b/arch/sh/boards/mach-highlander/Kconfig @@ -18,7 +18,7 @@ config SH_R7780MP config SH_R7785RP bool "R7785RP board support" depends on CPU_SUBTYPE_SH7785 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB endchoice diff --git a/arch/sh/boards/mach-rsk/Kconfig b/arch/sh/boards/mach-rsk/Kconfig index 458a11ffd022..0b9b2c4952c1 100644 --- a/arch/sh/boards/mach-rsk/Kconfig +++ b/arch/sh/boards/mach-rsk/Kconfig @@ -10,17 +10,17 @@ config SH_RSK7201 config SH_RSK7203 bool "RSK7203" - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB depends on CPU_SUBTYPE_SH7203 config SH_RSK7264 bool "RSK2+SH7264" - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB depends on CPU_SUBTYPE_SH7264 config SH_RSK7269 bool "RSK2+SH7269" - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB depends on CPU_SUBTYPE_SH7269 endchoice diff --git a/arch/sh/include/asm/atomic-grb.h b/arch/sh/include/asm/atomic-grb.h index b94df40e5f2d..d755e96c3064 100644 --- a/arch/sh/include/asm/atomic-grb.h +++ b/arch/sh/include/asm/atomic-grb.h @@ -43,16 +43,42 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return tmp; \ } -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int res, tmp; \ + \ + __asm__ __volatile__ ( \ + " .align 2 \n\t" \ + " mova 1f, r0 \n\t" /* r0 = end point */ \ + " mov r15, r1 \n\t" /* r1 = saved sp */ \ + " mov #-6, r15 \n\t" /* LOGIN: r15 = size */ \ + " mov.l @%2, %0 \n\t" /* load old value */ \ + " mov %0, %1 \n\t" /* save old value */ \ + " " #op " %3, %0 \n\t" /* $op */ \ + " mov.l %0, @%2 \n\t" /* store new value */ \ + "1: mov r1, r15 \n\t" /* LOGOUT */ \ + : "=&r" (tmp), "=&r" (res), "+r" (v) \ + : "r" (i) \ + : "memory" , "r0", "r1"); \ + \ + return res; \ +} + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/sh/include/asm/atomic-irq.h b/arch/sh/include/asm/atomic-irq.h index 23fcdad5773e..8e2da5fa0178 100644 --- a/arch/sh/include/asm/atomic-irq.h +++ b/arch/sh/include/asm/atomic-irq.h @@ -33,15 +33,38 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return temp; \ } -#define ATOMIC_OPS(op, c_op) ATOMIC_OP(op, c_op) ATOMIC_OP_RETURN(op, c_op) +#define ATOMIC_FETCH_OP(op, c_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned long temp, flags; \ + \ + raw_local_irq_save(flags); \ + temp = v->counter; \ + v->counter c_op i; \ + raw_local_irq_restore(flags); \ + \ + return temp; \ +} + +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_OP_RETURN(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) -ATOMIC_OP(and, &=) -ATOMIC_OP(or, |=) -ATOMIC_OP(xor, ^=) #undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) + +ATOMIC_OPS(and, &=) +ATOMIC_OPS(or, |=) +ATOMIC_OPS(xor, ^=) + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/sh/include/asm/atomic-llsc.h b/arch/sh/include/asm/atomic-llsc.h index 33d34b16d4d6..caea2c45f6c2 100644 --- a/arch/sh/include/asm/atomic-llsc.h +++ b/arch/sh/include/asm/atomic-llsc.h @@ -48,15 +48,39 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return temp; \ } -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned long res, temp; \ + \ + __asm__ __volatile__ ( \ +"1: movli.l @%3, %0 ! atomic_fetch_" #op " \n" \ +" mov %0, %1 \n" \ +" " #op " %2, %0 \n" \ +" movco.l %0, @%3 \n" \ +" bf 1b \n" \ +" synco \n" \ + : "=&z" (temp), "=&z" (res) \ + : "r" (i), "r" (&v->counter) \ + : "t"); \ + \ + return res; \ +} + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) #undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h index a33673b3687d..f3f42c84c40f 100644 --- a/arch/sh/include/asm/pgalloc.h +++ b/arch/sh/include/asm/pgalloc.h @@ -34,7 +34,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - return quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL); + return quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm, @@ -43,7 +43,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, struct page *page; void *pg; - pg = quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL); + pg = quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL); if (!pg) return NULL; page = virt_to_page(pg); diff --git a/arch/sh/include/asm/spinlock.h b/arch/sh/include/asm/spinlock.h index bdc0f3b6c56a..416834b60ad0 100644 --- a/arch/sh/include/asm/spinlock.h +++ b/arch/sh/include/asm/spinlock.h @@ -19,14 +19,20 @@ #error "Need movli.l/movco.l for spinlocks" #endif +#include +#include + /* * Your basic SMP spinlocks, allowing only a single CPU anywhere */ #define arch_spin_is_locked(x) ((x)->lock <= 0) #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) -#define arch_spin_unlock_wait(x) \ - do { while (arch_spin_is_locked(x)) cpu_relax(); } while (0) + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->lock, VAL > 0); +} /* * Simple spin lock operations. There are two variants, one clears IRQ's diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h index 62f80d2a9df9..025cdb1032f6 100644 --- a/arch/sh/include/asm/tlb.h +++ b/arch/sh/include/asm/tlb.h @@ -101,7 +101,7 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) { free_page_and_swap_cache(page); - return 1; /* avoid calling tlb_flush_mmu */ + return false; /* avoid calling tlb_flush_mmu */ } static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) @@ -109,6 +109,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) __tlb_remove_page(tlb, page); } +static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return __tlb_remove_page(tlb, page); +} + +static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, + struct page *page) +{ + return __tlb_remove_page(tlb, page); +} + +static inline void tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return tlb_remove_page(tlb, page); +} + #define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep) #define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) #define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp) diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 79d8276377d1..9bf876780cef 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -487,7 +487,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR))) if (mm_fault_error(regs, error_code, address, fault)) diff --git a/arch/sh/mm/pgtable.c b/arch/sh/mm/pgtable.c index 26e03a1f7ca4..a62bd8696779 100644 --- a/arch/sh/mm/pgtable.c +++ b/arch/sh/mm/pgtable.c @@ -1,7 +1,7 @@ #include #include -#define PGALLOC_GFP GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO +#define PGALLOC_GFP GFP_KERNEL | __GFP_ZERO static struct kmem_cache *pgd_cachep; #if PAGETABLE_LEVELS > 2 diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index 7dcbebbcaec6..ee3f11c43cda 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -20,9 +20,10 @@ #define ATOMIC_INIT(i) { (i) } int atomic_add_return(int, atomic_t *); -void atomic_and(int, atomic_t *); -void atomic_or(int, atomic_t *); -void atomic_xor(int, atomic_t *); +int atomic_fetch_add(int, atomic_t *); +int atomic_fetch_and(int, atomic_t *); +int atomic_fetch_or(int, atomic_t *); +int atomic_fetch_xor(int, atomic_t *); int atomic_cmpxchg(atomic_t *, int, int); int atomic_xchg(atomic_t *, int); int __atomic_add_unless(atomic_t *, int, int); @@ -35,7 +36,13 @@ void atomic_set(atomic_t *, int); #define atomic_inc(v) ((void)atomic_add_return( 1, (v))) #define atomic_dec(v) ((void)atomic_add_return( -1, (v))) +#define atomic_and(i, v) ((void)atomic_fetch_and((i), (v))) +#define atomic_or(i, v) ((void)atomic_fetch_or((i), (v))) +#define atomic_xor(i, v) ((void)atomic_fetch_xor((i), (v))) + #define atomic_sub_return(i, v) (atomic_add_return(-(int)(i), (v))) +#define atomic_fetch_sub(i, v) (atomic_fetch_add (-(int)(i), (v))) + #define atomic_inc_return(v) (atomic_add_return( 1, (v))) #define atomic_dec_return(v) (atomic_add_return( -1, (v))) diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index f2fbf9e16faf..24827a3f733a 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -28,16 +28,24 @@ void atomic64_##op(long, atomic64_t *); int atomic_##op##_return(int, atomic_t *); \ long atomic64_##op##_return(long, atomic64_t *); -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +int atomic_fetch_##op(int, atomic_t *); \ +long atomic64_fetch_##op(long, atomic64_t *); + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index 5e3187185b4a..3529f1378cd8 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -41,8 +41,7 @@ static inline void __pud_populate(pud_t *pud, pmd_t *pmd) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(pgtable_cache, - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) @@ -52,8 +51,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(pgtable_cache, - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) diff --git a/arch/sparc/include/asm/spinlock_32.h b/arch/sparc/include/asm/spinlock_32.h index bcc98fc35281..d9c5876c6121 100644 --- a/arch/sparc/include/asm/spinlock_32.h +++ b/arch/sparc/include/asm/spinlock_32.h @@ -9,12 +9,15 @@ #ifndef __ASSEMBLY__ #include +#include #include /* for cpu_relax */ #define arch_spin_is_locked(lock) (*((volatile unsigned char *)(lock)) != 0) -#define arch_spin_unlock_wait(lock) \ - do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->lock, !VAL); +} static inline void arch_spin_lock(arch_spinlock_t *lock) { diff --git a/arch/sparc/include/asm/spinlock_64.h b/arch/sparc/include/asm/spinlock_64.h index 968917694978..87990b7c6b0d 100644 --- a/arch/sparc/include/asm/spinlock_64.h +++ b/arch/sparc/include/asm/spinlock_64.h @@ -8,6 +8,9 @@ #ifndef __ASSEMBLY__ +#include +#include + /* To get debugging spinlocks which detect and catch * deadlock situations, set CONFIG_DEBUG_SPINLOCK * and rebuild your kernel. @@ -23,9 +26,10 @@ #define arch_spin_is_locked(lp) ((lp)->lock != 0) -#define arch_spin_unlock_wait(lp) \ - do { rmb(); \ - } while((lp)->lock) +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->lock, !VAL); +} static inline void arch_spin_lock(arch_spinlock_t *lock) { diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index e22416ce56ea..34a7930b76ef 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -242,7 +242,7 @@ unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino) { int irq; - irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL); + irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL, NULL); if (irq <= 0) goto out; diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index b9d63c0a7aab..2c373329d5cb 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -27,39 +27,44 @@ static DEFINE_SPINLOCK(dummy); #endif /* SMP */ -#define ATOMIC_OP_RETURN(op, c_op) \ -int atomic_##op##_return(int i, atomic_t *v) \ +#define ATOMIC_FETCH_OP(op, c_op) \ +int atomic_fetch_##op(int i, atomic_t *v) \ { \ int ret; \ unsigned long flags; \ spin_lock_irqsave(ATOMIC_HASH(v), flags); \ \ - ret = (v->counter c_op i); \ + ret = v->counter; \ + v->counter c_op i; \ \ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ return ret; \ } \ -EXPORT_SYMBOL(atomic_##op##_return); +EXPORT_SYMBOL(atomic_fetch_##op); -#define ATOMIC_OP(op, c_op) \ -void atomic_##op(int i, atomic_t *v) \ +#define ATOMIC_OP_RETURN(op, c_op) \ +int atomic_##op##_return(int i, atomic_t *v) \ { \ + int ret; \ unsigned long flags; \ spin_lock_irqsave(ATOMIC_HASH(v), flags); \ \ - v->counter c_op i; \ + ret = (v->counter c_op i); \ \ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ + return ret; \ } \ -EXPORT_SYMBOL(atomic_##op); +EXPORT_SYMBOL(atomic_##op##_return); ATOMIC_OP_RETURN(add, +=) -ATOMIC_OP(and, &=) -ATOMIC_OP(or, |=) -ATOMIC_OP(xor, ^=) +ATOMIC_FETCH_OP(add, +=) +ATOMIC_FETCH_OP(and, &=) +ATOMIC_FETCH_OP(or, |=) +ATOMIC_FETCH_OP(xor, ^=) + +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN -#undef ATOMIC_OP int atomic_xchg(atomic_t *v, int new) { diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S index d6b0363f345b..a5c5a0279ccc 100644 --- a/arch/sparc/lib/atomic_64.S +++ b/arch/sparc/lib/atomic_64.S @@ -9,10 +9,11 @@ .text - /* Two versions of the atomic routines, one that + /* Three versions of the atomic routines, one that * does not return a value and does not perform - * memory barriers, and a second which returns - * a value and does the barriers. + * memory barriers, and a two which return + * a value, the new and old value resp. and does the + * barriers. */ #define ATOMIC_OP(op) \ @@ -43,15 +44,34 @@ ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ ENDPROC(atomic_##op##_return); -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: lduw [%o1], %g1; \ + op %g1, %o0, %g7; \ + cas [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + sra %g1, 0, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(atomic_fetch_##op); + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) #undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -83,15 +103,34 @@ ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ ENDPROC(atomic64_##op##_return); -#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) +#define ATOMIC64_FETCH_OP(op) \ +ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: ldx [%o1], %g1; \ + op %g1, %o0, %g7; \ + casx [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + mov %g1, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(atomic64_fetch_##op); + +#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op) ATOMIC64_OPS(add) ATOMIC64_OPS(sub) -ATOMIC64_OP(and) -ATOMIC64_OP(or) -ATOMIC64_OP(xor) #undef ATOMIC64_OPS +#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_FETCH_OP(op) + +ATOMIC64_OPS(and) +ATOMIC64_OPS(or) +ATOMIC64_OPS(xor) + +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP diff --git a/arch/sparc/lib/ksyms.c b/arch/sparc/lib/ksyms.c index 8eb454cfe05c..de5e97817bdb 100644 --- a/arch/sparc/lib/ksyms.c +++ b/arch/sparc/lib/ksyms.c @@ -107,15 +107,24 @@ EXPORT_SYMBOL(atomic64_##op); EXPORT_SYMBOL(atomic_##op##_return); \ EXPORT_SYMBOL(atomic64_##op##_return); -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +EXPORT_SYMBOL(atomic_fetch_##op); \ +EXPORT_SYMBOL(atomic64_fetch_##op); + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) #undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index b6c559cbd64d..4714061d6cd3 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -241,7 +241,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; @@ -411,7 +411,7 @@ good_area: if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - switch (handle_mm_fault(mm, vma, address, flags)) { + switch (handle_mm_fault(vma, address, flags)) { case VM_FAULT_SIGBUS: case VM_FAULT_OOM: goto do_sigbus; diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index cb841a33da59..6c43b924a7a2 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -436,7 +436,7 @@ good_area: goto bad_area; } - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) goto exit_exception; diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 14bb0d5ed3c6..aec508e37490 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -2704,8 +2704,7 @@ void __flush_tlb_all(void) pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | - __GFP_REPEAT | __GFP_ZERO); + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO); pte_t *pte = NULL; if (page) @@ -2717,8 +2716,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { - struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | - __GFP_REPEAT | __GFP_ZERO); + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO); if (!page) return NULL; if (!pgtable_page_ctor(page)) { diff --git a/arch/tile/include/asm/atomic.h b/arch/tile/include/asm/atomic.h index 9fc0107a9c5e..8dda3c8ff5ab 100644 --- a/arch/tile/include/asm/atomic.h +++ b/arch/tile/include/asm/atomic.h @@ -46,6 +46,8 @@ static inline int atomic_read(const atomic_t *v) */ #define atomic_sub_return(i, v) atomic_add_return((int)(-(i)), (v)) +#define atomic_fetch_sub(i, v) atomic_fetch_add(-(int)(i), (v)) + /** * atomic_sub - subtract integer from atomic variable * @i: integer value to subtract diff --git a/arch/tile/include/asm/atomic_32.h b/arch/tile/include/asm/atomic_32.h index d320ce253d86..a93774255136 100644 --- a/arch/tile/include/asm/atomic_32.h +++ b/arch/tile/include/asm/atomic_32.h @@ -34,18 +34,29 @@ static inline void atomic_add(int i, atomic_t *v) _atomic_xchg_add(&v->counter, i); } -#define ATOMIC_OP(op) \ -unsigned long _atomic_##op(volatile unsigned long *p, unsigned long mask); \ +#define ATOMIC_OPS(op) \ +unsigned long _atomic_fetch_##op(volatile unsigned long *p, unsigned long mask); \ static inline void atomic_##op(int i, atomic_t *v) \ { \ - _atomic_##op((unsigned long *)&v->counter, i); \ + _atomic_fetch_##op((unsigned long *)&v->counter, i); \ +} \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + smp_mb(); \ + return _atomic_fetch_##op((unsigned long *)&v->counter, i); \ } -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) -#undef ATOMIC_OP +#undef ATOMIC_OPS + +static inline int atomic_fetch_add(int i, atomic_t *v) +{ + smp_mb(); + return _atomic_xchg_add(&v->counter, i); +} /** * atomic_add_return - add integer and return @@ -126,16 +137,29 @@ static inline void atomic64_add(long long i, atomic64_t *v) _atomic64_xchg_add(&v->counter, i); } -#define ATOMIC64_OP(op) \ -long long _atomic64_##op(long long *v, long long n); \ +#define ATOMIC64_OPS(op) \ +long long _atomic64_fetch_##op(long long *v, long long n); \ static inline void atomic64_##op(long long i, atomic64_t *v) \ { \ - _atomic64_##op(&v->counter, i); \ + _atomic64_fetch_##op(&v->counter, i); \ +} \ +static inline long long atomic64_fetch_##op(long long i, atomic64_t *v) \ +{ \ + smp_mb(); \ + return _atomic64_fetch_##op(&v->counter, i); \ } -ATOMIC64_OP(and) -ATOMIC64_OP(or) -ATOMIC64_OP(xor) +ATOMIC64_OPS(and) +ATOMIC64_OPS(or) +ATOMIC64_OPS(xor) + +#undef ATOMIC64_OPS + +static inline long long atomic64_fetch_add(long long i, atomic64_t *v) +{ + smp_mb(); + return _atomic64_xchg_add(&v->counter, i); +} /** * atomic64_add_return - add integer and return @@ -186,6 +210,7 @@ static inline void atomic64_set(atomic64_t *v, long long n) #define atomic64_inc_return(v) atomic64_add_return(1LL, (v)) #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) #define atomic64_sub_return(i, v) atomic64_add_return(-(i), (v)) +#define atomic64_fetch_sub(i, v) atomic64_fetch_add(-(i), (v)) #define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) #define atomic64_sub(i, v) atomic64_add(-(i), (v)) #define atomic64_dec(v) atomic64_sub(1LL, (v)) @@ -193,7 +218,6 @@ static inline void atomic64_set(atomic64_t *v, long long n) #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL) - #endif /* !__ASSEMBLY__ */ /* @@ -242,16 +266,16 @@ struct __get_user { unsigned long val; int err; }; -extern struct __get_user __atomic_cmpxchg(volatile int *p, +extern struct __get_user __atomic32_cmpxchg(volatile int *p, int *lock, int o, int n); -extern struct __get_user __atomic_xchg(volatile int *p, int *lock, int n); -extern struct __get_user __atomic_xchg_add(volatile int *p, int *lock, int n); -extern struct __get_user __atomic_xchg_add_unless(volatile int *p, +extern struct __get_user __atomic32_xchg(volatile int *p, int *lock, int n); +extern struct __get_user __atomic32_xchg_add(volatile int *p, int *lock, int n); +extern struct __get_user __atomic32_xchg_add_unless(volatile int *p, int *lock, int o, int n); -extern struct __get_user __atomic_or(volatile int *p, int *lock, int n); -extern struct __get_user __atomic_and(volatile int *p, int *lock, int n); -extern struct __get_user __atomic_andn(volatile int *p, int *lock, int n); -extern struct __get_user __atomic_xor(volatile int *p, int *lock, int n); +extern struct __get_user __atomic32_fetch_or(volatile int *p, int *lock, int n); +extern struct __get_user __atomic32_fetch_and(volatile int *p, int *lock, int n); +extern struct __get_user __atomic32_fetch_andn(volatile int *p, int *lock, int n); +extern struct __get_user __atomic32_fetch_xor(volatile int *p, int *lock, int n); extern long long __atomic64_cmpxchg(volatile long long *p, int *lock, long long o, long long n); extern long long __atomic64_xchg(volatile long long *p, int *lock, long long n); @@ -259,9 +283,9 @@ extern long long __atomic64_xchg_add(volatile long long *p, int *lock, long long n); extern long long __atomic64_xchg_add_unless(volatile long long *p, int *lock, long long o, long long n); -extern long long __atomic64_and(volatile long long *p, int *lock, long long n); -extern long long __atomic64_or(volatile long long *p, int *lock, long long n); -extern long long __atomic64_xor(volatile long long *p, int *lock, long long n); +extern long long __atomic64_fetch_and(volatile long long *p, int *lock, long long n); +extern long long __atomic64_fetch_or(volatile long long *p, int *lock, long long n); +extern long long __atomic64_fetch_xor(volatile long long *p, int *lock, long long n); /* Return failure from the atomic wrappers. */ struct __get_user __atomic_bad_address(int __user *addr); diff --git a/arch/tile/include/asm/atomic_64.h b/arch/tile/include/asm/atomic_64.h index b0531a623653..4cefa0c9fd81 100644 --- a/arch/tile/include/asm/atomic_64.h +++ b/arch/tile/include/asm/atomic_64.h @@ -32,11 +32,6 @@ * on any routine which updates memory and returns a value. */ -static inline void atomic_add(int i, atomic_t *v) -{ - __insn_fetchadd4((void *)&v->counter, i); -} - /* * Note a subtlety of the locking here. We are required to provide a * full memory barrier before and after the operation. However, we @@ -59,28 +54,39 @@ static inline int atomic_add_return(int i, atomic_t *v) return val; } -static inline int __atomic_add_unless(atomic_t *v, int a, int u) +#define ATOMIC_OPS(op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int val; \ + smp_mb(); \ + val = __insn_fetch##op##4((void *)&v->counter, i); \ + smp_mb(); \ + return val; \ +} \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + __insn_fetch##op##4((void *)&v->counter, i); \ +} + +ATOMIC_OPS(add) +ATOMIC_OPS(and) +ATOMIC_OPS(or) + +#undef ATOMIC_OPS + +static inline int atomic_fetch_xor(int i, atomic_t *v) { int guess, oldval = v->counter; + smp_mb(); do { - if (oldval == u) - break; guess = oldval; - oldval = cmpxchg(&v->counter, guess, guess + a); + __insn_mtspr(SPR_CMPEXCH_VALUE, guess); + oldval = __insn_cmpexch4(&v->counter, guess ^ i); } while (guess != oldval); + smp_mb(); return oldval; } -static inline void atomic_and(int i, atomic_t *v) -{ - __insn_fetchand4((void *)&v->counter, i); -} - -static inline void atomic_or(int i, atomic_t *v) -{ - __insn_fetchor4((void *)&v->counter, i); -} - static inline void atomic_xor(int i, atomic_t *v) { int guess, oldval = v->counter; @@ -91,6 +97,18 @@ static inline void atomic_xor(int i, atomic_t *v) } while (guess != oldval); } +static inline int __atomic_add_unless(atomic_t *v, int a, int u) +{ + int guess, oldval = v->counter; + do { + if (oldval == u) + break; + guess = oldval; + oldval = cmpxchg(&v->counter, guess, guess + a); + } while (guess != oldval); + return oldval; +} + /* Now the true 64-bit operations. */ #define ATOMIC64_INIT(i) { (i) } @@ -98,11 +116,6 @@ static inline void atomic_xor(int i, atomic_t *v) #define atomic64_read(v) READ_ONCE((v)->counter) #define atomic64_set(v, i) WRITE_ONCE((v)->counter, (i)) -static inline void atomic64_add(long i, atomic64_t *v) -{ - __insn_fetchadd((void *)&v->counter, i); -} - static inline long atomic64_add_return(long i, atomic64_t *v) { int val; @@ -112,26 +125,37 @@ static inline long atomic64_add_return(long i, atomic64_t *v) return val; } -static inline long atomic64_add_unless(atomic64_t *v, long a, long u) +#define ATOMIC64_OPS(op) \ +static inline long atomic64_fetch_##op(long i, atomic64_t *v) \ +{ \ + long val; \ + smp_mb(); \ + val = __insn_fetch##op((void *)&v->counter, i); \ + smp_mb(); \ + return val; \ +} \ +static inline void atomic64_##op(long i, atomic64_t *v) \ +{ \ + __insn_fetch##op((void *)&v->counter, i); \ +} + +ATOMIC64_OPS(add) +ATOMIC64_OPS(and) +ATOMIC64_OPS(or) + +#undef ATOMIC64_OPS + +static inline long atomic64_fetch_xor(long i, atomic64_t *v) { long guess, oldval = v->counter; + smp_mb(); do { - if (oldval == u) - break; guess = oldval; - oldval = cmpxchg(&v->counter, guess, guess + a); + __insn_mtspr(SPR_CMPEXCH_VALUE, guess); + oldval = __insn_cmpexch(&v->counter, guess ^ i); } while (guess != oldval); - return oldval != u; -} - -static inline void atomic64_and(long i, atomic64_t *v) -{ - __insn_fetchand((void *)&v->counter, i); -} - -static inline void atomic64_or(long i, atomic64_t *v) -{ - __insn_fetchor((void *)&v->counter, i); + smp_mb(); + return oldval; } static inline void atomic64_xor(long i, atomic64_t *v) @@ -144,7 +168,20 @@ static inline void atomic64_xor(long i, atomic64_t *v) } while (guess != oldval); } +static inline long atomic64_add_unless(atomic64_t *v, long a, long u) +{ + long guess, oldval = v->counter; + do { + if (oldval == u) + break; + guess = oldval; + oldval = cmpxchg(&v->counter, guess, guess + a); + } while (guess != oldval); + return oldval != u; +} + #define atomic64_sub_return(i, v) atomic64_add_return(-(i), (v)) +#define atomic64_fetch_sub(i, v) atomic64_fetch_add(-(i), (v)) #define atomic64_sub(i, v) atomic64_add(-(i), (v)) #define atomic64_inc_return(v) atomic64_add_return(1, (v)) #define atomic64_dec_return(v) atomic64_sub_return(1, (v)) diff --git a/arch/tile/include/asm/barrier.h b/arch/tile/include/asm/barrier.h index d55222806c2f..4c419ab95ab7 100644 --- a/arch/tile/include/asm/barrier.h +++ b/arch/tile/include/asm/barrier.h @@ -87,6 +87,13 @@ mb_incoherent(void) #define __smp_mb__after_atomic() __smp_mb() #endif +/* + * The TILE architecture does not do speculative reads; this ensures + * that a control dependency also orders against loads and already provides + * a LOAD->{LOAD,STORE} order and can forgo the additional RMB. + */ +#define smp_acquire__after_ctrl_dep() barrier() + #include #endif /* !__ASSEMBLY__ */ diff --git a/arch/tile/include/asm/bitops_32.h b/arch/tile/include/asm/bitops_32.h index bbf7b666f21d..d1406a95f6b7 100644 --- a/arch/tile/include/asm/bitops_32.h +++ b/arch/tile/include/asm/bitops_32.h @@ -19,9 +19,9 @@ #include /* Tile-specific routines to support . */ -unsigned long _atomic_or(volatile unsigned long *p, unsigned long mask); -unsigned long _atomic_andn(volatile unsigned long *p, unsigned long mask); -unsigned long _atomic_xor(volatile unsigned long *p, unsigned long mask); +unsigned long _atomic_fetch_or(volatile unsigned long *p, unsigned long mask); +unsigned long _atomic_fetch_andn(volatile unsigned long *p, unsigned long mask); +unsigned long _atomic_fetch_xor(volatile unsigned long *p, unsigned long mask); /** * set_bit - Atomically set a bit in memory @@ -35,7 +35,7 @@ unsigned long _atomic_xor(volatile unsigned long *p, unsigned long mask); */ static inline void set_bit(unsigned nr, volatile unsigned long *addr) { - _atomic_or(addr + BIT_WORD(nr), BIT_MASK(nr)); + _atomic_fetch_or(addr + BIT_WORD(nr), BIT_MASK(nr)); } /** @@ -54,7 +54,7 @@ static inline void set_bit(unsigned nr, volatile unsigned long *addr) */ static inline void clear_bit(unsigned nr, volatile unsigned long *addr) { - _atomic_andn(addr + BIT_WORD(nr), BIT_MASK(nr)); + _atomic_fetch_andn(addr + BIT_WORD(nr), BIT_MASK(nr)); } /** @@ -69,7 +69,7 @@ static inline void clear_bit(unsigned nr, volatile unsigned long *addr) */ static inline void change_bit(unsigned nr, volatile unsigned long *addr) { - _atomic_xor(addr + BIT_WORD(nr), BIT_MASK(nr)); + _atomic_fetch_xor(addr + BIT_WORD(nr), BIT_MASK(nr)); } /** @@ -85,7 +85,7 @@ static inline int test_and_set_bit(unsigned nr, volatile unsigned long *addr) unsigned long mask = BIT_MASK(nr); addr += BIT_WORD(nr); smp_mb(); /* barrier for proper semantics */ - return (_atomic_or(addr, mask) & mask) != 0; + return (_atomic_fetch_or(addr, mask) & mask) != 0; } /** @@ -101,7 +101,7 @@ static inline int test_and_clear_bit(unsigned nr, volatile unsigned long *addr) unsigned long mask = BIT_MASK(nr); addr += BIT_WORD(nr); smp_mb(); /* barrier for proper semantics */ - return (_atomic_andn(addr, mask) & mask) != 0; + return (_atomic_fetch_andn(addr, mask) & mask) != 0; } /** @@ -118,7 +118,7 @@ static inline int test_and_change_bit(unsigned nr, unsigned long mask = BIT_MASK(nr); addr += BIT_WORD(nr); smp_mb(); /* barrier for proper semantics */ - return (_atomic_xor(addr, mask) & mask) != 0; + return (_atomic_fetch_xor(addr, mask) & mask) != 0; } #include diff --git a/arch/tile/include/asm/elf.h b/arch/tile/include/asm/elf.h index c505d77e4d06..e9d54a06736f 100644 --- a/arch/tile/include/asm/elf.h +++ b/arch/tile/include/asm/elf.h @@ -129,6 +129,7 @@ extern int dump_task_regs(struct task_struct *, elf_gregset_t *); struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, int executable_stack); +/* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */ #define ARCH_DLINFO \ do { \ NEW_AUX_ENT(AT_SYSINFO_EHDR, VDSO_BASE); \ diff --git a/arch/tile/include/asm/futex.h b/arch/tile/include/asm/futex.h index 1a6ef1b69cb1..e64a1b75fc38 100644 --- a/arch/tile/include/asm/futex.h +++ b/arch/tile/include/asm/futex.h @@ -80,16 +80,16 @@ ret = gu.err; \ } -#define __futex_set() __futex_call(__atomic_xchg) -#define __futex_add() __futex_call(__atomic_xchg_add) -#define __futex_or() __futex_call(__atomic_or) -#define __futex_andn() __futex_call(__atomic_andn) -#define __futex_xor() __futex_call(__atomic_xor) +#define __futex_set() __futex_call(__atomic32_xchg) +#define __futex_add() __futex_call(__atomic32_xchg_add) +#define __futex_or() __futex_call(__atomic32_fetch_or) +#define __futex_andn() __futex_call(__atomic32_fetch_andn) +#define __futex_xor() __futex_call(__atomic32_fetch_xor) #define __futex_cmpxchg() \ { \ - struct __get_user gu = __atomic_cmpxchg((u32 __force *)uaddr, \ - lock, oldval, oparg); \ + struct __get_user gu = __atomic32_cmpxchg((u32 __force *)uaddr, \ + lock, oldval, oparg); \ val = gu.val; \ ret = gu.err; \ } diff --git a/arch/tile/include/asm/setup.h b/arch/tile/include/asm/setup.h index e98909033e5b..2a0347af0702 100644 --- a/arch/tile/include/asm/setup.h +++ b/arch/tile/include/asm/setup.h @@ -25,7 +25,12 @@ #define MAXMEM_PFN PFN_DOWN(MAXMEM) int tile_console_write(const char *buf, int count); + +#ifdef CONFIG_EARLY_PRINTK void early_panic(const char *fmt, ...); +#else +#define early_panic panic +#endif /* Init-time routine to do tile-specific per-cpu setup. */ void setup_cpu(int boot); diff --git a/arch/tile/include/asm/thread_info.h b/arch/tile/include/asm/thread_info.h index 4b7cef9e94e0..c1467ac59ce6 100644 --- a/arch/tile/include/asm/thread_info.h +++ b/arch/tile/include/asm/thread_info.h @@ -78,7 +78,7 @@ struct thread_info { #ifndef __ASSEMBLY__ -void arch_release_thread_info(struct thread_info *info); +void arch_release_thread_stack(unsigned long *stack); /* How to get the thread information struct from C. */ register unsigned long stack_pointer __asm__("sp"); diff --git a/arch/tile/include/uapi/asm/auxvec.h b/arch/tile/include/uapi/asm/auxvec.h index c93e92709f14..f497123ed980 100644 --- a/arch/tile/include/uapi/asm/auxvec.h +++ b/arch/tile/include/uapi/asm/auxvec.h @@ -18,4 +18,6 @@ /* The vDSO location. */ #define AT_SYSINFO_EHDR 33 +#define AT_VECTOR_SIZE_ARCH 1 /* entries in ARCH_DLINFO */ + #endif /* _ASM_TILE_AUXVEC_H */ diff --git a/arch/tile/kernel/compat.c b/arch/tile/kernel/compat.c index 49120843ff96..bdaf71d31a4a 100644 --- a/arch/tile/kernel/compat.c +++ b/arch/tile/kernel/compat.c @@ -23,42 +23,50 @@ #include #include #include +#include /* * Syscalls that take 64-bit numbers traditionally take them in 32-bit * "high" and "low" value parts on 32-bit architectures. * In principle, one could imagine passing some register arguments as * fully 64-bit on TILE-Gx in 32-bit mode, but it seems easier to - * adapt the usual convention. + * adopt the usual convention. */ +#ifdef __BIG_ENDIAN +#define SYSCALL_PAIR(name) u32, name ## _hi, u32, name ## _lo +#else +#define SYSCALL_PAIR(name) u32, name ## _lo, u32, name ## _hi +#endif + COMPAT_SYSCALL_DEFINE4(truncate64, char __user *, filename, u32, dummy, - u32, low, u32, high) + SYSCALL_PAIR(length)) { - return sys_truncate(filename, ((loff_t)high << 32) | low); + return sys_truncate(filename, ((loff_t)length_hi << 32) | length_lo); } COMPAT_SYSCALL_DEFINE4(ftruncate64, unsigned int, fd, u32, dummy, - u32, low, u32, high) + SYSCALL_PAIR(length)) { - return sys_ftruncate(fd, ((loff_t)high << 32) | low); + return sys_ftruncate(fd, ((loff_t)length_hi << 32) | length_lo); } COMPAT_SYSCALL_DEFINE6(pread64, unsigned int, fd, char __user *, ubuf, - size_t, count, u32, dummy, u32, low, u32, high) + size_t, count, u32, dummy, SYSCALL_PAIR(offset)) { - return sys_pread64(fd, ubuf, count, ((loff_t)high << 32) | low); + return sys_pread64(fd, ubuf, count, + ((loff_t)offset_hi << 32) | offset_lo); } COMPAT_SYSCALL_DEFINE6(pwrite64, unsigned int, fd, char __user *, ubuf, - size_t, count, u32, dummy, u32, low, u32, high) + size_t, count, u32, dummy, SYSCALL_PAIR(offset)) { - return sys_pwrite64(fd, ubuf, count, ((loff_t)high << 32) | low); + return sys_pwrite64(fd, ubuf, count, + ((loff_t)offset_hi << 32) | offset_lo); } COMPAT_SYSCALL_DEFINE6(sync_file_range2, int, fd, unsigned int, flags, - u32, offset_lo, u32, offset_hi, - u32, nbytes_lo, u32, nbytes_hi) + SYSCALL_PAIR(offset), SYSCALL_PAIR(nbytes)) { return sys_sync_file_range(fd, ((loff_t)offset_hi << 32) | offset_lo, ((loff_t)nbytes_hi << 32) | nbytes_lo, @@ -66,8 +74,7 @@ COMPAT_SYSCALL_DEFINE6(sync_file_range2, int, fd, unsigned int, flags, } COMPAT_SYSCALL_DEFINE6(fallocate, int, fd, int, mode, - u32, offset_lo, u32, offset_hi, - u32, len_lo, u32, len_hi) + SYSCALL_PAIR(offset), SYSCALL_PAIR(len)) { return sys_fallocate(fd, mode, ((loff_t)offset_hi << 32) | offset_lo, ((loff_t)len_hi << 32) | len_lo); @@ -77,6 +84,8 @@ COMPAT_SYSCALL_DEFINE6(fallocate, int, fd, int, mode, * Avoid bug in generic sys_llseek() that specifies offset_high and * offset_low as "unsigned long", thus making it possible to pass * a sign-extended high 32 bits in offset_low. + * Note that we do not use SYSCALL_PAIR here since glibc passes the + * high and low parts explicitly in that order. */ COMPAT_SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned int, offset_high, unsigned int, offset_low, loff_t __user *, result, diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c index 6b705ccc9cc1..a465d8372edd 100644 --- a/arch/tile/kernel/process.c +++ b/arch/tile/kernel/process.c @@ -73,8 +73,9 @@ void arch_cpu_idle(void) /* * Release a thread_info structure */ -void arch_release_thread_info(struct thread_info *info) +void arch_release_thread_stack(unsigned long *stack) { + struct thread_info *info = (void *)stack; struct single_step_state *step_state = info->step_state; if (step_state) { diff --git a/arch/tile/kernel/sys.c b/arch/tile/kernel/sys.c index 38debe706061..c7418dcbbb08 100644 --- a/arch/tile/kernel/sys.c +++ b/arch/tile/kernel/sys.c @@ -33,6 +33,7 @@ #include #include #include +#include #include SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, len, @@ -59,13 +60,19 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, len, #if !defined(__tilegx__) || defined(CONFIG_COMPAT) -ssize_t sys32_readahead(int fd, u32 offset_lo, u32 offset_hi, u32 count) +#ifdef __BIG_ENDIAN +#define SYSCALL_PAIR(name) u32 name ## _hi, u32 name ## _lo +#else +#define SYSCALL_PAIR(name) u32 name ## _lo, u32 name ## _hi +#endif + +ssize_t sys32_readahead(int fd, SYSCALL_PAIR(offset), u32 count) { return sys_readahead(fd, ((loff_t)offset_hi << 32) | offset_lo, count); } -int sys32_fadvise64_64(int fd, u32 offset_lo, u32 offset_hi, - u32 len_lo, u32 len_hi, int advice) +int sys32_fadvise64_64(int fd, SYSCALL_PAIR(offset), + SYSCALL_PAIR(len), int advice) { return sys_fadvise64_64(fd, ((loff_t)offset_hi << 32) | offset_lo, ((loff_t)len_hi << 32) | len_lo, advice); diff --git a/arch/tile/lib/atomic_32.c b/arch/tile/lib/atomic_32.c index 298df1e9912a..f8128800dbf5 100644 --- a/arch/tile/lib/atomic_32.c +++ b/arch/tile/lib/atomic_32.c @@ -61,13 +61,13 @@ static inline int *__atomic_setup(volatile void *v) int _atomic_xchg(int *v, int n) { - return __atomic_xchg(v, __atomic_setup(v), n).val; + return __atomic32_xchg(v, __atomic_setup(v), n).val; } EXPORT_SYMBOL(_atomic_xchg); int _atomic_xchg_add(int *v, int i) { - return __atomic_xchg_add(v, __atomic_setup(v), i).val; + return __atomic32_xchg_add(v, __atomic_setup(v), i).val; } EXPORT_SYMBOL(_atomic_xchg_add); @@ -78,39 +78,39 @@ int _atomic_xchg_add_unless(int *v, int a, int u) * to use the first argument consistently as the "old value" * in the assembly, as is done for _atomic_cmpxchg(). */ - return __atomic_xchg_add_unless(v, __atomic_setup(v), u, a).val; + return __atomic32_xchg_add_unless(v, __atomic_setup(v), u, a).val; } EXPORT_SYMBOL(_atomic_xchg_add_unless); int _atomic_cmpxchg(int *v, int o, int n) { - return __atomic_cmpxchg(v, __atomic_setup(v), o, n).val; + return __atomic32_cmpxchg(v, __atomic_setup(v), o, n).val; } EXPORT_SYMBOL(_atomic_cmpxchg); -unsigned long _atomic_or(volatile unsigned long *p, unsigned long mask) +unsigned long _atomic_fetch_or(volatile unsigned long *p, unsigned long mask) { - return __atomic_or((int *)p, __atomic_setup(p), mask).val; + return __atomic32_fetch_or((int *)p, __atomic_setup(p), mask).val; } -EXPORT_SYMBOL(_atomic_or); +EXPORT_SYMBOL(_atomic_fetch_or); -unsigned long _atomic_and(volatile unsigned long *p, unsigned long mask) +unsigned long _atomic_fetch_and(volatile unsigned long *p, unsigned long mask) { - return __atomic_and((int *)p, __atomic_setup(p), mask).val; + return __atomic32_fetch_and((int *)p, __atomic_setup(p), mask).val; } -EXPORT_SYMBOL(_atomic_and); +EXPORT_SYMBOL(_atomic_fetch_and); -unsigned long _atomic_andn(volatile unsigned long *p, unsigned long mask) +unsigned long _atomic_fetch_andn(volatile unsigned long *p, unsigned long mask) { - return __atomic_andn((int *)p, __atomic_setup(p), mask).val; + return __atomic32_fetch_andn((int *)p, __atomic_setup(p), mask).val; } -EXPORT_SYMBOL(_atomic_andn); +EXPORT_SYMBOL(_atomic_fetch_andn); -unsigned long _atomic_xor(volatile unsigned long *p, unsigned long mask) +unsigned long _atomic_fetch_xor(volatile unsigned long *p, unsigned long mask) { - return __atomic_xor((int *)p, __atomic_setup(p), mask).val; + return __atomic32_fetch_xor((int *)p, __atomic_setup(p), mask).val; } -EXPORT_SYMBOL(_atomic_xor); +EXPORT_SYMBOL(_atomic_fetch_xor); long long _atomic64_xchg(long long *v, long long n) @@ -142,23 +142,23 @@ long long _atomic64_cmpxchg(long long *v, long long o, long long n) } EXPORT_SYMBOL(_atomic64_cmpxchg); -long long _atomic64_and(long long *v, long long n) +long long _atomic64_fetch_and(long long *v, long long n) { - return __atomic64_and(v, __atomic_setup(v), n); + return __atomic64_fetch_and(v, __atomic_setup(v), n); } -EXPORT_SYMBOL(_atomic64_and); +EXPORT_SYMBOL(_atomic64_fetch_and); -long long _atomic64_or(long long *v, long long n) +long long _atomic64_fetch_or(long long *v, long long n) { - return __atomic64_or(v, __atomic_setup(v), n); + return __atomic64_fetch_or(v, __atomic_setup(v), n); } -EXPORT_SYMBOL(_atomic64_or); +EXPORT_SYMBOL(_atomic64_fetch_or); -long long _atomic64_xor(long long *v, long long n) +long long _atomic64_fetch_xor(long long *v, long long n) { - return __atomic64_xor(v, __atomic_setup(v), n); + return __atomic64_fetch_xor(v, __atomic_setup(v), n); } -EXPORT_SYMBOL(_atomic64_xor); +EXPORT_SYMBOL(_atomic64_fetch_xor); /* * If any of the atomic or futex routines hit a bad address (not in diff --git a/arch/tile/lib/atomic_asm_32.S b/arch/tile/lib/atomic_asm_32.S index f611265633d6..1a70e6c0f259 100644 --- a/arch/tile/lib/atomic_asm_32.S +++ b/arch/tile/lib/atomic_asm_32.S @@ -172,15 +172,20 @@ STD_ENTRY_SECTION(__atomic\name, .text.atomic) .endif .endm -atomic_op _cmpxchg, 32, "seq r26, r22, r2; { bbns r26, 3f; move r24, r3 }" -atomic_op _xchg, 32, "move r24, r2" -atomic_op _xchg_add, 32, "add r24, r22, r2" -atomic_op _xchg_add_unless, 32, \ + +/* + * Use __atomic32 prefix to avoid collisions with GCC builtin __atomic functions. + */ + +atomic_op 32_cmpxchg, 32, "seq r26, r22, r2; { bbns r26, 3f; move r24, r3 }" +atomic_op 32_xchg, 32, "move r24, r2" +atomic_op 32_xchg_add, 32, "add r24, r22, r2" +atomic_op 32_xchg_add_unless, 32, \ "sne r26, r22, r2; { bbns r26, 3f; add r24, r22, r3 }" -atomic_op _or, 32, "or r24, r22, r2" -atomic_op _and, 32, "and r24, r22, r2" -atomic_op _andn, 32, "nor r2, r2, zero; and r24, r22, r2" -atomic_op _xor, 32, "xor r24, r22, r2" +atomic_op 32_fetch_or, 32, "or r24, r22, r2" +atomic_op 32_fetch_and, 32, "and r24, r22, r2" +atomic_op 32_fetch_andn, 32, "nor r2, r2, zero; and r24, r22, r2" +atomic_op 32_fetch_xor, 32, "xor r24, r22, r2" atomic_op 64_cmpxchg, 64, "{ seq r26, r22, r2; seq r27, r23, r3 }; \ { bbns r26, 3f; move r24, r4 }; { bbns r27, 3f; move r25, r5 }" @@ -192,9 +197,9 @@ atomic_op 64_xchg_add_unless, 64, \ { bbns r26, 3f; add r24, r22, r4 }; \ { bbns r27, 3f; add r25, r23, r5 }; \ slt_u r26, r24, r22; add r25, r25, r26" -atomic_op 64_or, 64, "{ or r24, r22, r2; or r25, r23, r3 }" -atomic_op 64_and, 64, "{ and r24, r22, r2; and r25, r23, r3 }" -atomic_op 64_xor, 64, "{ xor r24, r22, r2; xor r25, r23, r3 }" +atomic_op 64_fetch_or, 64, "{ or r24, r22, r2; or r25, r23, r3 }" +atomic_op 64_fetch_and, 64, "{ and r24, r22, r2; and r25, r23, r3 }" +atomic_op 64_fetch_xor, 64, "{ xor r24, r22, r2; xor r25, r23, r3 }" jrp lr /* happy backtracer */ diff --git a/arch/tile/lib/exports.c b/arch/tile/lib/exports.c index 9d171ca4302c..c5369fe643c7 100644 --- a/arch/tile/lib/exports.c +++ b/arch/tile/lib/exports.c @@ -77,7 +77,11 @@ uint64_t __umoddi3(uint64_t dividend, uint64_t divisor); EXPORT_SYMBOL(__umoddi3); int64_t __moddi3(int64_t dividend, int64_t divisor); EXPORT_SYMBOL(__moddi3); -#ifndef __tilegx__ +#ifdef __tilegx__ +typedef int TItype __attribute__((mode(TI))); +TItype __multi3(TItype a, TItype b); +EXPORT_SYMBOL(__multi3); /* required for gcc 7 and later */ +#else int64_t __muldi3(int64_t, int64_t); EXPORT_SYMBOL(__muldi3); uint64_t __lshrdi3(uint64_t, unsigned int); diff --git a/arch/tile/lib/spinlock_32.c b/arch/tile/lib/spinlock_32.c index 88c2a53362e7..076c6cc43113 100644 --- a/arch/tile/lib/spinlock_32.c +++ b/arch/tile/lib/spinlock_32.c @@ -76,6 +76,12 @@ void arch_spin_unlock_wait(arch_spinlock_t *lock) do { delay_backoff(iterations++); } while (READ_ONCE(lock->current_ticket) == curr); + + /* + * The TILE architecture doesn't do read speculation; therefore + * a control dependency guarantees a LOAD->{LOAD,STORE} order. + */ + barrier(); } EXPORT_SYMBOL(arch_spin_unlock_wait); diff --git a/arch/tile/lib/spinlock_64.c b/arch/tile/lib/spinlock_64.c index c8d1f94ff1fe..a4b5b2cbce93 100644 --- a/arch/tile/lib/spinlock_64.c +++ b/arch/tile/lib/spinlock_64.c @@ -76,6 +76,12 @@ void arch_spin_unlock_wait(arch_spinlock_t *lock) do { delay_backoff(iterations++); } while (arch_spin_current(READ_ONCE(lock->lock)) == curr); + + /* + * The TILE architecture doesn't do read speculation; therefore + * a control dependency guarantees a LOAD->{LOAD,STORE} order. + */ + barrier(); } EXPORT_SYMBOL(arch_spin_unlock_wait); diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c index 26734214818c..beba986589e5 100644 --- a/arch/tile/mm/fault.c +++ b/arch/tile/mm/fault.c @@ -434,7 +434,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return 0; diff --git a/arch/tile/mm/pgtable.c b/arch/tile/mm/pgtable.c index 7bf2491a9c1f..c4d5bf841a7f 100644 --- a/arch/tile/mm/pgtable.c +++ b/arch/tile/mm/pgtable.c @@ -231,7 +231,7 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd) struct page *pgtable_alloc_one(struct mm_struct *mm, unsigned long address, int order) { - gfp_t flags = GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO; + gfp_t flags = GFP_KERNEL|__GFP_ZERO; struct page *p; int i; diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index 17e96dc29596..f3540270d096 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -801,6 +801,7 @@ static void ubd_device_release(struct device *dev) static int ubd_disk_register(int major, u64 size, int unit, struct gendisk **disk_out) { + struct device *parent = NULL; struct gendisk *disk; disk = alloc_disk(1 << UBD_SHIFT); @@ -823,12 +824,12 @@ static int ubd_disk_register(int major, u64 size, int unit, ubd_devs[unit].pdev.dev.release = ubd_device_release; dev_set_drvdata(&ubd_devs[unit].pdev.dev, &ubd_devs[unit]); platform_device_register(&ubd_devs[unit].pdev); - disk->driverfs_dev = &ubd_devs[unit].pdev.dev; + parent = &ubd_devs[unit].pdev.dev; } disk->private_data = &ubd_devs[unit]; disk->queue = ubd_devs[unit].queue; - add_disk(disk); + device_add_disk(parent, disk); *disk_out = disk; return 0; @@ -1286,7 +1287,7 @@ static void do_ubd_request(struct request_queue *q) req = dev->request; - if (req->cmd_flags & REQ_FLUSH) { + if (req_op(req) == REQ_OP_FLUSH) { io_req = kmalloc(sizeof(struct io_thread_req), GFP_ATOMIC); if (io_req == NULL) { diff --git a/arch/um/include/asm/tlb.h b/arch/um/include/asm/tlb.h index 16eb63fac57d..821ff0acfe17 100644 --- a/arch/um/include/asm/tlb.h +++ b/arch/um/include/asm/tlb.h @@ -102,7 +102,7 @@ static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) { tlb->need_flush = 1; free_page_and_swap_cache(page); - return 1; /* avoid calling tlb_flush_mmu */ + return false; /* avoid calling tlb_flush_mmu */ } static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) @@ -110,6 +110,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) __tlb_remove_page(tlb, page); } +static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return __tlb_remove_page(tlb, page); +} + +static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, + struct page *page) +{ + return __tlb_remove_page(tlb, page); +} + +static inline void tlb_remove_page_size(struct mmu_gather *tlb, + struct page *page, int page_size) +{ + return tlb_remove_page(tlb, page); +} + /** * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation. * diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index b2a2dff50b4e..e7437ec62710 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -204,7 +204,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { pte_t *pte; - pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); return pte; } @@ -212,7 +212,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *pte; - pte = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pte = alloc_page(GFP_KERNEL|__GFP_ZERO); if (!pte) return NULL; if (!pgtable_page_ctor(pte)) { diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index 98783dd0fa2e..ad8f206ab5e8 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -73,7 +73,7 @@ good_area: do { int fault; - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) goto out_nosemaphore; diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig index e5602ee9c610..0769066929c6 100644 --- a/arch/unicore32/Kconfig +++ b/arch/unicore32/Kconfig @@ -80,7 +80,7 @@ config ARCH_PUV3 select CPU_UCV2 select GENERIC_CLOCKEVENTS select HAVE_CLK - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB # CONFIGs for ARCH_PUV3 diff --git a/arch/unicore32/include/asm/pgalloc.h b/arch/unicore32/include/asm/pgalloc.h index 2e02d1356fdf..26775793c204 100644 --- a/arch/unicore32/include/asm/pgalloc.h +++ b/arch/unicore32/include/asm/pgalloc.h @@ -28,7 +28,7 @@ extern void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd); #define pgd_alloc(mm) get_pgd_slow(mm) #define pgd_free(mm, pgd) free_pgd_slow(mm, pgd) -#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) +#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO) /* * Allocate one PTE table. diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c index 2ec3d3adcefc..6c7f70bcaae3 100644 --- a/arch/unicore32/mm/fault.c +++ b/arch/unicore32/mm/fault.c @@ -194,7 +194,7 @@ good_area: * If for any reason at all we couldn't handle the fault, make * sure we exit gracefully rather than endlessly redo the fault. */ - fault = handle_mm_fault(mm, vma, addr & PAGE_MASK, flags); + fault = handle_mm_fault(vma, addr & PAGE_MASK, flags); return fault; check_stack: diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index d9a94da0c29f..2fa55851d2a9 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -22,6 +22,7 @@ config X86 select ANON_INODES select ARCH_CLOCKSOURCE_DATA select ARCH_DISCARD_MEMBLOCK + select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE select ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS select ARCH_HAS_DEVMEM_IS_ALLOWED @@ -49,7 +50,6 @@ config X86 select ARCH_WANTS_DYNAMIC_TASK_STRUCT select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_IPC_PARSE_VERSION if X86_32 - select ARCH_WANT_OPTIONAL_GPIOLIB select BUILDTIME_EXTABLE_SORT select CLKEVT_I8253 select CLKSRC_I8253 if X86_32 @@ -294,11 +294,6 @@ config X86_32_LAZY_GS def_bool y depends on X86_32 && !CC_STACKPROTECTOR -config ARCH_HWEIGHT_CFLAGS - string - default "-fcall-saved-ecx -fcall-saved-edx" if X86_32 - default "-fcall-saved-rdi -fcall-saved-rsi -fcall-saved-rdx -fcall-saved-rcx -fcall-saved-r8 -fcall-saved-r9 -fcall-saved-r10 -fcall-saved-r11" if X86_64 - config ARCH_SUPPORTS_UPROBES def_bool y @@ -643,7 +638,7 @@ config STA2X11 select X86_DMA_REMAP select SWIOTLB select MFD_STA2X11 - select ARCH_REQUIRE_GPIOLIB + select GPIOLIB default n ---help--- This adds support for boards based on the STA2X11 IO-Hub, @@ -1934,21 +1929,26 @@ config RANDOMIZE_BASE attempts relying on knowledge of the location of kernel code internals. - The kernel physical and virtual address can be randomized - from 16MB up to 1GB on 64-bit and 512MB on 32-bit. (Note that - using RANDOMIZE_BASE reduces the memory space available to - kernel modules from 1.5GB to 1GB.) + On 64-bit, the kernel physical and virtual addresses are + randomized separately. The physical address will be anywhere + between 16MB and the top of physical memory (up to 64TB). The + virtual address will be randomized from 16MB up to 1GB (9 bits + of entropy). Note that this also reduces the memory space + available to kernel modules from 1.5GB to 1GB. + + On 32-bit, the kernel physical and virtual addresses are + randomized together. They will be randomized from 16MB up to + 512MB (8 bits of entropy). Entropy is generated using the RDRAND instruction if it is supported. If RDTSC is supported, its value is mixed into the entropy pool as well. If neither RDRAND nor RDTSC are - supported, then entropy is read from the i8254 timer. - - Since the kernel is built using 2GB addressing, and - PHYSICAL_ALIGN must be at a minimum of 2MB, only 10 bits of - entropy is theoretically possible. Currently, with the - default value for PHYSICAL_ALIGN and due to page table - layouts, 64-bit uses 9 bits of entropy and 32-bit uses 8 bits. + supported, then entropy is read from the i8254 timer. The + usable entropy is limited by the kernel being built using + 2GB addressing, and that PHYSICAL_ALIGN must be at a + minimum of 2MB. As a result, only 10 bits of entropy are + theoretically possible, but the implementations are further + limited due to memory layouts. If CONFIG_HIBERNATE is also enabled, KASLR is disabled at boot time. To enable it, boot with "kaslr" on the kernel command @@ -1988,6 +1988,38 @@ config PHYSICAL_ALIGN Don't change this unless you know what you are doing. +config RANDOMIZE_MEMORY + bool "Randomize the kernel memory sections" + depends on X86_64 + depends on RANDOMIZE_BASE + default RANDOMIZE_BASE + ---help--- + Randomizes the base virtual address of kernel memory sections + (physical memory mapping, vmalloc & vmemmap). This security feature + makes exploits relying on predictable memory locations less reliable. + + The order of allocations remains unchanged. Entropy is generated in + the same way as RANDOMIZE_BASE. Current implementation in the optimal + configuration have in average 30,000 different possible virtual + addresses for each memory section. + + If unsure, say N. + +config RANDOMIZE_MEMORY_PHYSICAL_PADDING + hex "Physical memory mapping padding" if EXPERT + depends on RANDOMIZE_MEMORY + default "0xa" if MEMORY_HOTPLUG + default "0x0" + range 0x1 0x40 if MEMORY_HOTPLUG + range 0x0 0x40 + ---help--- + Define the padding in terabytes added to the existing physical + memory size during kernel memory randomization. It is useful + for memory hotplug support but reduces the entropy available for + address randomization. + + If unsure, leave at the default value. + config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SMP diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 6fce7f096b88..830ed391e7ef 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -126,14 +126,6 @@ else KBUILD_CFLAGS += $(call cc-option,-maccumulate-outgoing-args) endif -# Make sure compiler does not have buggy stack-protector support. -ifdef CONFIG_CC_STACKPROTECTOR - cc_has_sp := $(srctree)/scripts/gcc-x86_$(BITS)-has-stack-protector.sh - ifneq ($(shell $(CONFIG_SHELL) $(cc_has_sp) $(CC) $(KBUILD_CPPFLAGS) $(biarch)),y) - $(warning stack-protector enabled but compiler support broken) - endif -endif - ifdef CONFIG_X86_X32 x32_ld_ok := $(call try-run,\ /bin/echo -e '1: .quad 1b' | \ diff --git a/arch/x86/boot/bitops.h b/arch/x86/boot/bitops.h index 878e4b9940d9..0d41d68131cc 100644 --- a/arch/x86/boot/bitops.h +++ b/arch/x86/boot/bitops.h @@ -16,14 +16,16 @@ #define BOOT_BITOPS_H #define _LINUX_BITOPS_H /* Inhibit inclusion of */ -static inline int constant_test_bit(int nr, const void *addr) +#include + +static inline bool constant_test_bit(int nr, const void *addr) { const u32 *p = (const u32 *)addr; return ((1UL << (nr & 31)) & (p[nr >> 5])) != 0; } -static inline int variable_test_bit(int nr, const void *addr) +static inline bool variable_test_bit(int nr, const void *addr) { - u8 v; + bool v; const u32 *p = (const u32 *)addr; asm("btl %2,%1; setc %0" : "=qm" (v) : "m" (*p), "Ir" (nr)); diff --git a/arch/x86/boot/boot.h b/arch/x86/boot/boot.h index 9011a88353de..e5612f3e3b57 100644 --- a/arch/x86/boot/boot.h +++ b/arch/x86/boot/boot.h @@ -24,6 +24,7 @@ #include #include #include +#include #include "bitops.h" #include "ctype.h" #include "cpuflags.h" @@ -176,18 +177,18 @@ static inline void wrgs32(u32 v, addr_t addr) } /* Note: these only return true/false, not a signed return value! */ -static inline int memcmp_fs(const void *s1, addr_t s2, size_t len) +static inline bool memcmp_fs(const void *s1, addr_t s2, size_t len) { - u8 diff; - asm volatile("fs; repe; cmpsb; setnz %0" - : "=qm" (diff), "+D" (s1), "+S" (s2), "+c" (len)); + bool diff; + asm volatile("fs; repe; cmpsb" CC_SET(nz) + : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len)); return diff; } -static inline int memcmp_gs(const void *s1, addr_t s2, size_t len) +static inline bool memcmp_gs(const void *s1, addr_t s2, size_t len) { - u8 diff; - asm volatile("gs; repe; cmpsb; setnz %0" - : "=qm" (diff), "+D" (s1), "+S" (s2), "+c" (len)); + bool diff; + asm volatile("gs; repe; cmpsb" CC_SET(nz) + : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len)); return diff; } @@ -294,6 +295,7 @@ static inline int cmdline_find_option_bool(const char *option) /* cpu.c, cpucheck.c */ int check_cpu(int *cpu_level_ptr, int *req_level_ptr, u32 **err_flags_ptr); +int check_knl_erratum(void); int validate_cpu(void); /* early_serial_console.c */ diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index f1356889204e..536ccfcc01c6 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -85,7 +85,25 @@ vmlinux-objs-$(CONFIG_EFI_STUB) += $(obj)/eboot.o $(obj)/efi_stub_$(BITS).o \ $(objtree)/drivers/firmware/efi/libstub/lib.a vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_thunk_$(BITS).o +# The compressed kernel is built with -fPIC/-fPIE so that a boot loader +# can place it anywhere in memory and it will still run. However, since +# it is executed as-is without any ELF relocation processing performed +# (and has already had all relocation sections stripped from the binary), +# none of the code can use data relocations (e.g. static assignments of +# pointer values), since they will be meaningless at runtime. This check +# will refuse to link the vmlinux if any of these relocations are found. +quiet_cmd_check_data_rel = DATAREL $@ +define cmd_check_data_rel + for obj in $(filter %.o,$^); do \ + readelf -S $$obj | grep -qF .rel.local && { \ + echo "error: $$obj has data relocations!" >&2; \ + exit 1; \ + } || true; \ + done +endef + $(obj)/vmlinux: $(vmlinux-objs-y) FORCE + $(call if_changed,check_data_rel) $(call if_changed,ld) OBJCOPYFLAGS_vmlinux.bin := -R .comment -S diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c index 52fef606bc54..ff574dad95cc 100644 --- a/arch/x86/boot/compressed/eboot.c +++ b/arch/x86/boot/compressed/eboot.c @@ -757,7 +757,6 @@ struct boot_params *make_boot_params(struct efi_config *c) struct boot_params *boot_params; struct apm_bios_info *bi; struct setup_header *hdr; - struct efi_info *efi; efi_loaded_image_t *image; void *options, *handle; efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID; @@ -800,7 +799,6 @@ struct boot_params *make_boot_params(struct efi_config *c) memset(boot_params, 0x0, 0x4000); hdr = &boot_params->hdr; - efi = &boot_params->efi_info; bi = &boot_params->apm_bios_info; /* Copy the second sector to boot_params */ diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index cfeb0259ed81..a66854d99ee1 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -12,10 +12,6 @@ #include "misc.h" #include "error.h" -#include -#include -#include - #include #include #include @@ -26,26 +22,6 @@ static const char build_str[] = UTS_RELEASE " (" LINUX_COMPILE_BY "@" LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION; -#define I8254_PORT_CONTROL 0x43 -#define I8254_PORT_COUNTER0 0x40 -#define I8254_CMD_READBACK 0xC0 -#define I8254_SELECT_COUNTER0 0x02 -#define I8254_STATUS_NOTREADY 0x40 -static inline u16 i8254(void) -{ - u16 status, timer; - - do { - outb(I8254_PORT_CONTROL, - I8254_CMD_READBACK | I8254_SELECT_COUNTER0); - status = inb(I8254_PORT_COUNTER0); - timer = inb(I8254_PORT_COUNTER0); - timer |= inb(I8254_PORT_COUNTER0) << 8; - } while (status & I8254_STATUS_NOTREADY); - - return timer; -} - static unsigned long rotate_xor(unsigned long hash, const void *area, size_t size) { @@ -62,7 +38,7 @@ static unsigned long rotate_xor(unsigned long hash, const void *area, } /* Attempt to create a simple but unpredictable starting entropy. */ -static unsigned long get_random_boot(void) +static unsigned long get_boot_seed(void) { unsigned long hash = 0; @@ -72,50 +48,8 @@ static unsigned long get_random_boot(void) return hash; } -static unsigned long get_random_long(const char *purpose) -{ -#ifdef CONFIG_X86_64 - const unsigned long mix_const = 0x5d6008cbf3848dd3UL; -#else - const unsigned long mix_const = 0x3f39e593UL; -#endif - unsigned long raw, random = get_random_boot(); - bool use_i8254 = true; - - debug_putstr(purpose); - debug_putstr(" KASLR using"); - - if (has_cpuflag(X86_FEATURE_RDRAND)) { - debug_putstr(" RDRAND"); - if (rdrand_long(&raw)) { - random ^= raw; - use_i8254 = false; - } - } - - if (has_cpuflag(X86_FEATURE_TSC)) { - debug_putstr(" RDTSC"); - raw = rdtsc(); - - random ^= raw; - use_i8254 = false; - } - - if (use_i8254) { - debug_putstr(" i8254"); - random ^= i8254(); - } - - /* Circular multiply for better bit diffusion */ - asm("mul %3" - : "=a" (random), "=d" (raw) - : "a" (random), "rm" (mix_const)); - random += raw; - - debug_putstr("...\n"); - - return random; -} +#define KASLR_COMPRESSED_BOOT +#include "../../lib/kaslr.c" struct mem_vector { unsigned long start; @@ -132,17 +66,6 @@ enum mem_avoid_index { static struct mem_vector mem_avoid[MEM_AVOID_MAX]; -static bool mem_contains(struct mem_vector *region, struct mem_vector *item) -{ - /* Item at least partially before region. */ - if (item->start < region->start) - return false; - /* Item at least partially after region. */ - if (item->start + item->size > region->start + region->size) - return false; - return true; -} - static bool mem_overlaps(struct mem_vector *one, struct mem_vector *two) { /* Item one is entirely before item two. */ @@ -296,6 +219,7 @@ static bool mem_avoid_overlap(struct mem_vector *img, if (mem_overlaps(img, &mem_avoid[i]) && mem_avoid[i].start < earliest) { *overlap = mem_avoid[i]; + earliest = overlap->start; is_overlapping = true; } } @@ -310,6 +234,7 @@ static bool mem_avoid_overlap(struct mem_vector *img, if (mem_overlaps(img, &avoid) && (avoid.start < earliest)) { *overlap = avoid; + earliest = overlap->start; is_overlapping = true; } @@ -319,8 +244,6 @@ static bool mem_avoid_overlap(struct mem_vector *img, return is_overlapping; } -static unsigned long slots[KERNEL_IMAGE_SIZE / CONFIG_PHYSICAL_ALIGN]; - struct slot_area { unsigned long addr; int num; @@ -351,36 +274,44 @@ static void store_slot_info(struct mem_vector *region, unsigned long image_size) } } -static void slots_append(unsigned long addr) -{ - /* Overflowing the slots list should be impossible. */ - if (slot_max >= KERNEL_IMAGE_SIZE / CONFIG_PHYSICAL_ALIGN) - return; - - slots[slot_max++] = addr; -} - static unsigned long slots_fetch_random(void) { + unsigned long slot; + int i; + /* Handle case of no slots stored. */ if (slot_max == 0) return 0; - return slots[get_random_long("Physical") % slot_max]; + slot = kaslr_get_random_long("Physical") % slot_max; + + for (i = 0; i < slot_area_index; i++) { + if (slot >= slot_areas[i].num) { + slot -= slot_areas[i].num; + continue; + } + return slot_areas[i].addr + slot * CONFIG_PHYSICAL_ALIGN; + } + + if (i == slot_area_index) + debug_putstr("slots_fetch_random() failed!?\n"); + return 0; } static void process_e820_entry(struct e820entry *entry, unsigned long minimum, unsigned long image_size) { - struct mem_vector region, img, overlap; + struct mem_vector region, overlap; + struct slot_area slot_area; + unsigned long start_orig; /* Skip non-RAM entries. */ if (entry->type != E820_RAM) return; - /* Ignore entries entirely above our maximum. */ - if (entry->addr >= KERNEL_IMAGE_SIZE) + /* On 32-bit, ignore entries entirely above our maximum. */ + if (IS_ENABLED(CONFIG_X86_32) && entry->addr >= KERNEL_IMAGE_SIZE) return; /* Ignore entries entirely below our minimum. */ @@ -390,31 +321,55 @@ static void process_e820_entry(struct e820entry *entry, region.start = entry->addr; region.size = entry->size; - /* Potentially raise address to minimum location. */ - if (region.start < minimum) - region.start = minimum; + /* Give up if slot area array is full. */ + while (slot_area_index < MAX_SLOT_AREA) { + start_orig = region.start; - /* Potentially raise address to meet alignment requirements. */ - region.start = ALIGN(region.start, CONFIG_PHYSICAL_ALIGN); + /* Potentially raise address to minimum location. */ + if (region.start < minimum) + region.start = minimum; - /* Did we raise the address above the bounds of this e820 region? */ - if (region.start > entry->addr + entry->size) - return; + /* Potentially raise address to meet alignment needs. */ + region.start = ALIGN(region.start, CONFIG_PHYSICAL_ALIGN); - /* Reduce size by any delta from the original address. */ - region.size -= region.start - entry->addr; + /* Did we raise the address above this e820 region? */ + if (region.start > entry->addr + entry->size) + return; - /* Reduce maximum size to fit end of image within maximum limit. */ - if (region.start + region.size > KERNEL_IMAGE_SIZE) - region.size = KERNEL_IMAGE_SIZE - region.start; + /* Reduce size by any delta from the original address. */ + region.size -= region.start - start_orig; - /* Walk each aligned slot and check for avoided areas. */ - for (img.start = region.start, img.size = image_size ; - mem_contains(®ion, &img) ; - img.start += CONFIG_PHYSICAL_ALIGN) { - if (mem_avoid_overlap(&img, &overlap)) - continue; - slots_append(img.start); + /* On 32-bit, reduce region size to fit within max size. */ + if (IS_ENABLED(CONFIG_X86_32) && + region.start + region.size > KERNEL_IMAGE_SIZE) + region.size = KERNEL_IMAGE_SIZE - region.start; + + /* Return if region can't contain decompressed kernel */ + if (region.size < image_size) + return; + + /* If nothing overlaps, store the region and return. */ + if (!mem_avoid_overlap(®ion, &overlap)) { + store_slot_info(®ion, image_size); + return; + } + + /* Store beginning of region if holds at least image_size. */ + if (overlap.start > region.start + image_size) { + struct mem_vector beginning; + + beginning.start = region.start; + beginning.size = overlap.start - region.start; + store_slot_info(&beginning, image_size); + } + + /* Return if overlap extends to or past end of region. */ + if (overlap.start + overlap.size >= region.start + region.size) + return; + + /* Clip off the overlapping region and start over. */ + region.size -= overlap.start - region.start + overlap.size; + region.start = overlap.start + overlap.size; } } @@ -431,6 +386,10 @@ static unsigned long find_random_phys_addr(unsigned long minimum, for (i = 0; i < boot_params->e820_entries; i++) { process_e820_entry(&boot_params->e820_map[i], minimum, image_size); + if (slot_area_index == MAX_SLOT_AREA) { + debug_putstr("Aborted e820 scan (slot_areas full)!\n"); + break; + } } return slots_fetch_random(); @@ -454,7 +413,7 @@ static unsigned long find_random_virt_addr(unsigned long minimum, slots = (KERNEL_IMAGE_SIZE - minimum - image_size) / CONFIG_PHYSICAL_ALIGN + 1; - random_addr = get_random_long("Virtual") % slots; + random_addr = kaslr_get_random_long("Virtual") % slots; return random_addr * CONFIG_PHYSICAL_ALIGN + minimum; } @@ -463,48 +422,54 @@ static unsigned long find_random_virt_addr(unsigned long minimum, * Since this function examines addresses much more numerically, * it takes the input and output pointers as 'unsigned long'. */ -unsigned char *choose_random_location(unsigned long input, - unsigned long input_size, - unsigned long output, - unsigned long output_size) +void choose_random_location(unsigned long input, + unsigned long input_size, + unsigned long *output, + unsigned long output_size, + unsigned long *virt_addr) { - unsigned long choice = output; - unsigned long random_addr; + unsigned long random_addr, min_addr; + + /* By default, keep output position unchanged. */ + *virt_addr = *output; -#ifdef CONFIG_HIBERNATION - if (!cmdline_find_option_bool("kaslr")) { - warn("KASLR disabled: 'kaslr' not on cmdline (hibernation selected)."); - goto out; - } -#else if (cmdline_find_option_bool("nokaslr")) { warn("KASLR disabled: 'nokaslr' on cmdline."); - goto out; + return; } -#endif boot_params->hdr.loadflags |= KASLR_FLAG; + /* Prepare to add new identity pagetables on demand. */ + initialize_identity_maps(); + /* Record the various known unsafe memory ranges. */ - mem_avoid_init(input, input_size, output); + mem_avoid_init(input, input_size, *output); + + /* + * Low end of the randomization range should be the + * smaller of 512M or the initial kernel image + * location: + */ + min_addr = min(*output, 512UL << 20); /* Walk e820 and find a random address. */ - random_addr = find_random_phys_addr(output, output_size); + random_addr = find_random_phys_addr(min_addr, output_size); if (!random_addr) { warn("KASLR disabled: could not find suitable E820 region!"); - goto out; + } else { + /* Update the new physical address location. */ + if (*output != random_addr) { + add_identity_map(random_addr, output_size); + *output = random_addr; + } } - /* Always enforce the minimum. */ - if (random_addr < choice) - goto out; - - choice = random_addr; - - add_identity_map(choice, output_size); - /* This actually loads the identity pagetable on x86_64. */ finalize_identity_maps(); -out: - return (unsigned char *)choice; + + /* Pick random virtual address starting from LOAD_PHYSICAL_ADDR. */ + if (IS_ENABLED(CONFIG_X86_64)) + random_addr = find_random_virt_addr(LOAD_PHYSICAL_ADDR, output_size); + *virt_addr = random_addr; } diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index f14db4e21654..b3c5a5f030ce 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -170,7 +170,8 @@ void __puthex(unsigned long value) } #if CONFIG_X86_NEED_RELOCS -static void handle_relocations(void *output, unsigned long output_len) +static void handle_relocations(void *output, unsigned long output_len, + unsigned long virt_addr) { int *reloc; unsigned long delta, map, ptr; @@ -182,11 +183,6 @@ static void handle_relocations(void *output, unsigned long output_len) * and where it was actually loaded. */ delta = min_addr - LOAD_PHYSICAL_ADDR; - if (!delta) { - debug_putstr("No relocation needed... "); - return; - } - debug_putstr("Performing relocations... "); /* * The kernel contains a table of relocation addresses. Those @@ -197,6 +193,20 @@ static void handle_relocations(void *output, unsigned long output_len) */ map = delta - __START_KERNEL_map; + /* + * 32-bit always performs relocations. 64-bit relocations are only + * needed if KASLR has chosen a different starting address offset + * from __START_KERNEL_map. + */ + if (IS_ENABLED(CONFIG_X86_64)) + delta = virt_addr - LOAD_PHYSICAL_ADDR; + + if (!delta) { + debug_putstr("No relocation needed... "); + return; + } + debug_putstr("Performing relocations... "); + /* * Process relocations: 32 bit relocations first then 64 bit after. * Three sets of binary relocations are added to the end of the kernel @@ -250,7 +260,8 @@ static void handle_relocations(void *output, unsigned long output_len) #endif } #else -static inline void handle_relocations(void *output, unsigned long output_len) +static inline void handle_relocations(void *output, unsigned long output_len, + unsigned long virt_addr) { } #endif @@ -327,7 +338,7 @@ asmlinkage __visible void *extract_kernel(void *rmode, memptr heap, unsigned long output_len) { const unsigned long kernel_total_size = VO__end - VO__text; - unsigned char *output_orig = output; + unsigned long virt_addr = (unsigned long)output; /* Retain x86 boot parameters pointer passed from startup_32/64. */ boot_params = rmode; @@ -366,13 +377,16 @@ asmlinkage __visible void *extract_kernel(void *rmode, memptr heap, * the entire decompressed kernel plus relocation table, or the * entire decompressed kernel plus .bss and .brk sections. */ - output = choose_random_location((unsigned long)input_data, input_len, - (unsigned long)output, - max(output_len, kernel_total_size)); + choose_random_location((unsigned long)input_data, input_len, + (unsigned long *)&output, + max(output_len, kernel_total_size), + &virt_addr); /* Validate memory location choices. */ if ((unsigned long)output & (MIN_KERNEL_ALIGN - 1)) - error("Destination address inappropriately aligned"); + error("Destination physical address inappropriately aligned"); + if (virt_addr & (MIN_KERNEL_ALIGN - 1)) + error("Destination virtual address inappropriately aligned"); #ifdef CONFIG_X86_64 if (heap > 0x3fffffffffffUL) error("Destination address too large"); @@ -382,19 +396,16 @@ asmlinkage __visible void *extract_kernel(void *rmode, memptr heap, #endif #ifndef CONFIG_RELOCATABLE if ((unsigned long)output != LOAD_PHYSICAL_ADDR) - error("Wrong destination address"); + error("Destination address does not match LOAD_PHYSICAL_ADDR"); + if ((unsigned long)output != virt_addr) + error("Destination virtual address changed when not relocatable"); #endif debug_putstr("\nDecompressing Linux... "); __decompress(input_data, input_len, NULL, NULL, output, output_len, NULL, error); parse_elf(output); - /* - * 32-bit always performs relocations. 64-bit relocations are only - * needed if kASLR has chosen a different load address. - */ - if (!IS_ENABLED(CONFIG_X86_64) || output != output_orig) - handle_relocations(output, output_len); + handle_relocations(output, output_len, virt_addr); debug_putstr("done.\nBooting the kernel.\n"); return output; } diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index b6fec1ff10e4..1c8355eadbd1 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -67,28 +67,33 @@ int cmdline_find_option_bool(const char *option); #if CONFIG_RANDOMIZE_BASE /* kaslr.c */ -unsigned char *choose_random_location(unsigned long input_ptr, - unsigned long input_size, - unsigned long output_ptr, - unsigned long output_size); +void choose_random_location(unsigned long input, + unsigned long input_size, + unsigned long *output, + unsigned long output_size, + unsigned long *virt_addr); /* cpuflags.c */ bool has_cpuflag(int flag); #else -static inline -unsigned char *choose_random_location(unsigned long input_ptr, - unsigned long input_size, - unsigned long output_ptr, - unsigned long output_size) +static inline void choose_random_location(unsigned long input, + unsigned long input_size, + unsigned long *output, + unsigned long output_size, + unsigned long *virt_addr) { - return (unsigned char *)output_ptr; + /* No change from existing output location. */ + *virt_addr = *output; } #endif #ifdef CONFIG_X86_64 +void initialize_identity_maps(void); void add_identity_map(unsigned long start, unsigned long size); void finalize_identity_maps(void); extern unsigned char _pgtable[]; #else +static inline void initialize_identity_maps(void) +{ } static inline void add_identity_map(unsigned long start, unsigned long size) { } static inline void finalize_identity_maps(void) diff --git a/arch/x86/boot/compressed/pagetable.c b/arch/x86/boot/compressed/pagetable.c index 34b95df14e69..56589d0a804b 100644 --- a/arch/x86/boot/compressed/pagetable.c +++ b/arch/x86/boot/compressed/pagetable.c @@ -2,6 +2,9 @@ * This code is used on x86_64 to create page table identity mappings on * demand by building up a new set of page tables (or appending to the * existing ones), and then switching over to them when ready. + * + * Copyright (C) 2015-2016 Yinghai Lu + * Copyright (C) 2016 Kees Cook */ /* @@ -17,6 +20,9 @@ /* These actually do the work of building the kernel identity maps. */ #include #include +/* Use the static base for this part of the boot process */ +#undef __PAGE_OFFSET +#define __PAGE_OFFSET __PAGE_OFFSET_BASE #include "../../mm/ident_map.c" /* Used by pgtable.h asm code to force instruction serialization. */ @@ -59,9 +65,21 @@ static struct alloc_pgt_data pgt_data; /* The top level page table entry pointer. */ static unsigned long level4p; +/* + * Mapping information structure passed to kernel_ident_mapping_init(). + * Due to relocation, pointers must be assigned at run time not build time. + */ +static struct x86_mapping_info mapping_info = { + .pmd_flag = __PAGE_KERNEL_LARGE_EXEC, +}; + /* Locates and clears a region for a new top level page table. */ -static void prepare_level4(void) +void initialize_identity_maps(void) { + /* Init mapping_info with run-time function/buffer pointers. */ + mapping_info.alloc_pgt_page = alloc_pgt_page; + mapping_info.context = &pgt_data; + /* * It should be impossible for this not to already be true, * but since calling this a second time would rewind the other @@ -96,17 +114,8 @@ static void prepare_level4(void) */ void add_identity_map(unsigned long start, unsigned long size) { - struct x86_mapping_info mapping_info = { - .alloc_pgt_page = alloc_pgt_page, - .context = &pgt_data, - .pmd_flag = __PAGE_KERNEL_LARGE_EXEC, - }; unsigned long end = start + size; - /* Make sure we have a top level page table ready to use. */ - if (!level4p) - prepare_level4(); - /* Align boundary to 2M. */ start = round_down(start, PMD_SIZE); end = round_up(end, PMD_SIZE); diff --git a/arch/x86/boot/cpu.c b/arch/x86/boot/cpu.c index 29207f69ae8c..26240dde081e 100644 --- a/arch/x86/boot/cpu.c +++ b/arch/x86/boot/cpu.c @@ -93,6 +93,8 @@ int validate_cpu(void) show_cap_strs(err_flags); putchar('\n'); return -1; + } else if (check_knl_erratum()) { + return -1; } else { return 0; } diff --git a/arch/x86/boot/cpucheck.c b/arch/x86/boot/cpucheck.c index 1fd7d575092e..4ad7d70e8739 100644 --- a/arch/x86/boot/cpucheck.c +++ b/arch/x86/boot/cpucheck.c @@ -24,6 +24,7 @@ # include "boot.h" #endif #include +#include #include #include #include @@ -175,6 +176,8 @@ int check_cpu(int *cpu_level_ptr, int *req_level_ptr, u32 **err_flags_ptr) puts("WARNING: PAE disabled. Use parameter 'forcepae' to enable at your own risk!\n"); } } + if (!err) + err = check_knl_erratum(); if (err_flags_ptr) *err_flags_ptr = err ? err_flags : NULL; @@ -185,3 +188,33 @@ int check_cpu(int *cpu_level_ptr, int *req_level_ptr, u32 **err_flags_ptr) return (cpu.level < req_level || err) ? -1 : 0; } + +int check_knl_erratum(void) +{ + /* + * First check for the affected model/family: + */ + if (!is_intel() || + cpu.family != 6 || + cpu.model != INTEL_FAM6_XEON_PHI_KNL) + return 0; + + /* + * This erratum affects the Accessed/Dirty bits, and can + * cause stray bits to be set in !Present PTEs. We have + * enough bits in our 64-bit PTEs (which we have on real + * 64-bit mode or PAE) to avoid using these troublesome + * bits. But, we do not have enough space in our 32-bit + * PTEs. So, refuse to run on 32-bit non-PAE kernels. + */ + if (IS_ENABLED(CONFIG_X86_64) || IS_ENABLED(CONFIG_X86_PAE)) + return 0; + + puts("This 32-bit kernel can not run on this Xeon Phi x200\n" + "processor due to a processor erratum. Use a 64-bit\n" + "kernel, or enable PAE in this 32-bit kernel.\n\n"); + + return -1; +} + + diff --git a/arch/x86/boot/cpuflags.c b/arch/x86/boot/cpuflags.c index 431fa5f84537..6687ab953257 100644 --- a/arch/x86/boot/cpuflags.c +++ b/arch/x86/boot/cpuflags.c @@ -102,6 +102,7 @@ void get_cpuflags(void) cpuid(0x1, &tfms, &ignored, &cpu.flags[4], &cpu.flags[0]); cpu.level = (tfms >> 8) & 15; + cpu.family = cpu.level; cpu.model = (tfms >> 4) & 15; if (cpu.level >= 6) cpu.model += ((tfms >> 16) & 0xf) << 4; diff --git a/arch/x86/boot/cpuflags.h b/arch/x86/boot/cpuflags.h index 4cb404fd45ce..15ad56a3f905 100644 --- a/arch/x86/boot/cpuflags.h +++ b/arch/x86/boot/cpuflags.h @@ -6,6 +6,7 @@ struct cpu_features { int level; /* Family, or 64 for x86-64 */ + int family; /* Family, always */ int model; u32 flags[NCAPINTS]; }; diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c index 318b8465d302..cc3bd583dce1 100644 --- a/arch/x86/boot/string.c +++ b/arch/x86/boot/string.c @@ -17,7 +17,7 @@ int memcmp(const void *s1, const void *s2, size_t len) { - u8 diff; + bool diff; asm("repe; cmpsb; setnz %0" : "=qm" (diff), "+D" (s1), "+S" (s2), "+c" (len)); return diff; diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index b9b912a44d61..34b3fa2889d1 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -49,7 +49,9 @@ endif ifeq ($(avx2_supported),yes) obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64) += camellia-aesni-avx2.o obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o - obj-$(CONFIG_CRYPTO_SHA1_MB) += sha-mb/ + obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb/ + obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb/ + obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb/ endif aes-i586-y := aes-i586-asm_32.o aes_glue.o diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 5b7fa1471007..0ab5ee1c26af 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -59,17 +59,6 @@ struct aesni_rfc4106_gcm_ctx { u8 nonce[4]; }; -struct aesni_gcm_set_hash_subkey_result { - int err; - struct completion completion; -}; - -struct aesni_hash_subkey_req_data { - u8 iv[16]; - struct aesni_gcm_set_hash_subkey_result result; - struct scatterlist sg; -}; - struct aesni_lrw_ctx { struct lrw_table_ctx lrw_table; u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; @@ -809,71 +798,28 @@ static void rfc4106_exit(struct crypto_aead *aead) cryptd_free_aead(*ctx); } -static void -rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) -{ - struct aesni_gcm_set_hash_subkey_result *result = req->data; - - if (err == -EINPROGRESS) - return; - result->err = err; - complete(&result->completion); -} - static int rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) { - struct crypto_ablkcipher *ctr_tfm; - struct ablkcipher_request *req; - int ret = -EINVAL; - struct aesni_hash_subkey_req_data *req_data; + struct crypto_cipher *tfm; + int ret; - ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); - if (IS_ERR(ctr_tfm)) - return PTR_ERR(ctr_tfm); + tfm = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); - ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); + ret = crypto_cipher_setkey(tfm, key, key_len); if (ret) - goto out_free_ablkcipher; - - ret = -ENOMEM; - req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); - if (!req) - goto out_free_ablkcipher; - - req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); - if (!req_data) - goto out_free_request; - - memset(req_data->iv, 0, sizeof(req_data->iv)); + goto out_free_cipher; /* Clear the data in the hash sub key container to zero.*/ /* We want to cipher all zeros to create the hash sub key. */ memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); - init_completion(&req_data->result.completion); - sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); - ablkcipher_request_set_tfm(req, ctr_tfm); - ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | - CRYPTO_TFM_REQ_MAY_BACKLOG, - rfc4106_set_hash_subkey_done, - &req_data->result); - - ablkcipher_request_set_crypt(req, &req_data->sg, - &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); - - ret = crypto_ablkcipher_encrypt(req); - if (ret == -EINPROGRESS || ret == -EBUSY) { - ret = wait_for_completion_interruptible - (&req_data->result.completion); - if (!ret) - ret = req_data->result.err; - } - kfree(req_data); -out_free_request: - ablkcipher_request_free(req); -out_free_ablkcipher: - crypto_free_ablkcipher(ctr_tfm); + crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey); + +out_free_cipher: + crypto_free_cipher(tfm); return ret; } @@ -1098,9 +1044,12 @@ static int rfc4106_encrypt(struct aead_request *req) struct cryptd_aead **ctx = crypto_aead_ctx(tfm); struct cryptd_aead *cryptd_tfm = *ctx; - aead_request_set_tfm(req, irq_fpu_usable() ? - cryptd_aead_child(cryptd_tfm) : - &cryptd_tfm->base); + tfm = &cryptd_tfm->base; + if (irq_fpu_usable() && (!in_atomic() || + !cryptd_aead_queued(cryptd_tfm))) + tfm = cryptd_aead_child(cryptd_tfm); + + aead_request_set_tfm(req, tfm); return crypto_aead_encrypt(req); } @@ -1111,9 +1060,12 @@ static int rfc4106_decrypt(struct aead_request *req) struct cryptd_aead **ctx = crypto_aead_ctx(tfm); struct cryptd_aead *cryptd_tfm = *ctx; - aead_request_set_tfm(req, irq_fpu_usable() ? - cryptd_aead_child(cryptd_tfm) : - &cryptd_tfm->base); + tfm = &cryptd_tfm->base; + if (irq_fpu_usable() && (!in_atomic() || + !cryptd_aead_queued(cryptd_tfm))) + tfm = cryptd_aead_child(cryptd_tfm); + + aead_request_set_tfm(req, tfm); return crypto_aead_decrypt(req); } diff --git a/arch/x86/crypto/chacha20_glue.c b/arch/x86/crypto/chacha20_glue.c index 2d5c2e0bd939..f910d1d449f0 100644 --- a/arch/x86/crypto/chacha20_glue.c +++ b/arch/x86/crypto/chacha20_glue.c @@ -70,7 +70,7 @@ static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst, struct blkcipher_walk walk; int err; - if (!may_use_simd()) + if (nbytes <= CHACHA20_BLOCK_SIZE || !may_use_simd()) return crypto_chacha20_crypt(desc, dst, src, nbytes); state = (u32 *)roundup((uintptr_t)state_buf, CHACHA20_STATE_ALIGN); diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index a69321a77783..0420bab19efb 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -168,30 +168,23 @@ static int ghash_async_init(struct ahash_request *req) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); struct ahash_request *cryptd_req = ahash_request_ctx(req); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; + struct shash_desc *desc = cryptd_shash_desc(cryptd_req); + struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); - if (!irq_fpu_usable()) { - memcpy(cryptd_req, req, sizeof(*req)); - ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); - return crypto_ahash_init(cryptd_req); - } else { - struct shash_desc *desc = cryptd_shash_desc(cryptd_req); - struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); - - desc->tfm = child; - desc->flags = req->base.flags; - return crypto_shash_init(desc); - } + desc->tfm = child; + desc->flags = req->base.flags; + return crypto_shash_init(desc); } static int ghash_async_update(struct ahash_request *req) { struct ahash_request *cryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!irq_fpu_usable()) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - + if (!irq_fpu_usable() || + (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); return crypto_ahash_update(cryptd_req); @@ -204,12 +197,12 @@ static int ghash_async_update(struct ahash_request *req) static int ghash_async_final(struct ahash_request *req) { struct ahash_request *cryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!irq_fpu_usable()) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); - struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - + if (!irq_fpu_usable() || + (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); return crypto_ahash_final(cryptd_req); @@ -249,7 +242,8 @@ static int ghash_async_digest(struct ahash_request *req) struct ahash_request *cryptd_req = ahash_request_ctx(req); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!irq_fpu_usable()) { + if (!irq_fpu_usable() || + (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); return crypto_ahash_digest(cryptd_req); diff --git a/arch/x86/crypto/sha-mb/Makefile b/arch/x86/crypto/sha1-mb/Makefile similarity index 100% rename from arch/x86/crypto/sha-mb/Makefile rename to arch/x86/crypto/sha1-mb/Makefile diff --git a/arch/x86/crypto/sha-mb/sha1_mb.c b/arch/x86/crypto/sha1-mb/sha1_mb.c similarity index 81% rename from arch/x86/crypto/sha-mb/sha1_mb.c rename to arch/x86/crypto/sha1-mb/sha1_mb.c index 9c5af331a956..9e5b67127a09 100644 --- a/arch/x86/crypto/sha-mb/sha1_mb.c +++ b/arch/x86/crypto/sha1-mb/sha1_mb.c @@ -67,7 +67,7 @@ #include #include #include -#include "sha_mb_ctx.h" +#include "sha1_mb_ctx.h" #define FLUSH_INTERVAL 1000 /* in usec */ @@ -77,30 +77,34 @@ struct sha1_mb_ctx { struct mcryptd_ahash *mcryptd_tfm; }; -static inline struct mcryptd_hash_request_ctx *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx) +static inline struct mcryptd_hash_request_ctx + *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx) { - struct shash_desc *desc; + struct ahash_request *areq; - desc = container_of((void *) hash_ctx, struct shash_desc, __ctx); - return container_of(desc, struct mcryptd_hash_request_ctx, desc); + areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); + return container_of(areq, struct mcryptd_hash_request_ctx, areq); } -static inline struct ahash_request *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) +static inline struct ahash_request + *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) { return container_of((void *) ctx, struct ahash_request, __ctx); } static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, - struct shash_desc *desc) + struct ahash_request *areq) { rctx->flag = HASH_UPDATE; } static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state); -static asmlinkage struct job_sha1* (*sha1_job_mgr_submit)(struct sha1_mb_mgr *state, - struct job_sha1 *job); -static asmlinkage struct job_sha1* (*sha1_job_mgr_flush)(struct sha1_mb_mgr *state); -static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)(struct sha1_mb_mgr *state); +static asmlinkage struct job_sha1* (*sha1_job_mgr_submit) + (struct sha1_mb_mgr *state, struct job_sha1 *job); +static asmlinkage struct job_sha1* (*sha1_job_mgr_flush) + (struct sha1_mb_mgr *state); +static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job) + (struct sha1_mb_mgr *state); static inline void sha1_init_digest(uint32_t *digest) { @@ -131,7 +135,8 @@ static inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2], return i >> SHA1_LOG2_BLOCK_SIZE; } -static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, struct sha1_hash_ctx *ctx) +static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, + struct sha1_hash_ctx *ctx) { while (ctx) { if (ctx->status & HASH_CTX_STS_COMPLETE) { @@ -177,8 +182,8 @@ static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, str ctx->job.buffer = (uint8_t *) buffer; ctx->job.len = len; - ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, - &ctx->job); + ctx = (struct sha1_hash_ctx *)sha1_job_mgr_submit(&mgr->mgr, + &ctx->job); continue; } } @@ -191,13 +196,15 @@ static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, str if (ctx->status & HASH_CTX_STS_LAST) { uint8_t *buf = ctx->partial_block_buffer; - uint32_t n_extra_blocks = sha1_pad(buf, ctx->total_length); + uint32_t n_extra_blocks = + sha1_pad(buf, ctx->total_length); ctx->status = (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_COMPLETE); ctx->job.buffer = buf; ctx->job.len = (uint32_t) n_extra_blocks; - ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job); + ctx = (struct sha1_hash_ctx *) + sha1_job_mgr_submit(&mgr->mgr, &ctx->job); continue; } @@ -208,14 +215,17 @@ static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, str return NULL; } -static struct sha1_hash_ctx *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr) +static struct sha1_hash_ctx + *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr) { /* * If get_comp_job returns NULL, there are no jobs complete. - * If get_comp_job returns a job, verify that it is safe to return to the user. + * If get_comp_job returns a job, verify that it is safe to return to + * the user. * If it is not ready, resubmit the job to finish processing. * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. - * Otherwise, all jobs currently being managed by the hash_ctx_mgr still need processing. + * Otherwise, all jobs currently being managed by the hash_ctx_mgr + * still need processing. */ struct sha1_hash_ctx *ctx; @@ -235,7 +245,10 @@ static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr, int flags) { if (flags & (~HASH_ENTIRE)) { - /* User should not pass anything other than FIRST, UPDATE, or LAST */ + /* + * User should not pass anything other than FIRST, UPDATE, or + * LAST + */ ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; return ctx; } @@ -264,14 +277,20 @@ static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr, ctx->partial_block_buffer_length = 0; } - /* If we made it here, there were no errors during this call to submit */ + /* + * If we made it here, there were no errors during this call to + * submit + */ ctx->error = HASH_CTX_ERROR_NONE; /* Store buffer ptr info from user */ ctx->incoming_buffer = buffer; ctx->incoming_buffer_length = len; - /* Store the user's request flags and mark this ctx as currently being processed. */ + /* + * Store the user's request flags and mark this ctx as currently + * being processed. + */ ctx->status = (flags & HASH_LAST) ? (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : HASH_CTX_STS_PROCESSING; @@ -285,9 +304,13 @@ static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr, * Or if the user's buffer contains less than a whole block, * append as much as possible to the extra block. */ - if ((ctx->partial_block_buffer_length) | (len < SHA1_BLOCK_SIZE)) { - /* Compute how many bytes to copy from user buffer into extra block */ - uint32_t copy_len = SHA1_BLOCK_SIZE - ctx->partial_block_buffer_length; + if (ctx->partial_block_buffer_length || len < SHA1_BLOCK_SIZE) { + /* + * Compute how many bytes to copy from user buffer into + * extra block + */ + uint32_t copy_len = SHA1_BLOCK_SIZE - + ctx->partial_block_buffer_length; if (len < copy_len) copy_len = len; @@ -297,20 +320,28 @@ static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr, buffer, copy_len); ctx->partial_block_buffer_length += copy_len; - ctx->incoming_buffer = (const void *)((const char *)buffer + copy_len); + ctx->incoming_buffer = (const void *) + ((const char *)buffer + copy_len); ctx->incoming_buffer_length = len - copy_len; } - /* The extra block should never contain more than 1 block here */ + /* + * The extra block should never contain more than 1 block + * here + */ assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE); - /* If the extra block buffer contains exactly 1 block, it can be hashed. */ + /* + * If the extra block buffer contains exactly 1 block, it can + * be hashed. + */ if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) { ctx->partial_block_buffer_length = 0; ctx->job.buffer = ctx->partial_block_buffer; ctx->job.len = 1; - ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job); + ctx = (struct sha1_hash_ctx *) + sha1_job_mgr_submit(&mgr->mgr, &ctx->job); } } @@ -329,23 +360,24 @@ static struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr) return NULL; /* - * If flush returned a job, resubmit the job to finish processing. + * If flush returned a job, resubmit the job to finish + * processing. */ ctx = sha1_ctx_mgr_resubmit(mgr, ctx); /* - * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. - * Otherwise, all jobs currently being managed by the sha1_ctx_mgr - * still need processing. Loop. + * If sha1_ctx_mgr_resubmit returned a job, it is ready to be + * returned. Otherwise, all jobs currently being managed by the + * sha1_ctx_mgr still need processing. Loop. */ if (ctx) return ctx; } } -static int sha1_mb_init(struct shash_desc *desc) +static int sha1_mb_init(struct ahash_request *areq) { - struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); + struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); hash_ctx_init(sctx); sctx->job.result_digest[0] = SHA1_H0; @@ -363,7 +395,7 @@ static int sha1_mb_init(struct shash_desc *desc) static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx) { int i; - struct sha1_hash_ctx *sctx = shash_desc_ctx(&rctx->desc); + struct sha1_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); __be32 *dst = (__be32 *) rctx->out; for (i = 0; i < 5; ++i) @@ -394,9 +426,11 @@ static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, flag |= HASH_LAST; } - sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(&rctx->desc); + sha_ctx = (struct sha1_hash_ctx *) + ahash_request_ctx(&rctx->areq); kernel_fpu_begin(); - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag); + sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, + rctx->walk.data, nbytes, flag); if (!sha_ctx) { if (flush) sha_ctx = sha1_ctx_mgr_flush(cstate->mgr); @@ -485,11 +519,10 @@ static void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx, mcryptd_arm_flusher(cstate, delay); } -static int sha1_mb_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha1_mb_update(struct ahash_request *areq) { struct mcryptd_hash_request_ctx *rctx = - container_of(desc, struct mcryptd_hash_request_ctx, desc); + container_of(areq, struct mcryptd_hash_request_ctx, areq); struct mcryptd_alg_cstate *cstate = this_cpu_ptr(sha1_mb_alg_state.alg_cstate); @@ -505,7 +538,7 @@ static int sha1_mb_update(struct shash_desc *desc, const u8 *data, } /* need to init context */ - req_ctx_init(rctx, desc); + req_ctx_init(rctx, areq); nbytes = crypto_ahash_walk_first(req, &rctx->walk); @@ -518,10 +551,11 @@ static int sha1_mb_update(struct shash_desc *desc, const u8 *data, rctx->flag |= HASH_DONE; /* submit */ - sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); + sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); sha1_mb_add_list(rctx, cstate); kernel_fpu_begin(); - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, HASH_UPDATE); + sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, + nbytes, HASH_UPDATE); kernel_fpu_end(); /* check if anything is returned */ @@ -544,11 +578,10 @@ done: return ret; } -static int sha1_mb_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int sha1_mb_finup(struct ahash_request *areq) { struct mcryptd_hash_request_ctx *rctx = - container_of(desc, struct mcryptd_hash_request_ctx, desc); + container_of(areq, struct mcryptd_hash_request_ctx, areq); struct mcryptd_alg_cstate *cstate = this_cpu_ptr(sha1_mb_alg_state.alg_cstate); @@ -563,7 +596,7 @@ static int sha1_mb_finup(struct shash_desc *desc, const u8 *data, } /* need to init context */ - req_ctx_init(rctx, desc); + req_ctx_init(rctx, areq); nbytes = crypto_ahash_walk_first(req, &rctx->walk); @@ -576,15 +609,15 @@ static int sha1_mb_finup(struct shash_desc *desc, const u8 *data, rctx->flag |= HASH_DONE; flag = HASH_LAST; } - rctx->out = out; /* submit */ rctx->flag |= HASH_FINAL; - sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); + sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); sha1_mb_add_list(rctx, cstate); kernel_fpu_begin(); - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag); + sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, + nbytes, flag); kernel_fpu_end(); /* check if anything is returned */ @@ -605,10 +638,10 @@ done: return ret; } -static int sha1_mb_final(struct shash_desc *desc, u8 *out) +static int sha1_mb_final(struct ahash_request *areq) { struct mcryptd_hash_request_ctx *rctx = - container_of(desc, struct mcryptd_hash_request_ctx, desc); + container_of(areq, struct mcryptd_hash_request_ctx, areq); struct mcryptd_alg_cstate *cstate = this_cpu_ptr(sha1_mb_alg_state.alg_cstate); @@ -623,16 +656,16 @@ static int sha1_mb_final(struct shash_desc *desc, u8 *out) } /* need to init context */ - req_ctx_init(rctx, desc); + req_ctx_init(rctx, areq); - rctx->out = out; rctx->flag |= HASH_DONE | HASH_FINAL; - sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); + sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); /* flag HASH_FINAL and 0 data size */ sha1_mb_add_list(rctx, cstate); kernel_fpu_begin(); - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, HASH_LAST); + sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, + HASH_LAST); kernel_fpu_end(); /* check if anything is returned */ @@ -654,48 +687,98 @@ done: return ret; } -static int sha1_mb_export(struct shash_desc *desc, void *out) +static int sha1_mb_export(struct ahash_request *areq, void *out) { - struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); + struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); memcpy(out, sctx, sizeof(*sctx)); return 0; } -static int sha1_mb_import(struct shash_desc *desc, const void *in) +static int sha1_mb_import(struct ahash_request *areq, const void *in) { - struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); + struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); memcpy(sctx, in, sizeof(*sctx)); return 0; } +static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm) +{ + struct mcryptd_ahash *mcryptd_tfm; + struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); + struct mcryptd_hash_ctx *mctx; -static struct shash_alg sha1_mb_shash_alg = { - .digestsize = SHA1_DIGEST_SIZE, + mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", + CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); + if (IS_ERR(mcryptd_tfm)) + return PTR_ERR(mcryptd_tfm); + mctx = crypto_ahash_ctx(&mcryptd_tfm->base); + mctx->alg_state = &sha1_mb_alg_state; + ctx->mcryptd_tfm = mcryptd_tfm; + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ahash_request) + + crypto_ahash_reqsize(&mcryptd_tfm->base)); + + return 0; +} + +static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm) +{ + struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); + + mcryptd_free_ahash(ctx->mcryptd_tfm); +} + +static int sha1_mb_areq_init_tfm(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ahash_request) + + sizeof(struct sha1_hash_ctx)); + + return 0; +} + +static void sha1_mb_areq_exit_tfm(struct crypto_tfm *tfm) +{ + struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); + + mcryptd_free_ahash(ctx->mcryptd_tfm); +} + +static struct ahash_alg sha1_mb_areq_alg = { .init = sha1_mb_init, .update = sha1_mb_update, .final = sha1_mb_final, .finup = sha1_mb_finup, .export = sha1_mb_export, .import = sha1_mb_import, - .descsize = sizeof(struct sha1_hash_ctx), - .statesize = sizeof(struct sha1_hash_ctx), - .base = { - .cra_name = "__sha1-mb", - .cra_driver_name = "__intel_sha1-mb", - .cra_priority = 100, - /* - * use ASYNC flag as some buffers in multi-buffer - * algo may not have completed before hashing thread sleep - */ - .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_ASYNC | - CRYPTO_ALG_INTERNAL, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(sha1_mb_shash_alg.base.cra_list), + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_hash_ctx), + .base = { + .cra_name = "__sha1-mb", + .cra_driver_name = "__intel_sha1-mb", + .cra_priority = 100, + /* + * use ASYNC flag as some buffers in multi-buffer + * algo may not have completed before hashing thread + * sleep + */ + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_INTERNAL, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT + (sha1_mb_areq_alg.halg.base.cra_list), + .cra_init = sha1_mb_areq_init_tfm, + .cra_exit = sha1_mb_areq_exit_tfm, + .cra_ctxsize = sizeof(struct sha1_hash_ctx), + } } }; @@ -780,46 +863,20 @@ static int sha1_mb_async_import(struct ahash_request *req, const void *in) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; - struct crypto_shash *child = mcryptd_ahash_child(mcryptd_tfm); + struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); struct mcryptd_hash_request_ctx *rctx; - struct shash_desc *desc; + struct ahash_request *areq; memcpy(mcryptd_req, req, sizeof(*req)); ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); rctx = ahash_request_ctx(mcryptd_req); - desc = &rctx->desc; - desc->tfm = child; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - - return crypto_ahash_import(mcryptd_req, in); -} - -static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm) -{ - struct mcryptd_ahash *mcryptd_tfm; - struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); - struct mcryptd_hash_ctx *mctx; + areq = &rctx->areq; - mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", - CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(mcryptd_tfm)) - return PTR_ERR(mcryptd_tfm); - mctx = crypto_ahash_ctx(&mcryptd_tfm->base); - mctx->alg_state = &sha1_mb_alg_state; - ctx->mcryptd_tfm = mcryptd_tfm; - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct ahash_request) + - crypto_ahash_reqsize(&mcryptd_tfm->base)); + ahash_request_set_tfm(areq, child); + ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, + rctx->complete, req); - return 0; -} - -static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm) -{ - struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); - - mcryptd_free_ahash(ctx->mcryptd_tfm); + return crypto_ahash_import(mcryptd_req, in); } static struct ahash_alg sha1_mb_async_alg = { @@ -866,7 +923,8 @@ static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate) if (time_before(cur_time, rctx->tag.expire)) break; kernel_fpu_begin(); - sha_ctx = (struct sha1_hash_ctx *) sha1_ctx_mgr_flush(cstate->mgr); + sha_ctx = (struct sha1_hash_ctx *) + sha1_ctx_mgr_flush(cstate->mgr); kernel_fpu_end(); if (!sha_ctx) { pr_err("sha1_mb error: nothing got flushed for non-empty list\n"); @@ -927,7 +985,7 @@ static int __init sha1_mb_mod_init(void) } sha1_mb_alg_state.flusher = &sha1_mb_flusher; - err = crypto_register_shash(&sha1_mb_shash_alg); + err = crypto_register_ahash(&sha1_mb_areq_alg); if (err) goto err2; err = crypto_register_ahash(&sha1_mb_async_alg); @@ -937,7 +995,7 @@ static int __init sha1_mb_mod_init(void) return 0; err1: - crypto_unregister_shash(&sha1_mb_shash_alg); + crypto_unregister_ahash(&sha1_mb_areq_alg); err2: for_each_possible_cpu(cpu) { cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); @@ -953,7 +1011,7 @@ static void __exit sha1_mb_mod_fini(void) struct mcryptd_alg_cstate *cpu_state; crypto_unregister_ahash(&sha1_mb_async_alg); - crypto_unregister_shash(&sha1_mb_shash_alg); + crypto_unregister_ahash(&sha1_mb_areq_alg); for_each_possible_cpu(cpu) { cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); kfree(cpu_state->mgr); diff --git a/arch/x86/crypto/sha-mb/sha_mb_ctx.h b/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h similarity index 99% rename from arch/x86/crypto/sha-mb/sha_mb_ctx.h rename to arch/x86/crypto/sha1-mb/sha1_mb_ctx.h index e36069d0c1bd..98a35bcc6f4a 100644 --- a/arch/x86/crypto/sha-mb/sha_mb_ctx.h +++ b/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h @@ -54,7 +54,7 @@ #ifndef _SHA_MB_CTX_INTERNAL_H #define _SHA_MB_CTX_INTERNAL_H -#include "sha_mb_mgr.h" +#include "sha1_mb_mgr.h" #define HASH_UPDATE 0x00 #define HASH_FIRST 0x01 diff --git a/arch/x86/crypto/sha-mb/sha_mb_mgr.h b/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h similarity index 100% rename from arch/x86/crypto/sha-mb/sha_mb_mgr.h rename to arch/x86/crypto/sha1-mb/sha1_mb_mgr.h diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_datastruct.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S similarity index 100% rename from arch/x86/crypto/sha-mb/sha1_mb_mgr_datastruct.S rename to arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S similarity index 100% rename from arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S rename to arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c similarity index 99% rename from arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c rename to arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c index 822acb5b464c..d2add0d35f43 100644 --- a/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c +++ b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c @@ -51,7 +51,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "sha_mb_mgr.h" +#include "sha1_mb_mgr.h" void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state) { diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S similarity index 100% rename from arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S rename to arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S diff --git a/arch/x86/crypto/sha-mb/sha1_x8_avx2.S b/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S similarity index 100% rename from arch/x86/crypto/sha-mb/sha1_x8_avx2.S rename to arch/x86/crypto/sha1-mb/sha1_x8_avx2.S diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 1024e378a358..fc61739150e7 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -374,3 +374,9 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated"); MODULE_ALIAS_CRYPTO("sha1"); +MODULE_ALIAS_CRYPTO("sha1-ssse3"); +MODULE_ALIAS_CRYPTO("sha1-avx"); +MODULE_ALIAS_CRYPTO("sha1-avx2"); +#ifdef CONFIG_AS_SHA1_NI +MODULE_ALIAS_CRYPTO("sha1-ni"); +#endif diff --git a/arch/x86/crypto/sha256-mb/Makefile b/arch/x86/crypto/sha256-mb/Makefile new file mode 100644 index 000000000000..41089e7c400c --- /dev/null +++ b/arch/x86/crypto/sha256-mb/Makefile @@ -0,0 +1,11 @@ +# +# Arch-specific CryptoAPI modules. +# + +avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ + $(comma)4)$(comma)%ymm2,yes,no) +ifeq ($(avx2_supported),yes) + obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb.o + sha256-mb-y := sha256_mb.o sha256_mb_mgr_flush_avx2.o \ + sha256_mb_mgr_init_avx2.o sha256_mb_mgr_submit_avx2.o sha256_x8_avx2.o +endif diff --git a/arch/x86/crypto/sha256-mb/sha256_mb.c b/arch/x86/crypto/sha256-mb/sha256_mb.c new file mode 100644 index 000000000000..89fa85e8b10c --- /dev/null +++ b/arch/x86/crypto/sha256-mb/sha256_mb.c @@ -0,0 +1,1030 @@ +/* + * Multi buffer SHA256 algorithm Glue Code + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sha256_mb_ctx.h" + +#define FLUSH_INTERVAL 1000 /* in usec */ + +static struct mcryptd_alg_state sha256_mb_alg_state; + +struct sha256_mb_ctx { + struct mcryptd_ahash *mcryptd_tfm; +}; + +static inline struct mcryptd_hash_request_ctx + *cast_hash_to_mcryptd_ctx(struct sha256_hash_ctx *hash_ctx) +{ + struct ahash_request *areq; + + areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); + return container_of(areq, struct mcryptd_hash_request_ctx, areq); +} + +static inline struct ahash_request + *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) +{ + return container_of((void *) ctx, struct ahash_request, __ctx); +} + +static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, + struct ahash_request *areq) +{ + rctx->flag = HASH_UPDATE; +} + +static asmlinkage void (*sha256_job_mgr_init)(struct sha256_mb_mgr *state); +static asmlinkage struct job_sha256* (*sha256_job_mgr_submit) + (struct sha256_mb_mgr *state, struct job_sha256 *job); +static asmlinkage struct job_sha256* (*sha256_job_mgr_flush) + (struct sha256_mb_mgr *state); +static asmlinkage struct job_sha256* (*sha256_job_mgr_get_comp_job) + (struct sha256_mb_mgr *state); + +inline void sha256_init_digest(uint32_t *digest) +{ + static const uint32_t initial_digest[SHA256_DIGEST_LENGTH] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7}; + memcpy(digest, initial_digest, sizeof(initial_digest)); +} + +inline uint32_t sha256_pad(uint8_t padblock[SHA256_BLOCK_SIZE * 2], + uint32_t total_len) +{ + uint32_t i = total_len & (SHA256_BLOCK_SIZE - 1); + + memset(&padblock[i], 0, SHA256_BLOCK_SIZE); + padblock[i] = 0x80; + + i += ((SHA256_BLOCK_SIZE - 1) & + (0 - (total_len + SHA256_PADLENGTHFIELD_SIZE + 1))) + + 1 + SHA256_PADLENGTHFIELD_SIZE; + +#if SHA256_PADLENGTHFIELD_SIZE == 16 + *((uint64_t *) &padblock[i - 16]) = 0; +#endif + + *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); + + /* Number of extra blocks to hash */ + return i >> SHA256_LOG2_BLOCK_SIZE; +} + +static struct sha256_hash_ctx + *sha256_ctx_mgr_resubmit(struct sha256_ctx_mgr *mgr, + struct sha256_hash_ctx *ctx) +{ + while (ctx) { + if (ctx->status & HASH_CTX_STS_COMPLETE) { + /* Clear PROCESSING bit */ + ctx->status = HASH_CTX_STS_COMPLETE; + return ctx; + } + + /* + * If the extra blocks are empty, begin hashing what remains + * in the user's buffer. + */ + if (ctx->partial_block_buffer_length == 0 && + ctx->incoming_buffer_length) { + + const void *buffer = ctx->incoming_buffer; + uint32_t len = ctx->incoming_buffer_length; + uint32_t copy_len; + + /* + * Only entire blocks can be hashed. + * Copy remainder to extra blocks buffer. + */ + copy_len = len & (SHA256_BLOCK_SIZE-1); + + if (copy_len) { + len -= copy_len; + memcpy(ctx->partial_block_buffer, + ((const char *) buffer + len), + copy_len); + ctx->partial_block_buffer_length = copy_len; + } + + ctx->incoming_buffer_length = 0; + + /* len should be a multiple of the block size now */ + assert((len % SHA256_BLOCK_SIZE) == 0); + + /* Set len to the number of blocks to be hashed */ + len >>= SHA256_LOG2_BLOCK_SIZE; + + if (len) { + + ctx->job.buffer = (uint8_t *) buffer; + ctx->job.len = len; + ctx = (struct sha256_hash_ctx *) + sha256_job_mgr_submit(&mgr->mgr, &ctx->job); + continue; + } + } + + /* + * If the extra blocks are not empty, then we are + * either on the last block(s) or we need more + * user input before continuing. + */ + if (ctx->status & HASH_CTX_STS_LAST) { + + uint8_t *buf = ctx->partial_block_buffer; + uint32_t n_extra_blocks = + sha256_pad(buf, ctx->total_length); + + ctx->status = (HASH_CTX_STS_PROCESSING | + HASH_CTX_STS_COMPLETE); + ctx->job.buffer = buf; + ctx->job.len = (uint32_t) n_extra_blocks; + ctx = (struct sha256_hash_ctx *) + sha256_job_mgr_submit(&mgr->mgr, &ctx->job); + continue; + } + + ctx->status = HASH_CTX_STS_IDLE; + return ctx; + } + + return NULL; +} + +static struct sha256_hash_ctx + *sha256_ctx_mgr_get_comp_ctx(struct sha256_ctx_mgr *mgr) +{ + /* + * If get_comp_job returns NULL, there are no jobs complete. + * If get_comp_job returns a job, verify that it is safe to return to + * the user. If it is not ready, resubmit the job to finish processing. + * If sha256_ctx_mgr_resubmit returned a job, it is ready to be + * returned. Otherwise, all jobs currently being managed by the + * hash_ctx_mgr still need processing. + */ + struct sha256_hash_ctx *ctx; + + ctx = (struct sha256_hash_ctx *) sha256_job_mgr_get_comp_job(&mgr->mgr); + return sha256_ctx_mgr_resubmit(mgr, ctx); +} + +static void sha256_ctx_mgr_init(struct sha256_ctx_mgr *mgr) +{ + sha256_job_mgr_init(&mgr->mgr); +} + +static struct sha256_hash_ctx *sha256_ctx_mgr_submit(struct sha256_ctx_mgr *mgr, + struct sha256_hash_ctx *ctx, + const void *buffer, + uint32_t len, + int flags) +{ + if (flags & (~HASH_ENTIRE)) { + /* User should not pass anything other than FIRST, UPDATE + * or LAST + */ + ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; + return ctx; + } + + if (ctx->status & HASH_CTX_STS_PROCESSING) { + /* Cannot submit to a currently processing job. */ + ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; + return ctx; + } + + if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) { + /* Cannot update a finished job. */ + ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; + return ctx; + } + + if (flags & HASH_FIRST) { + /* Init digest */ + sha256_init_digest(ctx->job.result_digest); + + /* Reset byte counter */ + ctx->total_length = 0; + + /* Clear extra blocks */ + ctx->partial_block_buffer_length = 0; + } + + /* If we made it here, there was no error during this call to submit */ + ctx->error = HASH_CTX_ERROR_NONE; + + /* Store buffer ptr info from user */ + ctx->incoming_buffer = buffer; + ctx->incoming_buffer_length = len; + + /* + * Store the user's request flags and mark this ctx as currently + * being processed. + */ + ctx->status = (flags & HASH_LAST) ? + (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : + HASH_CTX_STS_PROCESSING; + + /* Advance byte counter */ + ctx->total_length += len; + + /* + * If there is anything currently buffered in the extra blocks, + * append to it until it contains a whole block. + * Or if the user's buffer contains less than a whole block, + * append as much as possible to the extra block. + */ + if (ctx->partial_block_buffer_length || len < SHA256_BLOCK_SIZE) { + /* + * Compute how many bytes to copy from user buffer into + * extra block + */ + uint32_t copy_len = SHA256_BLOCK_SIZE - + ctx->partial_block_buffer_length; + if (len < copy_len) + copy_len = len; + + if (copy_len) { + /* Copy and update relevant pointers and counters */ + memcpy( + &ctx->partial_block_buffer[ctx->partial_block_buffer_length], + buffer, copy_len); + + ctx->partial_block_buffer_length += copy_len; + ctx->incoming_buffer = (const void *) + ((const char *)buffer + copy_len); + ctx->incoming_buffer_length = len - copy_len; + } + + /* The extra block should never contain more than 1 block */ + assert(ctx->partial_block_buffer_length <= SHA256_BLOCK_SIZE); + + /* + * If the extra block buffer contains exactly 1 block, + * it can be hashed. + */ + if (ctx->partial_block_buffer_length >= SHA256_BLOCK_SIZE) { + ctx->partial_block_buffer_length = 0; + + ctx->job.buffer = ctx->partial_block_buffer; + ctx->job.len = 1; + ctx = (struct sha256_hash_ctx *) + sha256_job_mgr_submit(&mgr->mgr, &ctx->job); + } + } + + return sha256_ctx_mgr_resubmit(mgr, ctx); +} + +static struct sha256_hash_ctx *sha256_ctx_mgr_flush(struct sha256_ctx_mgr *mgr) +{ + struct sha256_hash_ctx *ctx; + + while (1) { + ctx = (struct sha256_hash_ctx *) + sha256_job_mgr_flush(&mgr->mgr); + + /* If flush returned 0, there are no more jobs in flight. */ + if (!ctx) + return NULL; + + /* + * If flush returned a job, resubmit the job to finish + * processing. + */ + ctx = sha256_ctx_mgr_resubmit(mgr, ctx); + + /* + * If sha256_ctx_mgr_resubmit returned a job, it is ready to + * be returned. Otherwise, all jobs currently being managed by + * the sha256_ctx_mgr still need processing. Loop. + */ + if (ctx) + return ctx; + } +} + +static int sha256_mb_init(struct ahash_request *areq) +{ + struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); + + hash_ctx_init(sctx); + sctx->job.result_digest[0] = SHA256_H0; + sctx->job.result_digest[1] = SHA256_H1; + sctx->job.result_digest[2] = SHA256_H2; + sctx->job.result_digest[3] = SHA256_H3; + sctx->job.result_digest[4] = SHA256_H4; + sctx->job.result_digest[5] = SHA256_H5; + sctx->job.result_digest[6] = SHA256_H6; + sctx->job.result_digest[7] = SHA256_H7; + sctx->total_length = 0; + sctx->partial_block_buffer_length = 0; + sctx->status = HASH_CTX_STS_IDLE; + + return 0; +} + +static int sha256_mb_set_results(struct mcryptd_hash_request_ctx *rctx) +{ + int i; + struct sha256_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); + __be32 *dst = (__be32 *) rctx->out; + + for (i = 0; i < 8; ++i) + dst[i] = cpu_to_be32(sctx->job.result_digest[i]); + + return 0; +} + +static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, + struct mcryptd_alg_cstate *cstate, bool flush) +{ + int flag = HASH_UPDATE; + int nbytes, err = 0; + struct mcryptd_hash_request_ctx *rctx = *ret_rctx; + struct sha256_hash_ctx *sha_ctx; + + /* more work ? */ + while (!(rctx->flag & HASH_DONE)) { + nbytes = crypto_ahash_walk_done(&rctx->walk, 0); + if (nbytes < 0) { + err = nbytes; + goto out; + } + /* check if the walk is done */ + if (crypto_ahash_walk_last(&rctx->walk)) { + rctx->flag |= HASH_DONE; + if (rctx->flag & HASH_FINAL) + flag |= HASH_LAST; + + } + sha_ctx = (struct sha256_hash_ctx *) + ahash_request_ctx(&rctx->areq); + kernel_fpu_begin(); + sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, + rctx->walk.data, nbytes, flag); + if (!sha_ctx) { + if (flush) + sha_ctx = sha256_ctx_mgr_flush(cstate->mgr); + } + kernel_fpu_end(); + if (sha_ctx) + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + else { + rctx = NULL; + goto out; + } + } + + /* copy the results */ + if (rctx->flag & HASH_FINAL) + sha256_mb_set_results(rctx); + +out: + *ret_rctx = rctx; + return err; +} + +static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, + struct mcryptd_alg_cstate *cstate, + int err) +{ + struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); + struct sha256_hash_ctx *sha_ctx; + struct mcryptd_hash_request_ctx *req_ctx; + int ret; + + /* remove from work list */ + spin_lock(&cstate->work_lock); + list_del(&rctx->waiter); + spin_unlock(&cstate->work_lock); + + if (irqs_disabled()) + rctx->complete(&req->base, err); + else { + local_bh_disable(); + rctx->complete(&req->base, err); + local_bh_enable(); + } + + /* check to see if there are other jobs that are done */ + sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr); + while (sha_ctx) { + req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); + ret = sha_finish_walk(&req_ctx, cstate, false); + if (req_ctx) { + spin_lock(&cstate->work_lock); + list_del(&req_ctx->waiter); + spin_unlock(&cstate->work_lock); + + req = cast_mcryptd_ctx_to_req(req_ctx); + if (irqs_disabled()) + rctx->complete(&req->base, ret); + else { + local_bh_disable(); + rctx->complete(&req->base, ret); + local_bh_enable(); + } + } + sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr); + } + + return 0; +} + +static void sha256_mb_add_list(struct mcryptd_hash_request_ctx *rctx, + struct mcryptd_alg_cstate *cstate) +{ + unsigned long next_flush; + unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); + + /* initialize tag */ + rctx->tag.arrival = jiffies; /* tag the arrival time */ + rctx->tag.seq_num = cstate->next_seq_num++; + next_flush = rctx->tag.arrival + delay; + rctx->tag.expire = next_flush; + + spin_lock(&cstate->work_lock); + list_add_tail(&rctx->waiter, &cstate->work_list); + spin_unlock(&cstate->work_lock); + + mcryptd_arm_flusher(cstate, delay); +} + +static int sha256_mb_update(struct ahash_request *areq) +{ + struct mcryptd_hash_request_ctx *rctx = + container_of(areq, struct mcryptd_hash_request_ctx, areq); + struct mcryptd_alg_cstate *cstate = + this_cpu_ptr(sha256_mb_alg_state.alg_cstate); + + struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); + struct sha256_hash_ctx *sha_ctx; + int ret = 0, nbytes; + + /* sanity check */ + if (rctx->tag.cpu != smp_processor_id()) { + pr_err("mcryptd error: cpu clash\n"); + goto done; + } + + /* need to init context */ + req_ctx_init(rctx, areq); + + nbytes = crypto_ahash_walk_first(req, &rctx->walk); + + if (nbytes < 0) { + ret = nbytes; + goto done; + } + + if (crypto_ahash_walk_last(&rctx->walk)) + rctx->flag |= HASH_DONE; + + /* submit */ + sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); + sha256_mb_add_list(rctx, cstate); + kernel_fpu_begin(); + sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, + nbytes, HASH_UPDATE); + kernel_fpu_end(); + + /* check if anything is returned */ + if (!sha_ctx) + return -EINPROGRESS; + + if (sha_ctx->error) { + ret = sha_ctx->error; + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + goto done; + } + + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + ret = sha_finish_walk(&rctx, cstate, false); + + if (!rctx) + return -EINPROGRESS; +done: + sha_complete_job(rctx, cstate, ret); + return ret; +} + +static int sha256_mb_finup(struct ahash_request *areq) +{ + struct mcryptd_hash_request_ctx *rctx = + container_of(areq, struct mcryptd_hash_request_ctx, areq); + struct mcryptd_alg_cstate *cstate = + this_cpu_ptr(sha256_mb_alg_state.alg_cstate); + + struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); + struct sha256_hash_ctx *sha_ctx; + int ret = 0, flag = HASH_UPDATE, nbytes; + + /* sanity check */ + if (rctx->tag.cpu != smp_processor_id()) { + pr_err("mcryptd error: cpu clash\n"); + goto done; + } + + /* need to init context */ + req_ctx_init(rctx, areq); + + nbytes = crypto_ahash_walk_first(req, &rctx->walk); + + if (nbytes < 0) { + ret = nbytes; + goto done; + } + + if (crypto_ahash_walk_last(&rctx->walk)) { + rctx->flag |= HASH_DONE; + flag = HASH_LAST; + } + + /* submit */ + rctx->flag |= HASH_FINAL; + sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); + sha256_mb_add_list(rctx, cstate); + + kernel_fpu_begin(); + sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, + nbytes, flag); + kernel_fpu_end(); + + /* check if anything is returned */ + if (!sha_ctx) + return -EINPROGRESS; + + if (sha_ctx->error) { + ret = sha_ctx->error; + goto done; + } + + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + ret = sha_finish_walk(&rctx, cstate, false); + if (!rctx) + return -EINPROGRESS; +done: + sha_complete_job(rctx, cstate, ret); + return ret; +} + +static int sha256_mb_final(struct ahash_request *areq) +{ + struct mcryptd_hash_request_ctx *rctx = + container_of(areq, struct mcryptd_hash_request_ctx, + areq); + struct mcryptd_alg_cstate *cstate = + this_cpu_ptr(sha256_mb_alg_state.alg_cstate); + + struct sha256_hash_ctx *sha_ctx; + int ret = 0; + u8 data; + + /* sanity check */ + if (rctx->tag.cpu != smp_processor_id()) { + pr_err("mcryptd error: cpu clash\n"); + goto done; + } + + /* need to init context */ + req_ctx_init(rctx, areq); + + rctx->flag |= HASH_DONE | HASH_FINAL; + + sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); + /* flag HASH_FINAL and 0 data size */ + sha256_mb_add_list(rctx, cstate); + kernel_fpu_begin(); + sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, + HASH_LAST); + kernel_fpu_end(); + + /* check if anything is returned */ + if (!sha_ctx) + return -EINPROGRESS; + + if (sha_ctx->error) { + ret = sha_ctx->error; + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + goto done; + } + + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + ret = sha_finish_walk(&rctx, cstate, false); + if (!rctx) + return -EINPROGRESS; +done: + sha_complete_job(rctx, cstate, ret); + return ret; +} + +static int sha256_mb_export(struct ahash_request *areq, void *out) +{ + struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha256_mb_import(struct ahash_request *areq, const void *in) +{ + struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static int sha256_mb_async_init_tfm(struct crypto_tfm *tfm) +{ + struct mcryptd_ahash *mcryptd_tfm; + struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); + struct mcryptd_hash_ctx *mctx; + + mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha256-mb", + CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); + if (IS_ERR(mcryptd_tfm)) + return PTR_ERR(mcryptd_tfm); + mctx = crypto_ahash_ctx(&mcryptd_tfm->base); + mctx->alg_state = &sha256_mb_alg_state; + ctx->mcryptd_tfm = mcryptd_tfm; + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ahash_request) + + crypto_ahash_reqsize(&mcryptd_tfm->base)); + + return 0; +} + +static void sha256_mb_async_exit_tfm(struct crypto_tfm *tfm) +{ + struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); + + mcryptd_free_ahash(ctx->mcryptd_tfm); +} + +static int sha256_mb_areq_init_tfm(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ahash_request) + + sizeof(struct sha256_hash_ctx)); + + return 0; +} + +static void sha256_mb_areq_exit_tfm(struct crypto_tfm *tfm) +{ + struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); + + mcryptd_free_ahash(ctx->mcryptd_tfm); +} + +static struct ahash_alg sha256_mb_areq_alg = { + .init = sha256_mb_init, + .update = sha256_mb_update, + .final = sha256_mb_final, + .finup = sha256_mb_finup, + .export = sha256_mb_export, + .import = sha256_mb_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_hash_ctx), + .base = { + .cra_name = "__sha256-mb", + .cra_driver_name = "__intel_sha256-mb", + .cra_priority = 100, + /* + * use ASYNC flag as some buffers in multi-buffer + * algo may not have completed before hashing thread + * sleep + */ + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_INTERNAL, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT + (sha256_mb_areq_alg.halg.base.cra_list), + .cra_init = sha256_mb_areq_init_tfm, + .cra_exit = sha256_mb_areq_exit_tfm, + .cra_ctxsize = sizeof(struct sha256_hash_ctx), + } + } +}; + +static int sha256_mb_async_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_init(mcryptd_req); +} + +static int sha256_mb_async_update(struct ahash_request *req) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_update(mcryptd_req); +} + +static int sha256_mb_async_finup(struct ahash_request *req) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_finup(mcryptd_req); +} + +static int sha256_mb_async_final(struct ahash_request *req) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_final(mcryptd_req); +} + +static int sha256_mb_async_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_digest(mcryptd_req); +} + +static int sha256_mb_async_export(struct ahash_request *req, void *out) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_export(mcryptd_req, out); +} + +static int sha256_mb_async_import(struct ahash_request *req, const void *in) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); + struct mcryptd_hash_request_ctx *rctx; + struct ahash_request *areq; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + rctx = ahash_request_ctx(mcryptd_req); + areq = &rctx->areq; + + ahash_request_set_tfm(areq, child); + ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, + rctx->complete, req); + + return crypto_ahash_import(mcryptd_req, in); +} + +static struct ahash_alg sha256_mb_async_alg = { + .init = sha256_mb_async_init, + .update = sha256_mb_async_update, + .final = sha256_mb_async_final, + .finup = sha256_mb_async_finup, + .export = sha256_mb_async_export, + .import = sha256_mb_async_import, + .digest = sha256_mb_async_digest, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_hash_ctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256_mb", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_type = &crypto_ahash_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT + (sha256_mb_async_alg.halg.base.cra_list), + .cra_init = sha256_mb_async_init_tfm, + .cra_exit = sha256_mb_async_exit_tfm, + .cra_ctxsize = sizeof(struct sha256_mb_ctx), + .cra_alignmask = 0, + }, + }, +}; + +static unsigned long sha256_mb_flusher(struct mcryptd_alg_cstate *cstate) +{ + struct mcryptd_hash_request_ctx *rctx; + unsigned long cur_time; + unsigned long next_flush = 0; + struct sha256_hash_ctx *sha_ctx; + + + cur_time = jiffies; + + while (!list_empty(&cstate->work_list)) { + rctx = list_entry(cstate->work_list.next, + struct mcryptd_hash_request_ctx, waiter); + if (time_before(cur_time, rctx->tag.expire)) + break; + kernel_fpu_begin(); + sha_ctx = (struct sha256_hash_ctx *) + sha256_ctx_mgr_flush(cstate->mgr); + kernel_fpu_end(); + if (!sha_ctx) { + pr_err("sha256_mb error: nothing got" + " flushed for non-empty list\n"); + break; + } + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + sha_finish_walk(&rctx, cstate, true); + sha_complete_job(rctx, cstate, 0); + } + + if (!list_empty(&cstate->work_list)) { + rctx = list_entry(cstate->work_list.next, + struct mcryptd_hash_request_ctx, waiter); + /* get the hash context and then flush time */ + next_flush = rctx->tag.expire; + mcryptd_arm_flusher(cstate, get_delay(next_flush)); + } + return next_flush; +} + +static int __init sha256_mb_mod_init(void) +{ + + int cpu; + int err; + struct mcryptd_alg_cstate *cpu_state; + + /* check for dependent cpu features */ + if (!boot_cpu_has(X86_FEATURE_AVX2) || + !boot_cpu_has(X86_FEATURE_BMI2)) + return -ENODEV; + + /* initialize multibuffer structures */ + sha256_mb_alg_state.alg_cstate = alloc_percpu + (struct mcryptd_alg_cstate); + + sha256_job_mgr_init = sha256_mb_mgr_init_avx2; + sha256_job_mgr_submit = sha256_mb_mgr_submit_avx2; + sha256_job_mgr_flush = sha256_mb_mgr_flush_avx2; + sha256_job_mgr_get_comp_job = sha256_mb_mgr_get_comp_job_avx2; + + if (!sha256_mb_alg_state.alg_cstate) + return -ENOMEM; + for_each_possible_cpu(cpu) { + cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); + cpu_state->next_flush = 0; + cpu_state->next_seq_num = 0; + cpu_state->flusher_engaged = false; + INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); + cpu_state->cpu = cpu; + cpu_state->alg_state = &sha256_mb_alg_state; + cpu_state->mgr = kzalloc(sizeof(struct sha256_ctx_mgr), + GFP_KERNEL); + if (!cpu_state->mgr) + goto err2; + sha256_ctx_mgr_init(cpu_state->mgr); + INIT_LIST_HEAD(&cpu_state->work_list); + spin_lock_init(&cpu_state->work_lock); + } + sha256_mb_alg_state.flusher = &sha256_mb_flusher; + + err = crypto_register_ahash(&sha256_mb_areq_alg); + if (err) + goto err2; + err = crypto_register_ahash(&sha256_mb_async_alg); + if (err) + goto err1; + + + return 0; +err1: + crypto_unregister_ahash(&sha256_mb_areq_alg); +err2: + for_each_possible_cpu(cpu) { + cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); + kfree(cpu_state->mgr); + } + free_percpu(sha256_mb_alg_state.alg_cstate); + return -ENODEV; +} + +static void __exit sha256_mb_mod_fini(void) +{ + int cpu; + struct mcryptd_alg_cstate *cpu_state; + + crypto_unregister_ahash(&sha256_mb_async_alg); + crypto_unregister_ahash(&sha256_mb_areq_alg); + for_each_possible_cpu(cpu) { + cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); + kfree(cpu_state->mgr); + } + free_percpu(sha256_mb_alg_state.alg_cstate); +} + +module_init(sha256_mb_mod_init); +module_exit(sha256_mb_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, multi buffer accelerated"); + +MODULE_ALIAS_CRYPTO("sha256"); diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h b/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h new file mode 100644 index 000000000000..edd252b73206 --- /dev/null +++ b/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h @@ -0,0 +1,136 @@ +/* + * Header file for multi buffer SHA256 context + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SHA_MB_CTX_INTERNAL_H +#define _SHA_MB_CTX_INTERNAL_H + +#include "sha256_mb_mgr.h" + +#define HASH_UPDATE 0x00 +#define HASH_FIRST 0x01 +#define HASH_LAST 0x02 +#define HASH_ENTIRE 0x03 +#define HASH_DONE 0x04 +#define HASH_FINAL 0x08 + +#define HASH_CTX_STS_IDLE 0x00 +#define HASH_CTX_STS_PROCESSING 0x01 +#define HASH_CTX_STS_LAST 0x02 +#define HASH_CTX_STS_COMPLETE 0x04 + +enum hash_ctx_error { + HASH_CTX_ERROR_NONE = 0, + HASH_CTX_ERROR_INVALID_FLAGS = -1, + HASH_CTX_ERROR_ALREADY_PROCESSING = -2, + HASH_CTX_ERROR_ALREADY_COMPLETED = -3, + +#ifdef HASH_CTX_DEBUG + HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4, +#endif +}; + + +#define hash_ctx_user_data(ctx) ((ctx)->user_data) +#define hash_ctx_digest(ctx) ((ctx)->job.result_digest) +#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING) +#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE) +#define hash_ctx_status(ctx) ((ctx)->status) +#define hash_ctx_error(ctx) ((ctx)->error) +#define hash_ctx_init(ctx) \ + do { \ + (ctx)->error = HASH_CTX_ERROR_NONE; \ + (ctx)->status = HASH_CTX_STS_COMPLETE; \ + } while (0) + + +/* Hash Constants and Typedefs */ +#define SHA256_DIGEST_LENGTH 8 +#define SHA256_LOG2_BLOCK_SIZE 6 + +#define SHA256_PADLENGTHFIELD_SIZE 8 + +#ifdef SHA_MB_DEBUG +#define assert(expr) \ +do { \ + if (unlikely(!(expr))) { \ + printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \ + #expr, __FILE__, __func__, __LINE__); \ + } \ +} while (0) +#else +#define assert(expr) do {} while (0) +#endif + +struct sha256_ctx_mgr { + struct sha256_mb_mgr mgr; +}; + +/* typedef struct sha256_ctx_mgr sha256_ctx_mgr; */ + +struct sha256_hash_ctx { + /* Must be at struct offset 0 */ + struct job_sha256 job; + /* status flag */ + int status; + /* error flag */ + int error; + + uint32_t total_length; + const void *incoming_buffer; + uint32_t incoming_buffer_length; + uint8_t partial_block_buffer[SHA256_BLOCK_SIZE * 2]; + uint32_t partial_block_buffer_length; + void *user_data; +}; + +#endif diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h b/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h new file mode 100644 index 000000000000..b01ae408c56d --- /dev/null +++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h @@ -0,0 +1,108 @@ +/* + * Header file for multi buffer SHA256 algorithm manager + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __SHA_MB_MGR_H +#define __SHA_MB_MGR_H + +#include + +#define NUM_SHA256_DIGEST_WORDS 8 + +enum job_sts { STS_UNKNOWN = 0, + STS_BEING_PROCESSED = 1, + STS_COMPLETED = 2, + STS_INTERNAL_ERROR = 3, + STS_ERROR = 4 +}; + +struct job_sha256 { + u8 *buffer; + u32 len; + u32 result_digest[NUM_SHA256_DIGEST_WORDS] __aligned(32); + enum job_sts status; + void *user_data; +}; + +/* SHA256 out-of-order scheduler */ + +/* typedef uint32_t sha8_digest_array[8][8]; */ + +struct sha256_args_x8 { + uint32_t digest[8][8]; + uint8_t *data_ptr[8]; +}; + +struct sha256_lane_data { + struct job_sha256 *job_in_lane; +}; + +struct sha256_mb_mgr { + struct sha256_args_x8 args; + + uint32_t lens[8]; + + /* each byte is index (0...7) of unused lanes */ + uint64_t unused_lanes; + /* byte 4 is set to FF as a flag */ + struct sha256_lane_data ldata[8]; +}; + + +#define SHA256_MB_MGR_NUM_LANES_AVX2 8 + +void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state); +struct job_sha256 *sha256_mb_mgr_submit_avx2(struct sha256_mb_mgr *state, + struct job_sha256 *job); +struct job_sha256 *sha256_mb_mgr_flush_avx2(struct sha256_mb_mgr *state); +struct job_sha256 *sha256_mb_mgr_get_comp_job_avx2(struct sha256_mb_mgr *state); + +#endif diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S new file mode 100644 index 000000000000..5c377bac21d0 --- /dev/null +++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S @@ -0,0 +1,304 @@ +/* + * Header file for multi buffer SHA256 algorithm data structure + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +# Macros for defining data structures + +# Usage example + +#START_FIELDS # JOB_AES +### name size align +#FIELD _plaintext, 8, 8 # pointer to plaintext +#FIELD _ciphertext, 8, 8 # pointer to ciphertext +#FIELD _IV, 16, 8 # IV +#FIELD _keys, 8, 8 # pointer to keys +#FIELD _len, 4, 4 # length in bytes +#FIELD _status, 4, 4 # status enumeration +#FIELD _user_data, 8, 8 # pointer to user data +#UNION _union, size1, align1, \ +# size2, align2, \ +# size3, align3, \ +# ... +#END_FIELDS +#%assign _JOB_AES_size _FIELD_OFFSET +#%assign _JOB_AES_align _STRUCT_ALIGN + +######################################################################### + +# Alternate "struc-like" syntax: +# STRUCT job_aes2 +# RES_Q .plaintext, 1 +# RES_Q .ciphertext, 1 +# RES_DQ .IV, 1 +# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN +# RES_U .union, size1, align1, \ +# size2, align2, \ +# ... +# ENDSTRUCT +# # Following only needed if nesting +# %assign job_aes2_size _FIELD_OFFSET +# %assign job_aes2_align _STRUCT_ALIGN +# +# RES_* macros take a name, a count and an optional alignment. +# The count in in terms of the base size of the macro, and the +# default alignment is the base size. +# The macros are: +# Macro Base size +# RES_B 1 +# RES_W 2 +# RES_D 4 +# RES_Q 8 +# RES_DQ 16 +# RES_Y 32 +# RES_Z 64 +# +# RES_U defines a union. It's arguments are a name and two or more +# pairs of "size, alignment" +# +# The two assigns are only needed if this structure is being nested +# within another. Even if the assigns are not done, one can still use +# STRUCT_NAME_size as the size of the structure. +# +# Note that for nesting, you still need to assign to STRUCT_NAME_size. +# +# The differences between this and using "struc" directly are that each +# type is implicitly aligned to its natural length (although this can be +# over-ridden with an explicit third parameter), and that the structure +# is padded at the end to its overall alignment. +# + +######################################################################### + +#ifndef _DATASTRUCT_ASM_ +#define _DATASTRUCT_ASM_ + +#define SZ8 8*SHA256_DIGEST_WORD_SIZE +#define ROUNDS 64*SZ8 +#define PTR_SZ 8 +#define SHA256_DIGEST_WORD_SIZE 4 +#define MAX_SHA256_LANES 8 +#define SHA256_DIGEST_WORDS 8 +#define SHA256_DIGEST_ROW_SIZE (MAX_SHA256_LANES * SHA256_DIGEST_WORD_SIZE) +#define SHA256_DIGEST_SIZE (SHA256_DIGEST_ROW_SIZE * SHA256_DIGEST_WORDS) +#define SHA256_BLK_SZ 64 + +# START_FIELDS +.macro START_FIELDS + _FIELD_OFFSET = 0 + _STRUCT_ALIGN = 0 +.endm + +# FIELD name size align +.macro FIELD name size align + _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1)) + \name = _FIELD_OFFSET + _FIELD_OFFSET = _FIELD_OFFSET + (\size) +.if (\align > _STRUCT_ALIGN) + _STRUCT_ALIGN = \align +.endif +.endm + +# END_FIELDS +.macro END_FIELDS + _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) +.endm + +######################################################################## + +.macro STRUCT p1 +START_FIELDS +.struc \p1 +.endm + +.macro ENDSTRUCT + tmp = _FIELD_OFFSET + END_FIELDS + tmp = (_FIELD_OFFSET - %%tmp) +.if (tmp > 0) + .lcomm tmp +.endif +.endstruc +.endm + +## RES_int name size align +.macro RES_int p1 p2 p3 + name = \p1 + size = \p2 + align = .\p3 + + _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1)) +.align align +.lcomm name size + _FIELD_OFFSET = _FIELD_OFFSET + (size) +.if (align > _STRUCT_ALIGN) + _STRUCT_ALIGN = align +.endif +.endm + +# macro RES_B name, size [, align] +.macro RES_B _name, _size, _align=1 +RES_int _name _size _align +.endm + +# macro RES_W name, size [, align] +.macro RES_W _name, _size, _align=2 +RES_int _name 2*(_size) _align +.endm + +# macro RES_D name, size [, align] +.macro RES_D _name, _size, _align=4 +RES_int _name 4*(_size) _align +.endm + +# macro RES_Q name, size [, align] +.macro RES_Q _name, _size, _align=8 +RES_int _name 8*(_size) _align +.endm + +# macro RES_DQ name, size [, align] +.macro RES_DQ _name, _size, _align=16 +RES_int _name 16*(_size) _align +.endm + +# macro RES_Y name, size [, align] +.macro RES_Y _name, _size, _align=32 +RES_int _name 32*(_size) _align +.endm + +# macro RES_Z name, size [, align] +.macro RES_Z _name, _size, _align=64 +RES_int _name 64*(_size) _align +.endm + +#endif + + +######################################################################## +#### Define SHA256 Out Of Order Data Structures +######################################################################## + +START_FIELDS # LANE_DATA +### name size align +FIELD _job_in_lane, 8, 8 # pointer to job object +END_FIELDS + + _LANE_DATA_size = _FIELD_OFFSET + _LANE_DATA_align = _STRUCT_ALIGN + +######################################################################## + +START_FIELDS # SHA256_ARGS_X4 +### name size align +FIELD _digest, 4*8*8, 4 # transposed digest +FIELD _data_ptr, 8*8, 8 # array of pointers to data +END_FIELDS + + _SHA256_ARGS_X4_size = _FIELD_OFFSET + _SHA256_ARGS_X4_align = _STRUCT_ALIGN + _SHA256_ARGS_X8_size = _FIELD_OFFSET + _SHA256_ARGS_X8_align = _STRUCT_ALIGN + +####################################################################### + +START_FIELDS # MB_MGR +### name size align +FIELD _args, _SHA256_ARGS_X4_size, _SHA256_ARGS_X4_align +FIELD _lens, 4*8, 8 +FIELD _unused_lanes, 8, 8 +FIELD _ldata, _LANE_DATA_size*8, _LANE_DATA_align +END_FIELDS + + _MB_MGR_size = _FIELD_OFFSET + _MB_MGR_align = _STRUCT_ALIGN + +_args_digest = _args + _digest +_args_data_ptr = _args + _data_ptr + +####################################################################### + +START_FIELDS #STACK_FRAME +### name size align +FIELD _data, 16*SZ8, 1 # transposed digest +FIELD _digest, 8*SZ8, 1 # array of pointers to data +FIELD _ytmp, 4*SZ8, 1 +FIELD _rsp, 8, 1 +END_FIELDS + + _STACK_FRAME_size = _FIELD_OFFSET + _STACK_FRAME_align = _STRUCT_ALIGN + +####################################################################### + +######################################################################## +#### Define constants +######################################################################## + +#define STS_UNKNOWN 0 +#define STS_BEING_PROCESSED 1 +#define STS_COMPLETED 2 + +######################################################################## +#### Define JOB_SHA256 structure +######################################################################## + +START_FIELDS # JOB_SHA256 + +### name size align +FIELD _buffer, 8, 8 # pointer to buffer +FIELD _len, 8, 8 # length in bytes +FIELD _result_digest, 8*4, 32 # Digest (output) +FIELD _status, 4, 4 +FIELD _user_data, 8, 8 +END_FIELDS + + _JOB_SHA256_size = _FIELD_OFFSET + _JOB_SHA256_align = _STRUCT_ALIGN diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S new file mode 100644 index 000000000000..b691da981cd9 --- /dev/null +++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S @@ -0,0 +1,304 @@ +/* + * Flush routine for SHA256 multibuffer + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include "sha256_mb_mgr_datastruct.S" + +.extern sha256_x8_avx2 + +#LINUX register definitions +#define arg1 %rdi +#define arg2 %rsi + +# Common register definitions +#define state arg1 +#define job arg2 +#define len2 arg2 + +# idx must be a register not clobberred by sha1_mult +#define idx %r8 +#define DWORD_idx %r8d + +#define unused_lanes %rbx +#define lane_data %rbx +#define tmp2 %rbx +#define tmp2_w %ebx + +#define job_rax %rax +#define tmp1 %rax +#define size_offset %rax +#define tmp %rax +#define start_offset %rax + +#define tmp3 %arg1 + +#define extra_blocks %arg2 +#define p %arg2 + +.macro LABEL prefix n +\prefix\n\(): +.endm + +.macro JNE_SKIP i +jne skip_\i +.endm + +.altmacro +.macro SET_OFFSET _offset +offset = \_offset +.endm +.noaltmacro + +# JOB_SHA256* sha256_mb_mgr_flush_avx2(MB_MGR *state) +# arg 1 : rcx : state +ENTRY(sha256_mb_mgr_flush_avx2) + FRAME_BEGIN + push %rbx + + # If bit (32+3) is set, then all lanes are empty + mov _unused_lanes(state), unused_lanes + bt $32+3, unused_lanes + jc return_null + + # find a lane with a non-null job + xor idx, idx + offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne one(%rip), idx + offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne two(%rip), idx + offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne three(%rip), idx + offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne four(%rip), idx + offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne five(%rip), idx + offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne six(%rip), idx + offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne seven(%rip), idx + + # copy idx to empty lanes +copy_lane_data: + offset = (_args + _data_ptr) + mov offset(state,idx,8), tmp + + I = 0 +.rep 8 + offset = (_ldata + I * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) +.altmacro + JNE_SKIP %I + offset = (_args + _data_ptr + 8*I) + mov tmp, offset(state) + offset = (_lens + 4*I) + movl $0xFFFFFFFF, offset(state) +LABEL skip_ %I + I = (I+1) +.noaltmacro +.endr + + # Find min length + vmovdqa _lens+0*16(state), %xmm0 + vmovdqa _lens+1*16(state), %xmm1 + + vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} + vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} + vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} + vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} + vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword + + vmovd %xmm2, DWORD_idx + mov idx, len2 + and $0xF, idx + shr $4, len2 + jz len_is_0 + + vpand clear_low_nibble(%rip), %xmm2, %xmm2 + vpshufd $0, %xmm2, %xmm2 + + vpsubd %xmm2, %xmm0, %xmm0 + vpsubd %xmm2, %xmm1, %xmm1 + + vmovdqa %xmm0, _lens+0*16(state) + vmovdqa %xmm1, _lens+1*16(state) + + # "state" and "args" are the same address, arg1 + # len is arg2 + call sha256_x8_avx2 + # state and idx are intact + +len_is_0: + # process completed job "idx" + imul $_LANE_DATA_size, idx, lane_data + lea _ldata(state, lane_data), lane_data + + mov _job_in_lane(lane_data), job_rax + movq $0, _job_in_lane(lane_data) + movl $STS_COMPLETED, _status(job_rax) + mov _unused_lanes(state), unused_lanes + shl $4, unused_lanes + or idx, unused_lanes + + mov unused_lanes, _unused_lanes(state) + movl $0xFFFFFFFF, _lens(state,idx,4) + + vmovd _args_digest(state , idx, 4) , %xmm0 + vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 + vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 + vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 + vmovd _args_digest+4*32(state, idx, 4), %xmm1 + vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1 + vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1 + vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1 + + vmovdqu %xmm0, _result_digest(job_rax) + offset = (_result_digest + 1*16) + vmovdqu %xmm1, offset(job_rax) + +return: + pop %rbx + FRAME_END + ret + +return_null: + xor job_rax, job_rax + jmp return +ENDPROC(sha256_mb_mgr_flush_avx2) + +############################################################################## + +.align 16 +ENTRY(sha256_mb_mgr_get_comp_job_avx2) + push %rbx + + ## if bit 32+3 is set, then all lanes are empty + mov _unused_lanes(state), unused_lanes + bt $(32+3), unused_lanes + jc .return_null + + # Find min length + vmovdqa _lens(state), %xmm0 + vmovdqa _lens+1*16(state), %xmm1 + + vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} + vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} + vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} + vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} + vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword + + vmovd %xmm2, DWORD_idx + test $~0xF, idx + jnz .return_null + + # process completed job "idx" + imul $_LANE_DATA_size, idx, lane_data + lea _ldata(state, lane_data), lane_data + + mov _job_in_lane(lane_data), job_rax + movq $0, _job_in_lane(lane_data) + movl $STS_COMPLETED, _status(job_rax) + mov _unused_lanes(state), unused_lanes + shl $4, unused_lanes + or idx, unused_lanes + mov unused_lanes, _unused_lanes(state) + + movl $0xFFFFFFFF, _lens(state, idx, 4) + + vmovd _args_digest(state, idx, 4), %xmm0 + vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 + vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 + vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 + movl _args_digest+4*32(state, idx, 4), tmp2_w + vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1 + vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1 + vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1 + + vmovdqu %xmm0, _result_digest(job_rax) + movl tmp2_w, _result_digest+1*16(job_rax) + + pop %rbx + + ret + +.return_null: + xor job_rax, job_rax + pop %rbx + ret +ENDPROC(sha256_mb_mgr_get_comp_job_avx2) + +.data + +.align 16 +clear_low_nibble: +.octa 0x000000000000000000000000FFFFFFF0 +one: +.quad 1 +two: +.quad 2 +three: +.quad 3 +four: +.quad 4 +five: +.quad 5 +six: +.quad 6 +seven: +.quad 7 diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c new file mode 100644 index 000000000000..b0c498371e67 --- /dev/null +++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c @@ -0,0 +1,65 @@ +/* + * Initialization code for multi buffer SHA256 algorithm for AVX2 + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "sha256_mb_mgr.h" + +void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state) +{ + unsigned int j; + + state->unused_lanes = 0xF76543210ULL; + for (j = 0; j < 8; j++) { + state->lens[j] = 0xFFFFFFFF; + state->ldata[j].job_in_lane = NULL; + } +} diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S new file mode 100644 index 000000000000..7ea670e25acc --- /dev/null +++ b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S @@ -0,0 +1,215 @@ +/* + * Buffer submit code for multi buffer SHA256 algorithm + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include "sha256_mb_mgr_datastruct.S" + +.extern sha256_x8_avx2 + +# LINUX register definitions +arg1 = %rdi +arg2 = %rsi +size_offset = %rcx +tmp2 = %rcx +extra_blocks = %rdx + +# Common definitions +#define state arg1 +#define job %rsi +#define len2 arg2 +#define p2 arg2 + +# idx must be a register not clobberred by sha1_x8_avx2 +idx = %r8 +DWORD_idx = %r8d +last_len = %r8 + +p = %r11 +start_offset = %r11 + +unused_lanes = %rbx +BYTE_unused_lanes = %bl + +job_rax = %rax +len = %rax +DWORD_len = %eax + +lane = %r12 +tmp3 = %r12 + +tmp = %r9 +DWORD_tmp = %r9d + +lane_data = %r10 + +# JOB* sha256_mb_mgr_submit_avx2(MB_MGR *state, JOB_SHA256 *job) +# arg 1 : rcx : state +# arg 2 : rdx : job +ENTRY(sha256_mb_mgr_submit_avx2) + FRAME_BEGIN + push %rbx + push %r12 + + mov _unused_lanes(state), unused_lanes + mov unused_lanes, lane + and $0xF, lane + shr $4, unused_lanes + imul $_LANE_DATA_size, lane, lane_data + movl $STS_BEING_PROCESSED, _status(job) + lea _ldata(state, lane_data), lane_data + mov unused_lanes, _unused_lanes(state) + movl _len(job), DWORD_len + + mov job, _job_in_lane(lane_data) + shl $4, len + or lane, len + + movl DWORD_len, _lens(state , lane, 4) + + # Load digest words from result_digest + vmovdqu _result_digest(job), %xmm0 + vmovdqu _result_digest+1*16(job), %xmm1 + vmovd %xmm0, _args_digest(state, lane, 4) + vpextrd $1, %xmm0, _args_digest+1*32(state , lane, 4) + vpextrd $2, %xmm0, _args_digest+2*32(state , lane, 4) + vpextrd $3, %xmm0, _args_digest+3*32(state , lane, 4) + vmovd %xmm1, _args_digest+4*32(state , lane, 4) + + vpextrd $1, %xmm1, _args_digest+5*32(state , lane, 4) + vpextrd $2, %xmm1, _args_digest+6*32(state , lane, 4) + vpextrd $3, %xmm1, _args_digest+7*32(state , lane, 4) + + mov _buffer(job), p + mov p, _args_data_ptr(state, lane, 8) + + cmp $0xF, unused_lanes + jne return_null + +start_loop: + # Find min length + vmovdqa _lens(state), %xmm0 + vmovdqa _lens+1*16(state), %xmm1 + + vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} + vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} + vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} + vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} + vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword + + vmovd %xmm2, DWORD_idx + mov idx, len2 + and $0xF, idx + shr $4, len2 + jz len_is_0 + + vpand clear_low_nibble(%rip), %xmm2, %xmm2 + vpshufd $0, %xmm2, %xmm2 + + vpsubd %xmm2, %xmm0, %xmm0 + vpsubd %xmm2, %xmm1, %xmm1 + + vmovdqa %xmm0, _lens + 0*16(state) + vmovdqa %xmm1, _lens + 1*16(state) + + # "state" and "args" are the same address, arg1 + # len is arg2 + call sha256_x8_avx2 + + # state and idx are intact + +len_is_0: + # process completed job "idx" + imul $_LANE_DATA_size, idx, lane_data + lea _ldata(state, lane_data), lane_data + + mov _job_in_lane(lane_data), job_rax + mov _unused_lanes(state), unused_lanes + movq $0, _job_in_lane(lane_data) + movl $STS_COMPLETED, _status(job_rax) + shl $4, unused_lanes + or idx, unused_lanes + mov unused_lanes, _unused_lanes(state) + + movl $0xFFFFFFFF, _lens(state,idx,4) + + vmovd _args_digest(state, idx, 4), %xmm0 + vpinsrd $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0 + vpinsrd $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0 + vpinsrd $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0 + vmovd _args_digest+4*32(state, idx, 4), %xmm1 + + vpinsrd $1, _args_digest+5*32(state , idx, 4), %xmm1, %xmm1 + vpinsrd $2, _args_digest+6*32(state , idx, 4), %xmm1, %xmm1 + vpinsrd $3, _args_digest+7*32(state , idx, 4), %xmm1, %xmm1 + + vmovdqu %xmm0, _result_digest(job_rax) + vmovdqu %xmm1, _result_digest+1*16(job_rax) + +return: + pop %r12 + pop %rbx + FRAME_END + ret + +return_null: + xor job_rax, job_rax + jmp return + +ENDPROC(sha256_mb_mgr_submit_avx2) + +.data + +.align 16 +clear_low_nibble: + .octa 0x000000000000000000000000FFFFFFF0 diff --git a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S new file mode 100644 index 000000000000..aa21aea4c722 --- /dev/null +++ b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S @@ -0,0 +1,593 @@ +/* + * Multi-buffer SHA256 algorithm hash compute routine + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "sha256_mb_mgr_datastruct.S" + +## code to compute oct SHA256 using SSE-256 +## outer calling routine takes care of save and restore of XMM registers +## Logic designed/laid out by JDG + +## Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; %ymm0-15 +## Linux clobbers: rax rbx rcx rdx rsi r9 r10 r11 r12 r13 r14 r15 +## Linux preserves: rdi rbp r8 +## +## clobbers %ymm0-15 + +arg1 = %rdi +arg2 = %rsi +reg3 = %rcx +reg4 = %rdx + +# Common definitions +STATE = arg1 +INP_SIZE = arg2 + +IDX = %rax +ROUND = %rbx +TBL = reg3 + +inp0 = %r9 +inp1 = %r10 +inp2 = %r11 +inp3 = %r12 +inp4 = %r13 +inp5 = %r14 +inp6 = %r15 +inp7 = reg4 + +a = %ymm0 +b = %ymm1 +c = %ymm2 +d = %ymm3 +e = %ymm4 +f = %ymm5 +g = %ymm6 +h = %ymm7 + +T1 = %ymm8 + +a0 = %ymm12 +a1 = %ymm13 +a2 = %ymm14 +TMP = %ymm15 +TMP0 = %ymm6 +TMP1 = %ymm7 + +TT0 = %ymm8 +TT1 = %ymm9 +TT2 = %ymm10 +TT3 = %ymm11 +TT4 = %ymm12 +TT5 = %ymm13 +TT6 = %ymm14 +TT7 = %ymm15 + +# Define stack usage + +# Assume stack aligned to 32 bytes before call +# Therefore FRAMESZ mod 32 must be 32-8 = 24 + +#define FRAMESZ 0x388 + +#define VMOVPS vmovups + +# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1 +# "transpose" data in {r0...r7} using temps {t0...t1} +# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7} +# r0 = {a7 a6 a5 a4 a3 a2 a1 a0} +# r1 = {b7 b6 b5 b4 b3 b2 b1 b0} +# r2 = {c7 c6 c5 c4 c3 c2 c1 c0} +# r3 = {d7 d6 d5 d4 d3 d2 d1 d0} +# r4 = {e7 e6 e5 e4 e3 e2 e1 e0} +# r5 = {f7 f6 f5 f4 f3 f2 f1 f0} +# r6 = {g7 g6 g5 g4 g3 g2 g1 g0} +# r7 = {h7 h6 h5 h4 h3 h2 h1 h0} +# +# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7} +# r0 = {h0 g0 f0 e0 d0 c0 b0 a0} +# r1 = {h1 g1 f1 e1 d1 c1 b1 a1} +# r2 = {h2 g2 f2 e2 d2 c2 b2 a2} +# r3 = {h3 g3 f3 e3 d3 c3 b3 a3} +# r4 = {h4 g4 f4 e4 d4 c4 b4 a4} +# r5 = {h5 g5 f5 e5 d5 c5 b5 a5} +# r6 = {h6 g6 f6 e6 d6 c6 b6 a6} +# r7 = {h7 g7 f7 e7 d7 c7 b7 a7} +# + +.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1 + # process top half (r0..r3) {a...d} + vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0} + vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2} + vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0} + vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2} + vshufps $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5 d1 c1 b1 a1} + vshufps $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6 d2 c2 b2 a2} + vshufps $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7 d3 c3 b3 a3} + vshufps $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4 d0 c0 b0 a0} + + # use r2 in place of t0 + # process bottom half (r4..r7) {e...h} + vshufps $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4 f1 f0 e1 e0} + vshufps $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6 f3 f2 e3 e2} + vshufps $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4 h1 h0 g1 g0} + vshufps $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6 h3 h2 g3 g2} + vshufps $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5 h1 g1 f1 e1} + vshufps $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6 h2 g2 f2 e2} + vshufps $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7 h3 g3 f3 e3} + vshufps $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4 h0 g0 f0 e0} + + vperm2f128 $0x13, \r1, \r5, \r6 # h6...a6 + vperm2f128 $0x02, \r1, \r5, \r2 # h2...a2 + vperm2f128 $0x13, \r3, \r7, \r5 # h5...a5 + vperm2f128 $0x02, \r3, \r7, \r1 # h1...a1 + vperm2f128 $0x13, \r0, \r4, \r7 # h7...a7 + vperm2f128 $0x02, \r0, \r4, \r3 # h3...a3 + vperm2f128 $0x13, \t0, \t1, \r4 # h4...a4 + vperm2f128 $0x02, \t0, \t1, \r0 # h0...a0 + +.endm + +.macro ROTATE_ARGS +TMP_ = h +h = g +g = f +f = e +e = d +d = c +c = b +b = a +a = TMP_ +.endm + +.macro _PRORD reg imm tmp + vpslld $(32-\imm),\reg,\tmp + vpsrld $\imm,\reg, \reg + vpor \tmp,\reg, \reg +.endm + +# PRORD_nd reg, imm, tmp, src +.macro _PRORD_nd reg imm tmp src + vpslld $(32-\imm), \src, \tmp + vpsrld $\imm, \src, \reg + vpor \tmp, \reg, \reg +.endm + +# PRORD dst/src, amt +.macro PRORD reg imm + _PRORD \reg,\imm,TMP +.endm + +# PRORD_nd dst, src, amt +.macro PRORD_nd reg tmp imm + _PRORD_nd \reg, \imm, TMP, \tmp +.endm + +# arguments passed implicitly in preprocessor symbols i, a...h +.macro ROUND_00_15 _T1 i + PRORD_nd a0,e,5 # sig1: a0 = (e >> 5) + + vpxor g, f, a2 # ch: a2 = f^g + vpand e,a2, a2 # ch: a2 = (f^g)&e + vpxor g, a2, a2 # a2 = ch + + PRORD_nd a1,e,25 # sig1: a1 = (e >> 25) + + vmovdqu \_T1,(SZ8*(\i & 0xf))(%rsp) + vpaddd (TBL,ROUND,1), \_T1, \_T1 # T1 = W + K + vpxor e,a0, a0 # sig1: a0 = e ^ (e >> 5) + PRORD a0, 6 # sig1: a0 = (e >> 6) ^ (e >> 11) + vpaddd a2, h, h # h = h + ch + PRORD_nd a2,a,11 # sig0: a2 = (a >> 11) + vpaddd \_T1,h, h # h = h + ch + W + K + vpxor a1, a0, a0 # a0 = sigma1 + PRORD_nd a1,a,22 # sig0: a1 = (a >> 22) + vpxor c, a, \_T1 # maj: T1 = a^c + add $SZ8, ROUND # ROUND++ + vpand b, \_T1, \_T1 # maj: T1 = (a^c)&b + vpaddd a0, h, h + vpaddd h, d, d + vpxor a, a2, a2 # sig0: a2 = a ^ (a >> 11) + PRORD a2,2 # sig0: a2 = (a >> 2) ^ (a >> 13) + vpxor a1, a2, a2 # a2 = sig0 + vpand c, a, a1 # maj: a1 = a&c + vpor \_T1, a1, a1 # a1 = maj + vpaddd a1, h, h # h = h + ch + W + K + maj + vpaddd a2, h, h # h = h + ch + W + K + maj + sigma0 + ROTATE_ARGS +.endm + +# arguments passed implicitly in preprocessor symbols i, a...h +.macro ROUND_16_XX _T1 i + vmovdqu (SZ8*((\i-15)&0xf))(%rsp), \_T1 + vmovdqu (SZ8*((\i-2)&0xf))(%rsp), a1 + vmovdqu \_T1, a0 + PRORD \_T1,11 + vmovdqu a1, a2 + PRORD a1,2 + vpxor a0, \_T1, \_T1 + PRORD \_T1, 7 + vpxor a2, a1, a1 + PRORD a1, 17 + vpsrld $3, a0, a0 + vpxor a0, \_T1, \_T1 + vpsrld $10, a2, a2 + vpxor a2, a1, a1 + vpaddd (SZ8*((\i-16)&0xf))(%rsp), \_T1, \_T1 + vpaddd (SZ8*((\i-7)&0xf))(%rsp), a1, a1 + vpaddd a1, \_T1, \_T1 + + ROUND_00_15 \_T1,\i +.endm + +# SHA256_ARGS: +# UINT128 digest[8]; // transposed digests +# UINT8 *data_ptr[4]; + +# void sha256_x8_avx2(SHA256_ARGS *args, UINT64 bytes); +# arg 1 : STATE : pointer to array of pointers to input data +# arg 2 : INP_SIZE : size of input in blocks + # general registers preserved in outer calling routine + # outer calling routine saves all the XMM registers + # save rsp, allocate 32-byte aligned for local variables +ENTRY(sha256_x8_avx2) + + # save callee-saved clobbered registers to comply with C function ABI + push %r12 + push %r13 + push %r14 + push %r15 + + mov %rsp, IDX + sub $FRAMESZ, %rsp + and $~0x1F, %rsp + mov IDX, _rsp(%rsp) + + # Load the pre-transposed incoming digest. + vmovdqu 0*SHA256_DIGEST_ROW_SIZE(STATE),a + vmovdqu 1*SHA256_DIGEST_ROW_SIZE(STATE),b + vmovdqu 2*SHA256_DIGEST_ROW_SIZE(STATE),c + vmovdqu 3*SHA256_DIGEST_ROW_SIZE(STATE),d + vmovdqu 4*SHA256_DIGEST_ROW_SIZE(STATE),e + vmovdqu 5*SHA256_DIGEST_ROW_SIZE(STATE),f + vmovdqu 6*SHA256_DIGEST_ROW_SIZE(STATE),g + vmovdqu 7*SHA256_DIGEST_ROW_SIZE(STATE),h + + lea K256_8(%rip),TBL + + # load the address of each of the 4 message lanes + # getting ready to transpose input onto stack + mov _args_data_ptr+0*PTR_SZ(STATE),inp0 + mov _args_data_ptr+1*PTR_SZ(STATE),inp1 + mov _args_data_ptr+2*PTR_SZ(STATE),inp2 + mov _args_data_ptr+3*PTR_SZ(STATE),inp3 + mov _args_data_ptr+4*PTR_SZ(STATE),inp4 + mov _args_data_ptr+5*PTR_SZ(STATE),inp5 + mov _args_data_ptr+6*PTR_SZ(STATE),inp6 + mov _args_data_ptr+7*PTR_SZ(STATE),inp7 + + xor IDX, IDX +lloop: + xor ROUND, ROUND + + # save old digest + vmovdqu a, _digest(%rsp) + vmovdqu b, _digest+1*SZ8(%rsp) + vmovdqu c, _digest+2*SZ8(%rsp) + vmovdqu d, _digest+3*SZ8(%rsp) + vmovdqu e, _digest+4*SZ8(%rsp) + vmovdqu f, _digest+5*SZ8(%rsp) + vmovdqu g, _digest+6*SZ8(%rsp) + vmovdqu h, _digest+7*SZ8(%rsp) + i = 0 +.rep 2 + VMOVPS i*32(inp0, IDX), TT0 + VMOVPS i*32(inp1, IDX), TT1 + VMOVPS i*32(inp2, IDX), TT2 + VMOVPS i*32(inp3, IDX), TT3 + VMOVPS i*32(inp4, IDX), TT4 + VMOVPS i*32(inp5, IDX), TT5 + VMOVPS i*32(inp6, IDX), TT6 + VMOVPS i*32(inp7, IDX), TT7 + vmovdqu g, _ytmp(%rsp) + vmovdqu h, _ytmp+1*SZ8(%rsp) + TRANSPOSE8 TT0, TT1, TT2, TT3, TT4, TT5, TT6, TT7, TMP0, TMP1 + vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP1 + vmovdqu _ytmp(%rsp), g + vpshufb TMP1, TT0, TT0 + vpshufb TMP1, TT1, TT1 + vpshufb TMP1, TT2, TT2 + vpshufb TMP1, TT3, TT3 + vpshufb TMP1, TT4, TT4 + vpshufb TMP1, TT5, TT5 + vpshufb TMP1, TT6, TT6 + vpshufb TMP1, TT7, TT7 + vmovdqu _ytmp+1*SZ8(%rsp), h + vmovdqu TT4, _ytmp(%rsp) + vmovdqu TT5, _ytmp+1*SZ8(%rsp) + vmovdqu TT6, _ytmp+2*SZ8(%rsp) + vmovdqu TT7, _ytmp+3*SZ8(%rsp) + ROUND_00_15 TT0,(i*8+0) + vmovdqu _ytmp(%rsp), TT0 + ROUND_00_15 TT1,(i*8+1) + vmovdqu _ytmp+1*SZ8(%rsp), TT1 + ROUND_00_15 TT2,(i*8+2) + vmovdqu _ytmp+2*SZ8(%rsp), TT2 + ROUND_00_15 TT3,(i*8+3) + vmovdqu _ytmp+3*SZ8(%rsp), TT3 + ROUND_00_15 TT0,(i*8+4) + ROUND_00_15 TT1,(i*8+5) + ROUND_00_15 TT2,(i*8+6) + ROUND_00_15 TT3,(i*8+7) + i = (i+1) +.endr + add $64, IDX + i = (i*8) + + jmp Lrounds_16_xx +.align 16 +Lrounds_16_xx: +.rep 16 + ROUND_16_XX T1, i + i = (i+1) +.endr + + cmp $ROUNDS,ROUND + jb Lrounds_16_xx + + # add old digest + vpaddd _digest+0*SZ8(%rsp), a, a + vpaddd _digest+1*SZ8(%rsp), b, b + vpaddd _digest+2*SZ8(%rsp), c, c + vpaddd _digest+3*SZ8(%rsp), d, d + vpaddd _digest+4*SZ8(%rsp), e, e + vpaddd _digest+5*SZ8(%rsp), f, f + vpaddd _digest+6*SZ8(%rsp), g, g + vpaddd _digest+7*SZ8(%rsp), h, h + + sub $1, INP_SIZE # unit is blocks + jne lloop + + # write back to memory (state object) the transposed digest + vmovdqu a, 0*SHA256_DIGEST_ROW_SIZE(STATE) + vmovdqu b, 1*SHA256_DIGEST_ROW_SIZE(STATE) + vmovdqu c, 2*SHA256_DIGEST_ROW_SIZE(STATE) + vmovdqu d, 3*SHA256_DIGEST_ROW_SIZE(STATE) + vmovdqu e, 4*SHA256_DIGEST_ROW_SIZE(STATE) + vmovdqu f, 5*SHA256_DIGEST_ROW_SIZE(STATE) + vmovdqu g, 6*SHA256_DIGEST_ROW_SIZE(STATE) + vmovdqu h, 7*SHA256_DIGEST_ROW_SIZE(STATE) + + # update input pointers + add IDX, inp0 + mov inp0, _args_data_ptr+0*8(STATE) + add IDX, inp1 + mov inp1, _args_data_ptr+1*8(STATE) + add IDX, inp2 + mov inp2, _args_data_ptr+2*8(STATE) + add IDX, inp3 + mov inp3, _args_data_ptr+3*8(STATE) + add IDX, inp4 + mov inp4, _args_data_ptr+4*8(STATE) + add IDX, inp5 + mov inp5, _args_data_ptr+5*8(STATE) + add IDX, inp6 + mov inp6, _args_data_ptr+6*8(STATE) + add IDX, inp7 + mov inp7, _args_data_ptr+7*8(STATE) + + # Postamble + mov _rsp(%rsp), %rsp + + # restore callee-saved clobbered registers + pop %r15 + pop %r14 + pop %r13 + pop %r12 + + ret +ENDPROC(sha256_x8_avx2) +.data +.align 64 +K256_8: + .octa 0x428a2f98428a2f98428a2f98428a2f98 + .octa 0x428a2f98428a2f98428a2f98428a2f98 + .octa 0x71374491713744917137449171374491 + .octa 0x71374491713744917137449171374491 + .octa 0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf + .octa 0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf + .octa 0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5 + .octa 0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5 + .octa 0x3956c25b3956c25b3956c25b3956c25b + .octa 0x3956c25b3956c25b3956c25b3956c25b + .octa 0x59f111f159f111f159f111f159f111f1 + .octa 0x59f111f159f111f159f111f159f111f1 + .octa 0x923f82a4923f82a4923f82a4923f82a4 + .octa 0x923f82a4923f82a4923f82a4923f82a4 + .octa 0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5 + .octa 0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5 + .octa 0xd807aa98d807aa98d807aa98d807aa98 + .octa 0xd807aa98d807aa98d807aa98d807aa98 + .octa 0x12835b0112835b0112835b0112835b01 + .octa 0x12835b0112835b0112835b0112835b01 + .octa 0x243185be243185be243185be243185be + .octa 0x243185be243185be243185be243185be + .octa 0x550c7dc3550c7dc3550c7dc3550c7dc3 + .octa 0x550c7dc3550c7dc3550c7dc3550c7dc3 + .octa 0x72be5d7472be5d7472be5d7472be5d74 + .octa 0x72be5d7472be5d7472be5d7472be5d74 + .octa 0x80deb1fe80deb1fe80deb1fe80deb1fe + .octa 0x80deb1fe80deb1fe80deb1fe80deb1fe + .octa 0x9bdc06a79bdc06a79bdc06a79bdc06a7 + .octa 0x9bdc06a79bdc06a79bdc06a79bdc06a7 + .octa 0xc19bf174c19bf174c19bf174c19bf174 + .octa 0xc19bf174c19bf174c19bf174c19bf174 + .octa 0xe49b69c1e49b69c1e49b69c1e49b69c1 + .octa 0xe49b69c1e49b69c1e49b69c1e49b69c1 + .octa 0xefbe4786efbe4786efbe4786efbe4786 + .octa 0xefbe4786efbe4786efbe4786efbe4786 + .octa 0x0fc19dc60fc19dc60fc19dc60fc19dc6 + .octa 0x0fc19dc60fc19dc60fc19dc60fc19dc6 + .octa 0x240ca1cc240ca1cc240ca1cc240ca1cc + .octa 0x240ca1cc240ca1cc240ca1cc240ca1cc + .octa 0x2de92c6f2de92c6f2de92c6f2de92c6f + .octa 0x2de92c6f2de92c6f2de92c6f2de92c6f + .octa 0x4a7484aa4a7484aa4a7484aa4a7484aa + .octa 0x4a7484aa4a7484aa4a7484aa4a7484aa + .octa 0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc + .octa 0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc + .octa 0x76f988da76f988da76f988da76f988da + .octa 0x76f988da76f988da76f988da76f988da + .octa 0x983e5152983e5152983e5152983e5152 + .octa 0x983e5152983e5152983e5152983e5152 + .octa 0xa831c66da831c66da831c66da831c66d + .octa 0xa831c66da831c66da831c66da831c66d + .octa 0xb00327c8b00327c8b00327c8b00327c8 + .octa 0xb00327c8b00327c8b00327c8b00327c8 + .octa 0xbf597fc7bf597fc7bf597fc7bf597fc7 + .octa 0xbf597fc7bf597fc7bf597fc7bf597fc7 + .octa 0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3 + .octa 0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3 + .octa 0xd5a79147d5a79147d5a79147d5a79147 + .octa 0xd5a79147d5a79147d5a79147d5a79147 + .octa 0x06ca635106ca635106ca635106ca6351 + .octa 0x06ca635106ca635106ca635106ca6351 + .octa 0x14292967142929671429296714292967 + .octa 0x14292967142929671429296714292967 + .octa 0x27b70a8527b70a8527b70a8527b70a85 + .octa 0x27b70a8527b70a8527b70a8527b70a85 + .octa 0x2e1b21382e1b21382e1b21382e1b2138 + .octa 0x2e1b21382e1b21382e1b21382e1b2138 + .octa 0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc + .octa 0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc + .octa 0x53380d1353380d1353380d1353380d13 + .octa 0x53380d1353380d1353380d1353380d13 + .octa 0x650a7354650a7354650a7354650a7354 + .octa 0x650a7354650a7354650a7354650a7354 + .octa 0x766a0abb766a0abb766a0abb766a0abb + .octa 0x766a0abb766a0abb766a0abb766a0abb + .octa 0x81c2c92e81c2c92e81c2c92e81c2c92e + .octa 0x81c2c92e81c2c92e81c2c92e81c2c92e + .octa 0x92722c8592722c8592722c8592722c85 + .octa 0x92722c8592722c8592722c8592722c85 + .octa 0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1 + .octa 0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1 + .octa 0xa81a664ba81a664ba81a664ba81a664b + .octa 0xa81a664ba81a664ba81a664ba81a664b + .octa 0xc24b8b70c24b8b70c24b8b70c24b8b70 + .octa 0xc24b8b70c24b8b70c24b8b70c24b8b70 + .octa 0xc76c51a3c76c51a3c76c51a3c76c51a3 + .octa 0xc76c51a3c76c51a3c76c51a3c76c51a3 + .octa 0xd192e819d192e819d192e819d192e819 + .octa 0xd192e819d192e819d192e819d192e819 + .octa 0xd6990624d6990624d6990624d6990624 + .octa 0xd6990624d6990624d6990624d6990624 + .octa 0xf40e3585f40e3585f40e3585f40e3585 + .octa 0xf40e3585f40e3585f40e3585f40e3585 + .octa 0x106aa070106aa070106aa070106aa070 + .octa 0x106aa070106aa070106aa070106aa070 + .octa 0x19a4c11619a4c11619a4c11619a4c116 + .octa 0x19a4c11619a4c11619a4c11619a4c116 + .octa 0x1e376c081e376c081e376c081e376c08 + .octa 0x1e376c081e376c081e376c081e376c08 + .octa 0x2748774c2748774c2748774c2748774c + .octa 0x2748774c2748774c2748774c2748774c + .octa 0x34b0bcb534b0bcb534b0bcb534b0bcb5 + .octa 0x34b0bcb534b0bcb534b0bcb534b0bcb5 + .octa 0x391c0cb3391c0cb3391c0cb3391c0cb3 + .octa 0x391c0cb3391c0cb3391c0cb3391c0cb3 + .octa 0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a + .octa 0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a + .octa 0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f + .octa 0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f + .octa 0x682e6ff3682e6ff3682e6ff3682e6ff3 + .octa 0x682e6ff3682e6ff3682e6ff3682e6ff3 + .octa 0x748f82ee748f82ee748f82ee748f82ee + .octa 0x748f82ee748f82ee748f82ee748f82ee + .octa 0x78a5636f78a5636f78a5636f78a5636f + .octa 0x78a5636f78a5636f78a5636f78a5636f + .octa 0x84c8781484c8781484c8781484c87814 + .octa 0x84c8781484c8781484c8781484c87814 + .octa 0x8cc702088cc702088cc702088cc70208 + .octa 0x8cc702088cc702088cc702088cc70208 + .octa 0x90befffa90befffa90befffa90befffa + .octa 0x90befffa90befffa90befffa90befffa + .octa 0xa4506ceba4506ceba4506ceba4506ceb + .octa 0xa4506ceba4506ceba4506ceba4506ceb + .octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7 + .octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7 + .octa 0xc67178f2c67178f2c67178f2c67178f2 + .octa 0xc67178f2c67178f2c67178f2c67178f2 +PSHUFFLE_BYTE_FLIP_MASK: +.octa 0x0c0d0e0f08090a0b0405060700010203 +.octa 0x0c0d0e0f08090a0b0405060700010203 + +.align 64 +.global K256 +K256: + .int 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .int 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .int 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .int 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .int 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .int 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .int 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .int 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .int 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .int 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .int 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .int 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .int 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .int 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .int 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .int 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index 3ae0f43ebd37..9e79baf03a4b 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -427,4 +427,14 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); MODULE_ALIAS_CRYPTO("sha256"); +MODULE_ALIAS_CRYPTO("sha256-ssse3"); +MODULE_ALIAS_CRYPTO("sha256-avx"); +MODULE_ALIAS_CRYPTO("sha256-avx2"); MODULE_ALIAS_CRYPTO("sha224"); +MODULE_ALIAS_CRYPTO("sha224-ssse3"); +MODULE_ALIAS_CRYPTO("sha224-avx"); +MODULE_ALIAS_CRYPTO("sha224-avx2"); +#ifdef CONFIG_AS_SHA256_NI +MODULE_ALIAS_CRYPTO("sha256-ni"); +MODULE_ALIAS_CRYPTO("sha224-ni"); +#endif diff --git a/arch/x86/crypto/sha512-mb/Makefile b/arch/x86/crypto/sha512-mb/Makefile new file mode 100644 index 000000000000..0a57e2103980 --- /dev/null +++ b/arch/x86/crypto/sha512-mb/Makefile @@ -0,0 +1,11 @@ +# +# Arch-specific CryptoAPI modules. +# + +avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ + $(comma)4)$(comma)%ymm2,yes,no) +ifeq ($(avx2_supported),yes) + obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb.o + sha512-mb-y := sha512_mb.o sha512_mb_mgr_flush_avx2.o \ + sha512_mb_mgr_init_avx2.o sha512_mb_mgr_submit_avx2.o sha512_x4_avx2.o +endif diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c new file mode 100644 index 000000000000..f4cf5b78fd36 --- /dev/null +++ b/arch/x86/crypto/sha512-mb/sha512_mb.c @@ -0,0 +1,1046 @@ +/* + * Multi buffer SHA512 algorithm Glue Code + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sha512_mb_ctx.h" + +#define FLUSH_INTERVAL 1000 /* in usec */ + +static struct mcryptd_alg_state sha512_mb_alg_state; + +struct sha512_mb_ctx { + struct mcryptd_ahash *mcryptd_tfm; +}; + +static inline struct mcryptd_hash_request_ctx + *cast_hash_to_mcryptd_ctx(struct sha512_hash_ctx *hash_ctx) +{ + struct ahash_request *areq; + + areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); + return container_of(areq, struct mcryptd_hash_request_ctx, areq); +} + +static inline struct ahash_request + *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) +{ + return container_of((void *) ctx, struct ahash_request, __ctx); +} + +static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, + struct ahash_request *areq) +{ + rctx->flag = HASH_UPDATE; +} + +static asmlinkage void (*sha512_job_mgr_init)(struct sha512_mb_mgr *state); +static asmlinkage struct job_sha512* (*sha512_job_mgr_submit) + (struct sha512_mb_mgr *state, + struct job_sha512 *job); +static asmlinkage struct job_sha512* (*sha512_job_mgr_flush) + (struct sha512_mb_mgr *state); +static asmlinkage struct job_sha512* (*sha512_job_mgr_get_comp_job) + (struct sha512_mb_mgr *state); + +inline void sha512_init_digest(uint64_t *digest) +{ + static const uint64_t initial_digest[SHA512_DIGEST_LENGTH] = { + SHA512_H0, SHA512_H1, SHA512_H2, + SHA512_H3, SHA512_H4, SHA512_H5, + SHA512_H6, SHA512_H7 }; + memcpy(digest, initial_digest, sizeof(initial_digest)); +} + +inline uint32_t sha512_pad(uint8_t padblock[SHA512_BLOCK_SIZE * 2], + uint32_t total_len) +{ + uint32_t i = total_len & (SHA512_BLOCK_SIZE - 1); + + memset(&padblock[i], 0, SHA512_BLOCK_SIZE); + padblock[i] = 0x80; + + i += ((SHA512_BLOCK_SIZE - 1) & + (0 - (total_len + SHA512_PADLENGTHFIELD_SIZE + 1))) + + 1 + SHA512_PADLENGTHFIELD_SIZE; + +#if SHA512_PADLENGTHFIELD_SIZE == 16 + *((uint64_t *) &padblock[i - 16]) = 0; +#endif + + *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); + + /* Number of extra blocks to hash */ + return i >> SHA512_LOG2_BLOCK_SIZE; +} + +static struct sha512_hash_ctx *sha512_ctx_mgr_resubmit + (struct sha512_ctx_mgr *mgr, struct sha512_hash_ctx *ctx) +{ + while (ctx) { + if (ctx->status & HASH_CTX_STS_COMPLETE) { + /* Clear PROCESSING bit */ + ctx->status = HASH_CTX_STS_COMPLETE; + return ctx; + } + + /* + * If the extra blocks are empty, begin hashing what remains + * in the user's buffer. + */ + if (ctx->partial_block_buffer_length == 0 && + ctx->incoming_buffer_length) { + + const void *buffer = ctx->incoming_buffer; + uint32_t len = ctx->incoming_buffer_length; + uint32_t copy_len; + + /* + * Only entire blocks can be hashed. + * Copy remainder to extra blocks buffer. + */ + copy_len = len & (SHA512_BLOCK_SIZE-1); + + if (copy_len) { + len -= copy_len; + memcpy(ctx->partial_block_buffer, + ((const char *) buffer + len), + copy_len); + ctx->partial_block_buffer_length = copy_len; + } + + ctx->incoming_buffer_length = 0; + + /* len should be a multiple of the block size now */ + assert((len % SHA512_BLOCK_SIZE) == 0); + + /* Set len to the number of blocks to be hashed */ + len >>= SHA512_LOG2_BLOCK_SIZE; + + if (len) { + + ctx->job.buffer = (uint8_t *) buffer; + ctx->job.len = len; + ctx = (struct sha512_hash_ctx *) + sha512_job_mgr_submit(&mgr->mgr, + &ctx->job); + continue; + } + } + + /* + * If the extra blocks are not empty, then we are + * either on the last block(s) or we need more + * user input before continuing. + */ + if (ctx->status & HASH_CTX_STS_LAST) { + + uint8_t *buf = ctx->partial_block_buffer; + uint32_t n_extra_blocks = + sha512_pad(buf, ctx->total_length); + + ctx->status = (HASH_CTX_STS_PROCESSING | + HASH_CTX_STS_COMPLETE); + ctx->job.buffer = buf; + ctx->job.len = (uint32_t) n_extra_blocks; + ctx = (struct sha512_hash_ctx *) + sha512_job_mgr_submit(&mgr->mgr, &ctx->job); + continue; + } + + if (ctx) + ctx->status = HASH_CTX_STS_IDLE; + return ctx; + } + + return NULL; +} + +static struct sha512_hash_ctx + *sha512_ctx_mgr_get_comp_ctx(struct sha512_ctx_mgr *mgr) +{ + /* + * If get_comp_job returns NULL, there are no jobs complete. + * If get_comp_job returns a job, verify that it is safe to return to + * the user. + * If it is not ready, resubmit the job to finish processing. + * If sha512_ctx_mgr_resubmit returned a job, it is ready to be + * returned. + * Otherwise, all jobs currently being managed by the hash_ctx_mgr + * still need processing. + */ + struct sha512_hash_ctx *ctx; + + ctx = (struct sha512_hash_ctx *) + sha512_job_mgr_get_comp_job(&mgr->mgr); + return sha512_ctx_mgr_resubmit(mgr, ctx); +} + +static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr) +{ + sha512_job_mgr_init(&mgr->mgr); +} + +static struct sha512_hash_ctx + *sha512_ctx_mgr_submit(struct sha512_ctx_mgr *mgr, + struct sha512_hash_ctx *ctx, + const void *buffer, + uint32_t len, + int flags) +{ + if (flags & (~HASH_ENTIRE)) { + /* + * User should not pass anything other than FIRST, UPDATE, or + * LAST + */ + ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; + return ctx; + } + + if (ctx->status & HASH_CTX_STS_PROCESSING) { + /* Cannot submit to a currently processing job. */ + ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; + return ctx; + } + + if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) { + /* Cannot update a finished job. */ + ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; + return ctx; + } + + + if (flags & HASH_FIRST) { + /* Init digest */ + sha512_init_digest(ctx->job.result_digest); + + /* Reset byte counter */ + ctx->total_length = 0; + + /* Clear extra blocks */ + ctx->partial_block_buffer_length = 0; + } + + /* + * If we made it here, there were no errors during this call to + * submit + */ + ctx->error = HASH_CTX_ERROR_NONE; + + /* Store buffer ptr info from user */ + ctx->incoming_buffer = buffer; + ctx->incoming_buffer_length = len; + + /* + * Store the user's request flags and mark this ctx as currently being + * processed. + */ + ctx->status = (flags & HASH_LAST) ? + (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : + HASH_CTX_STS_PROCESSING; + + /* Advance byte counter */ + ctx->total_length += len; + + /* + * If there is anything currently buffered in the extra blocks, + * append to it until it contains a whole block. + * Or if the user's buffer contains less than a whole block, + * append as much as possible to the extra block. + */ + if (ctx->partial_block_buffer_length || len < SHA512_BLOCK_SIZE) { + /* Compute how many bytes to copy from user buffer into extra + * block + */ + uint32_t copy_len = SHA512_BLOCK_SIZE - + ctx->partial_block_buffer_length; + if (len < copy_len) + copy_len = len; + + if (copy_len) { + /* Copy and update relevant pointers and counters */ + memcpy + (&ctx->partial_block_buffer[ctx->partial_block_buffer_length], + buffer, copy_len); + + ctx->partial_block_buffer_length += copy_len; + ctx->incoming_buffer = (const void *) + ((const char *)buffer + copy_len); + ctx->incoming_buffer_length = len - copy_len; + } + + /* The extra block should never contain more than 1 block + * here + */ + assert(ctx->partial_block_buffer_length <= SHA512_BLOCK_SIZE); + + /* If the extra block buffer contains exactly 1 block, it can + * be hashed. + */ + if (ctx->partial_block_buffer_length >= SHA512_BLOCK_SIZE) { + ctx->partial_block_buffer_length = 0; + + ctx->job.buffer = ctx->partial_block_buffer; + ctx->job.len = 1; + ctx = (struct sha512_hash_ctx *) + sha512_job_mgr_submit(&mgr->mgr, &ctx->job); + } + } + + return sha512_ctx_mgr_resubmit(mgr, ctx); +} + +static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct sha512_ctx_mgr *mgr) +{ + struct sha512_hash_ctx *ctx; + + while (1) { + ctx = (struct sha512_hash_ctx *) + sha512_job_mgr_flush(&mgr->mgr); + + /* If flush returned 0, there are no more jobs in flight. */ + if (!ctx) + return NULL; + + /* + * If flush returned a job, resubmit the job to finish + * processing. + */ + ctx = sha512_ctx_mgr_resubmit(mgr, ctx); + + /* + * If sha512_ctx_mgr_resubmit returned a job, it is ready to + * be returned. Otherwise, all jobs currently being managed by + * the sha512_ctx_mgr still need processing. Loop. + */ + if (ctx) + return ctx; + } +} + +static int sha512_mb_init(struct ahash_request *areq) +{ + struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); + + hash_ctx_init(sctx); + sctx->job.result_digest[0] = SHA512_H0; + sctx->job.result_digest[1] = SHA512_H1; + sctx->job.result_digest[2] = SHA512_H2; + sctx->job.result_digest[3] = SHA512_H3; + sctx->job.result_digest[4] = SHA512_H4; + sctx->job.result_digest[5] = SHA512_H5; + sctx->job.result_digest[6] = SHA512_H6; + sctx->job.result_digest[7] = SHA512_H7; + sctx->total_length = 0; + sctx->partial_block_buffer_length = 0; + sctx->status = HASH_CTX_STS_IDLE; + + return 0; +} + +static int sha512_mb_set_results(struct mcryptd_hash_request_ctx *rctx) +{ + int i; + struct sha512_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); + __be64 *dst = (__be64 *) rctx->out; + + for (i = 0; i < 8; ++i) + dst[i] = cpu_to_be64(sctx->job.result_digest[i]); + + return 0; +} + +static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, + struct mcryptd_alg_cstate *cstate, bool flush) +{ + int flag = HASH_UPDATE; + int nbytes, err = 0; + struct mcryptd_hash_request_ctx *rctx = *ret_rctx; + struct sha512_hash_ctx *sha_ctx; + + /* more work ? */ + while (!(rctx->flag & HASH_DONE)) { + nbytes = crypto_ahash_walk_done(&rctx->walk, 0); + if (nbytes < 0) { + err = nbytes; + goto out; + } + /* check if the walk is done */ + if (crypto_ahash_walk_last(&rctx->walk)) { + rctx->flag |= HASH_DONE; + if (rctx->flag & HASH_FINAL) + flag |= HASH_LAST; + + } + sha_ctx = (struct sha512_hash_ctx *) + ahash_request_ctx(&rctx->areq); + kernel_fpu_begin(); + sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, + rctx->walk.data, nbytes, flag); + if (!sha_ctx) { + if (flush) + sha_ctx = sha512_ctx_mgr_flush(cstate->mgr); + } + kernel_fpu_end(); + if (sha_ctx) + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + else { + rctx = NULL; + goto out; + } + } + + /* copy the results */ + if (rctx->flag & HASH_FINAL) + sha512_mb_set_results(rctx); + +out: + *ret_rctx = rctx; + return err; +} + +static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, + struct mcryptd_alg_cstate *cstate, + int err) +{ + struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); + struct sha512_hash_ctx *sha_ctx; + struct mcryptd_hash_request_ctx *req_ctx; + int ret; + + /* remove from work list */ + spin_lock(&cstate->work_lock); + list_del(&rctx->waiter); + spin_unlock(&cstate->work_lock); + + if (irqs_disabled()) + rctx->complete(&req->base, err); + else { + local_bh_disable(); + rctx->complete(&req->base, err); + local_bh_enable(); + } + + /* check to see if there are other jobs that are done */ + sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate->mgr); + while (sha_ctx) { + req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); + ret = sha_finish_walk(&req_ctx, cstate, false); + if (req_ctx) { + spin_lock(&cstate->work_lock); + list_del(&req_ctx->waiter); + spin_unlock(&cstate->work_lock); + + req = cast_mcryptd_ctx_to_req(req_ctx); + if (irqs_disabled()) + rctx->complete(&req->base, ret); + else { + local_bh_disable(); + rctx->complete(&req->base, ret); + local_bh_enable(); + } + } + sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate->mgr); + } + + return 0; +} + +static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx, + struct mcryptd_alg_cstate *cstate) +{ + unsigned long next_flush; + unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); + + /* initialize tag */ + rctx->tag.arrival = jiffies; /* tag the arrival time */ + rctx->tag.seq_num = cstate->next_seq_num++; + next_flush = rctx->tag.arrival + delay; + rctx->tag.expire = next_flush; + + spin_lock(&cstate->work_lock); + list_add_tail(&rctx->waiter, &cstate->work_list); + spin_unlock(&cstate->work_lock); + + mcryptd_arm_flusher(cstate, delay); +} + +static int sha512_mb_update(struct ahash_request *areq) +{ + struct mcryptd_hash_request_ctx *rctx = + container_of(areq, struct mcryptd_hash_request_ctx, + areq); + struct mcryptd_alg_cstate *cstate = + this_cpu_ptr(sha512_mb_alg_state.alg_cstate); + + struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); + struct sha512_hash_ctx *sha_ctx; + int ret = 0, nbytes; + + + /* sanity check */ + if (rctx->tag.cpu != smp_processor_id()) { + pr_err("mcryptd error: cpu clash\n"); + goto done; + } + + /* need to init context */ + req_ctx_init(rctx, areq); + + nbytes = crypto_ahash_walk_first(req, &rctx->walk); + + if (nbytes < 0) { + ret = nbytes; + goto done; + } + + if (crypto_ahash_walk_last(&rctx->walk)) + rctx->flag |= HASH_DONE; + + /* submit */ + sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); + sha512_mb_add_list(rctx, cstate); + kernel_fpu_begin(); + sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, + nbytes, HASH_UPDATE); + kernel_fpu_end(); + + /* check if anything is returned */ + if (!sha_ctx) + return -EINPROGRESS; + + if (sha_ctx->error) { + ret = sha_ctx->error; + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + goto done; + } + + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + ret = sha_finish_walk(&rctx, cstate, false); + + if (!rctx) + return -EINPROGRESS; +done: + sha_complete_job(rctx, cstate, ret); + return ret; +} + +static int sha512_mb_finup(struct ahash_request *areq) +{ + struct mcryptd_hash_request_ctx *rctx = + container_of(areq, struct mcryptd_hash_request_ctx, + areq); + struct mcryptd_alg_cstate *cstate = + this_cpu_ptr(sha512_mb_alg_state.alg_cstate); + + struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); + struct sha512_hash_ctx *sha_ctx; + int ret = 0, flag = HASH_UPDATE, nbytes; + + /* sanity check */ + if (rctx->tag.cpu != smp_processor_id()) { + pr_err("mcryptd error: cpu clash\n"); + goto done; + } + + /* need to init context */ + req_ctx_init(rctx, areq); + + nbytes = crypto_ahash_walk_first(req, &rctx->walk); + + if (nbytes < 0) { + ret = nbytes; + goto done; + } + + if (crypto_ahash_walk_last(&rctx->walk)) { + rctx->flag |= HASH_DONE; + flag = HASH_LAST; + } + + /* submit */ + rctx->flag |= HASH_FINAL; + sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); + sha512_mb_add_list(rctx, cstate); + + kernel_fpu_begin(); + sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, + nbytes, flag); + kernel_fpu_end(); + + /* check if anything is returned */ + if (!sha_ctx) + return -EINPROGRESS; + + if (sha_ctx->error) { + ret = sha_ctx->error; + goto done; + } + + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + ret = sha_finish_walk(&rctx, cstate, false); + if (!rctx) + return -EINPROGRESS; +done: + sha_complete_job(rctx, cstate, ret); + return ret; +} + +static int sha512_mb_final(struct ahash_request *areq) +{ + struct mcryptd_hash_request_ctx *rctx = + container_of(areq, struct mcryptd_hash_request_ctx, + areq); + struct mcryptd_alg_cstate *cstate = + this_cpu_ptr(sha512_mb_alg_state.alg_cstate); + + struct sha512_hash_ctx *sha_ctx; + int ret = 0; + u8 data; + + /* sanity check */ + if (rctx->tag.cpu != smp_processor_id()) { + pr_err("mcryptd error: cpu clash\n"); + goto done; + } + + /* need to init context */ + req_ctx_init(rctx, areq); + + rctx->flag |= HASH_DONE | HASH_FINAL; + + sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); + /* flag HASH_FINAL and 0 data size */ + sha512_mb_add_list(rctx, cstate); + kernel_fpu_begin(); + sha_ctx = sha512_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, + HASH_LAST); + kernel_fpu_end(); + + /* check if anything is returned */ + if (!sha_ctx) + return -EINPROGRESS; + + if (sha_ctx->error) { + ret = sha_ctx->error; + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + goto done; + } + + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + ret = sha_finish_walk(&rctx, cstate, false); + if (!rctx) + return -EINPROGRESS; +done: + sha_complete_job(rctx, cstate, ret); + return ret; +} + +static int sha512_mb_export(struct ahash_request *areq, void *out) +{ + struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha512_mb_import(struct ahash_request *areq, const void *in) +{ + struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static int sha512_mb_async_init_tfm(struct crypto_tfm *tfm) +{ + struct mcryptd_ahash *mcryptd_tfm; + struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); + struct mcryptd_hash_ctx *mctx; + + mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha512-mb", + CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); + if (IS_ERR(mcryptd_tfm)) + return PTR_ERR(mcryptd_tfm); + mctx = crypto_ahash_ctx(&mcryptd_tfm->base); + mctx->alg_state = &sha512_mb_alg_state; + ctx->mcryptd_tfm = mcryptd_tfm; + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ahash_request) + + crypto_ahash_reqsize(&mcryptd_tfm->base)); + + return 0; +} + +static void sha512_mb_async_exit_tfm(struct crypto_tfm *tfm) +{ + struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); + + mcryptd_free_ahash(ctx->mcryptd_tfm); +} + +static int sha512_mb_areq_init_tfm(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ahash_request) + + sizeof(struct sha512_hash_ctx)); + + return 0; +} + +static void sha512_mb_areq_exit_tfm(struct crypto_tfm *tfm) +{ + struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); + + mcryptd_free_ahash(ctx->mcryptd_tfm); +} + +static struct ahash_alg sha512_mb_areq_alg = { + .init = sha512_mb_init, + .update = sha512_mb_update, + .final = sha512_mb_final, + .finup = sha512_mb_finup, + .export = sha512_mb_export, + .import = sha512_mb_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_hash_ctx), + .base = { + .cra_name = "__sha512-mb", + .cra_driver_name = "__intel_sha512-mb", + .cra_priority = 100, + /* + * use ASYNC flag as some buffers in multi-buffer + * algo may not have completed before hashing thread + * sleep + */ + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_INTERNAL, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT + (sha512_mb_areq_alg.halg.base.cra_list), + .cra_init = sha512_mb_areq_init_tfm, + .cra_exit = sha512_mb_areq_exit_tfm, + .cra_ctxsize = sizeof(struct sha512_hash_ctx), + } + } +}; + +static int sha512_mb_async_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_init(mcryptd_req); +} + +static int sha512_mb_async_update(struct ahash_request *req) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_update(mcryptd_req); +} + +static int sha512_mb_async_finup(struct ahash_request *req) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_finup(mcryptd_req); +} + +static int sha512_mb_async_final(struct ahash_request *req) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_final(mcryptd_req); +} + +static int sha512_mb_async_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_digest(mcryptd_req); +} + +static int sha512_mb_async_export(struct ahash_request *req, void *out) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + return crypto_ahash_export(mcryptd_req, out); +} + +static int sha512_mb_async_import(struct ahash_request *req, const void *in) +{ + struct ahash_request *mcryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); + struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; + struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); + struct mcryptd_hash_request_ctx *rctx; + struct ahash_request *areq; + + memcpy(mcryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); + rctx = ahash_request_ctx(mcryptd_req); + + areq = &rctx->areq; + + ahash_request_set_tfm(areq, child); + ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, + rctx->complete, req); + + return crypto_ahash_import(mcryptd_req, in); +} + +static struct ahash_alg sha512_mb_async_alg = { + .init = sha512_mb_async_init, + .update = sha512_mb_async_update, + .final = sha512_mb_async_final, + .finup = sha512_mb_async_finup, + .digest = sha512_mb_async_digest, + .export = sha512_mb_async_export, + .import = sha512_mb_async_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct sha512_hash_ctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512_mb", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_type = &crypto_ahash_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT + (sha512_mb_async_alg.halg.base.cra_list), + .cra_init = sha512_mb_async_init_tfm, + .cra_exit = sha512_mb_async_exit_tfm, + .cra_ctxsize = sizeof(struct sha512_mb_ctx), + .cra_alignmask = 0, + }, + }, +}; + +static unsigned long sha512_mb_flusher(struct mcryptd_alg_cstate *cstate) +{ + struct mcryptd_hash_request_ctx *rctx; + unsigned long cur_time; + unsigned long next_flush = 0; + struct sha512_hash_ctx *sha_ctx; + + + cur_time = jiffies; + + while (!list_empty(&cstate->work_list)) { + rctx = list_entry(cstate->work_list.next, + struct mcryptd_hash_request_ctx, waiter); + if time_before(cur_time, rctx->tag.expire) + break; + kernel_fpu_begin(); + sha_ctx = (struct sha512_hash_ctx *) + sha512_ctx_mgr_flush(cstate->mgr); + kernel_fpu_end(); + if (!sha_ctx) { + pr_err("sha512_mb error: nothing got flushed for" + " non-empty list\n"); + break; + } + rctx = cast_hash_to_mcryptd_ctx(sha_ctx); + sha_finish_walk(&rctx, cstate, true); + sha_complete_job(rctx, cstate, 0); + } + + if (!list_empty(&cstate->work_list)) { + rctx = list_entry(cstate->work_list.next, + struct mcryptd_hash_request_ctx, waiter); + /* get the hash context and then flush time */ + next_flush = rctx->tag.expire; + mcryptd_arm_flusher(cstate, get_delay(next_flush)); + } + return next_flush; +} + +static int __init sha512_mb_mod_init(void) +{ + + int cpu; + int err; + struct mcryptd_alg_cstate *cpu_state; + + /* check for dependent cpu features */ + if (!boot_cpu_has(X86_FEATURE_AVX2) || + !boot_cpu_has(X86_FEATURE_BMI2)) + return -ENODEV; + + /* initialize multibuffer structures */ + sha512_mb_alg_state.alg_cstate = + alloc_percpu(struct mcryptd_alg_cstate); + + sha512_job_mgr_init = sha512_mb_mgr_init_avx2; + sha512_job_mgr_submit = sha512_mb_mgr_submit_avx2; + sha512_job_mgr_flush = sha512_mb_mgr_flush_avx2; + sha512_job_mgr_get_comp_job = sha512_mb_mgr_get_comp_job_avx2; + + if (!sha512_mb_alg_state.alg_cstate) + return -ENOMEM; + for_each_possible_cpu(cpu) { + cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); + cpu_state->next_flush = 0; + cpu_state->next_seq_num = 0; + cpu_state->flusher_engaged = false; + INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); + cpu_state->cpu = cpu; + cpu_state->alg_state = &sha512_mb_alg_state; + cpu_state->mgr = kzalloc(sizeof(struct sha512_ctx_mgr), + GFP_KERNEL); + if (!cpu_state->mgr) + goto err2; + sha512_ctx_mgr_init(cpu_state->mgr); + INIT_LIST_HEAD(&cpu_state->work_list); + spin_lock_init(&cpu_state->work_lock); + } + sha512_mb_alg_state.flusher = &sha512_mb_flusher; + + err = crypto_register_ahash(&sha512_mb_areq_alg); + if (err) + goto err2; + err = crypto_register_ahash(&sha512_mb_async_alg); + if (err) + goto err1; + + + return 0; +err1: + crypto_unregister_ahash(&sha512_mb_areq_alg); +err2: + for_each_possible_cpu(cpu) { + cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); + kfree(cpu_state->mgr); + } + free_percpu(sha512_mb_alg_state.alg_cstate); + return -ENODEV; +} + +static void __exit sha512_mb_mod_fini(void) +{ + int cpu; + struct mcryptd_alg_cstate *cpu_state; + + crypto_unregister_ahash(&sha512_mb_async_alg); + crypto_unregister_ahash(&sha512_mb_areq_alg); + for_each_possible_cpu(cpu) { + cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); + kfree(cpu_state->mgr); + } + free_percpu(sha512_mb_alg_state.alg_cstate); +} + +module_init(sha512_mb_mod_init); +module_exit(sha512_mb_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, multi buffer accelerated"); + +MODULE_ALIAS("sha512"); diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h b/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h new file mode 100644 index 000000000000..9d4b2c8208d5 --- /dev/null +++ b/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h @@ -0,0 +1,130 @@ +/* + * Header file for multi buffer SHA512 context + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SHA_MB_CTX_INTERNAL_H +#define _SHA_MB_CTX_INTERNAL_H + +#include "sha512_mb_mgr.h" + +#define HASH_UPDATE 0x00 +#define HASH_FIRST 0x01 +#define HASH_LAST 0x02 +#define HASH_ENTIRE 0x03 +#define HASH_DONE 0x04 +#define HASH_FINAL 0x08 + +#define HASH_CTX_STS_IDLE 0x00 +#define HASH_CTX_STS_PROCESSING 0x01 +#define HASH_CTX_STS_LAST 0x02 +#define HASH_CTX_STS_COMPLETE 0x04 + +enum hash_ctx_error { + HASH_CTX_ERROR_NONE = 0, + HASH_CTX_ERROR_INVALID_FLAGS = -1, + HASH_CTX_ERROR_ALREADY_PROCESSING = -2, + HASH_CTX_ERROR_ALREADY_COMPLETED = -3, +}; + +#define hash_ctx_user_data(ctx) ((ctx)->user_data) +#define hash_ctx_digest(ctx) ((ctx)->job.result_digest) +#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING) +#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE) +#define hash_ctx_status(ctx) ((ctx)->status) +#define hash_ctx_error(ctx) ((ctx)->error) +#define hash_ctx_init(ctx) \ + do { \ + (ctx)->error = HASH_CTX_ERROR_NONE; \ + (ctx)->status = HASH_CTX_STS_COMPLETE; \ + } while (0) + +/* Hash Constants and Typedefs */ +#define SHA512_DIGEST_LENGTH 8 +#define SHA512_LOG2_BLOCK_SIZE 7 + +#define SHA512_PADLENGTHFIELD_SIZE 16 + +#ifdef SHA_MB_DEBUG +#define assert(expr) \ +do { \ + if (unlikely(!(expr))) { \ + printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \ + #expr, __FILE__, __func__, __LINE__); \ + } \ +} while (0) +#else +#define assert(expr) do {} while (0) +#endif + +struct sha512_ctx_mgr { + struct sha512_mb_mgr mgr; +}; + +/* typedef struct sha512_ctx_mgr sha512_ctx_mgr; */ + +struct sha512_hash_ctx { + /* Must be at struct offset 0 */ + struct job_sha512 job; + /* status flag */ + int status; + /* error flag */ + int error; + + uint32_t total_length; + const void *incoming_buffer; + uint32_t incoming_buffer_length; + uint8_t partial_block_buffer[SHA512_BLOCK_SIZE * 2]; + uint32_t partial_block_buffer_length; + void *user_data; +}; + +#endif diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h b/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h new file mode 100644 index 000000000000..178f17eef382 --- /dev/null +++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h @@ -0,0 +1,104 @@ +/* + * Header file for multi buffer SHA512 algorithm manager + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SHA_MB_MGR_H +#define __SHA_MB_MGR_H + +#include + +#define NUM_SHA512_DIGEST_WORDS 8 + +enum job_sts {STS_UNKNOWN = 0, + STS_BEING_PROCESSED = 1, + STS_COMPLETED = 2, + STS_INTERNAL_ERROR = 3, + STS_ERROR = 4 +}; + +struct job_sha512 { + u8 *buffer; + u64 len; + u64 result_digest[NUM_SHA512_DIGEST_WORDS] __aligned(32); + enum job_sts status; + void *user_data; +}; + +struct sha512_args_x4 { + uint64_t digest[8][4]; + uint8_t *data_ptr[4]; +}; + +struct sha512_lane_data { + struct job_sha512 *job_in_lane; +}; + +struct sha512_mb_mgr { + struct sha512_args_x4 args; + + uint64_t lens[4]; + + /* each byte is index (0...7) of unused lanes */ + uint64_t unused_lanes; + /* byte 4 is set to FF as a flag */ + struct sha512_lane_data ldata[4]; +}; + +#define SHA512_MB_MGR_NUM_LANES_AVX2 4 + +void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state); +struct job_sha512 *sha512_mb_mgr_submit_avx2(struct sha512_mb_mgr *state, + struct job_sha512 *job); +struct job_sha512 *sha512_mb_mgr_flush_avx2(struct sha512_mb_mgr *state); +struct job_sha512 *sha512_mb_mgr_get_comp_job_avx2(struct sha512_mb_mgr *state); + +#endif diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S new file mode 100644 index 000000000000..cf2636d4c9ba --- /dev/null +++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S @@ -0,0 +1,281 @@ +/* + * Header file for multi buffer SHA256 algorithm data structure + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +# Macros for defining data structures + +# Usage example + +#START_FIELDS # JOB_AES +### name size align +#FIELD _plaintext, 8, 8 # pointer to plaintext +#FIELD _ciphertext, 8, 8 # pointer to ciphertext +#FIELD _IV, 16, 8 # IV +#FIELD _keys, 8, 8 # pointer to keys +#FIELD _len, 4, 4 # length in bytes +#FIELD _status, 4, 4 # status enumeration +#FIELD _user_data, 8, 8 # pointer to user data +#UNION _union, size1, align1, \ +# size2, align2, \ +# size3, align3, \ +# ... +#END_FIELDS +#%assign _JOB_AES_size _FIELD_OFFSET +#%assign _JOB_AES_align _STRUCT_ALIGN + +######################################################################### + +# Alternate "struc-like" syntax: +# STRUCT job_aes2 +# RES_Q .plaintext, 1 +# RES_Q .ciphertext, 1 +# RES_DQ .IV, 1 +# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN +# RES_U .union, size1, align1, \ +# size2, align2, \ +# ... +# ENDSTRUCT +# # Following only needed if nesting +# %assign job_aes2_size _FIELD_OFFSET +# %assign job_aes2_align _STRUCT_ALIGN +# +# RES_* macros take a name, a count and an optional alignment. +# The count in in terms of the base size of the macro, and the +# default alignment is the base size. +# The macros are: +# Macro Base size +# RES_B 1 +# RES_W 2 +# RES_D 4 +# RES_Q 8 +# RES_DQ 16 +# RES_Y 32 +# RES_Z 64 +# +# RES_U defines a union. It's arguments are a name and two or more +# pairs of "size, alignment" +# +# The two assigns are only needed if this structure is being nested +# within another. Even if the assigns are not done, one can still use +# STRUCT_NAME_size as the size of the structure. +# +# Note that for nesting, you still need to assign to STRUCT_NAME_size. +# +# The differences between this and using "struc" directly are that each +# type is implicitly aligned to its natural length (although this can be +# over-ridden with an explicit third parameter), and that the structure +# is padded at the end to its overall alignment. +# + +######################################################################### + +#ifndef _DATASTRUCT_ASM_ +#define _DATASTRUCT_ASM_ + +#define PTR_SZ 8 +#define SHA512_DIGEST_WORD_SIZE 8 +#define SHA512_MB_MGR_NUM_LANES_AVX2 4 +#define NUM_SHA512_DIGEST_WORDS 8 +#define SZ4 4*SHA512_DIGEST_WORD_SIZE +#define ROUNDS 80*SZ4 +#define SHA512_DIGEST_ROW_SIZE (SHA512_MB_MGR_NUM_LANES_AVX2 * 8) + +# START_FIELDS +.macro START_FIELDS + _FIELD_OFFSET = 0 + _STRUCT_ALIGN = 0 +.endm + +# FIELD name size align +.macro FIELD name size align + _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1)) + \name = _FIELD_OFFSET + _FIELD_OFFSET = _FIELD_OFFSET + (\size) +.if (\align > _STRUCT_ALIGN) + _STRUCT_ALIGN = \align +.endif +.endm + +# END_FIELDS +.macro END_FIELDS + _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) +.endm + +.macro STRUCT p1 +START_FIELDS +.struc \p1 +.endm + +.macro ENDSTRUCT + tmp = _FIELD_OFFSET + END_FIELDS + tmp = (_FIELD_OFFSET - ##tmp) +.if (tmp > 0) + .lcomm tmp +.endm + +## RES_int name size align +.macro RES_int p1 p2 p3 + name = \p1 + size = \p2 + align = .\p3 + + _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1)) +.align align +.lcomm name size + _FIELD_OFFSET = _FIELD_OFFSET + (size) +.if (align > _STRUCT_ALIGN) + _STRUCT_ALIGN = align +.endif +.endm + +# macro RES_B name, size [, align] +.macro RES_B _name, _size, _align=1 +RES_int _name _size _align +.endm + +# macro RES_W name, size [, align] +.macro RES_W _name, _size, _align=2 +RES_int _name 2*(_size) _align +.endm + +# macro RES_D name, size [, align] +.macro RES_D _name, _size, _align=4 +RES_int _name 4*(_size) _align +.endm + +# macro RES_Q name, size [, align] +.macro RES_Q _name, _size, _align=8 +RES_int _name 8*(_size) _align +.endm + +# macro RES_DQ name, size [, align] +.macro RES_DQ _name, _size, _align=16 +RES_int _name 16*(_size) _align +.endm + +# macro RES_Y name, size [, align] +.macro RES_Y _name, _size, _align=32 +RES_int _name 32*(_size) _align +.endm + +# macro RES_Z name, size [, align] +.macro RES_Z _name, _size, _align=64 +RES_int _name 64*(_size) _align +.endm + +#endif + +################################################################### +### Define SHA512 Out Of Order Data Structures +################################################################### + +START_FIELDS # LANE_DATA +### name size align +FIELD _job_in_lane, 8, 8 # pointer to job object +END_FIELDS + + _LANE_DATA_size = _FIELD_OFFSET + _LANE_DATA_align = _STRUCT_ALIGN + +#################################################################### + +START_FIELDS # SHA512_ARGS_X4 +### name size align +FIELD _digest, 8*8*4, 4 # transposed digest +FIELD _data_ptr, 8*4, 8 # array of pointers to data +END_FIELDS + + _SHA512_ARGS_X4_size = _FIELD_OFFSET + _SHA512_ARGS_X4_align = _STRUCT_ALIGN + +##################################################################### + +START_FIELDS # MB_MGR +### name size align +FIELD _args, _SHA512_ARGS_X4_size, _SHA512_ARGS_X4_align +FIELD _lens, 8*4, 8 +FIELD _unused_lanes, 8, 8 +FIELD _ldata, _LANE_DATA_size*4, _LANE_DATA_align +END_FIELDS + + _MB_MGR_size = _FIELD_OFFSET + _MB_MGR_align = _STRUCT_ALIGN + +_args_digest = _args + _digest +_args_data_ptr = _args + _data_ptr + +####################################################################### + +####################################################################### +#### Define constants +####################################################################### + +#define STS_UNKNOWN 0 +#define STS_BEING_PROCESSED 1 +#define STS_COMPLETED 2 + +####################################################################### +#### Define JOB_SHA512 structure +####################################################################### + +START_FIELDS # JOB_SHA512 +### name size align +FIELD _buffer, 8, 8 # pointer to buffer +FIELD _len, 8, 8 # length in bytes +FIELD _result_digest, 8*8, 32 # Digest (output) +FIELD _status, 4, 4 +FIELD _user_data, 8, 8 +END_FIELDS + + _JOB_SHA512_size = _FIELD_OFFSET + _JOB_SHA512_align = _STRUCT_ALIGN diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S new file mode 100644 index 000000000000..3ddba19a0db6 --- /dev/null +++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S @@ -0,0 +1,291 @@ +/* + * Flush routine for SHA512 multibuffer + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include "sha512_mb_mgr_datastruct.S" + +.extern sha512_x4_avx2 + +# LINUX register definitions +#define arg1 %rdi +#define arg2 %rsi + +# idx needs to be other than arg1, arg2, rbx, r12 +#define idx %rdx + +# Common definitions +#define state arg1 +#define job arg2 +#define len2 arg2 + +#define unused_lanes %rbx +#define lane_data %rbx +#define tmp2 %rbx + +#define job_rax %rax +#define tmp1 %rax +#define size_offset %rax +#define tmp %rax +#define start_offset %rax + +#define tmp3 arg1 + +#define extra_blocks arg2 +#define p arg2 + +#define tmp4 %r8 +#define lens0 %r8 + +#define lens1 %r9 +#define lens2 %r10 +#define lens3 %r11 + +.macro LABEL prefix n +\prefix\n\(): +.endm + +.macro JNE_SKIP i +jne skip_\i +.endm + +.altmacro +.macro SET_OFFSET _offset +offset = \_offset +.endm +.noaltmacro + +# JOB* sha512_mb_mgr_flush_avx2(MB_MGR *state) +# arg 1 : rcx : state +ENTRY(sha512_mb_mgr_flush_avx2) + FRAME_BEGIN + push %rbx + + # If bit (32+3) is set, then all lanes are empty + mov _unused_lanes(state), unused_lanes + bt $32+7, unused_lanes + jc return_null + + # find a lane with a non-null job + xor idx, idx + offset = (_ldata + 1*_LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne one(%rip), idx + offset = (_ldata + 2*_LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne two(%rip), idx + offset = (_ldata + 3*_LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) + cmovne three(%rip), idx + + # copy idx to empty lanes +copy_lane_data: + offset = (_args + _data_ptr) + mov offset(state,idx,8), tmp + + I = 0 +.rep 4 + offset = (_ldata + I * _LANE_DATA_size + _job_in_lane) + cmpq $0, offset(state) +.altmacro + JNE_SKIP %I + offset = (_args + _data_ptr + 8*I) + mov tmp, offset(state) + offset = (_lens + 8*I +4) + movl $0xFFFFFFFF, offset(state) +LABEL skip_ %I + I = (I+1) +.noaltmacro +.endr + + # Find min length + mov _lens + 0*8(state),lens0 + mov lens0,idx + mov _lens + 1*8(state),lens1 + cmp idx,lens1 + cmovb lens1,idx + mov _lens + 2*8(state),lens2 + cmp idx,lens2 + cmovb lens2,idx + mov _lens + 3*8(state),lens3 + cmp idx,lens3 + cmovb lens3,idx + mov idx,len2 + and $0xF,idx + and $~0xFF,len2 + jz len_is_0 + + sub len2, lens0 + sub len2, lens1 + sub len2, lens2 + sub len2, lens3 + shr $32,len2 + mov lens0, _lens + 0*8(state) + mov lens1, _lens + 1*8(state) + mov lens2, _lens + 2*8(state) + mov lens3, _lens + 3*8(state) + + # "state" and "args" are the same address, arg1 + # len is arg2 + call sha512_x4_avx2 + # state and idx are intact + +len_is_0: + # process completed job "idx" + imul $_LANE_DATA_size, idx, lane_data + lea _ldata(state, lane_data), lane_data + + mov _job_in_lane(lane_data), job_rax + movq $0, _job_in_lane(lane_data) + movl $STS_COMPLETED, _status(job_rax) + mov _unused_lanes(state), unused_lanes + shl $8, unused_lanes + or idx, unused_lanes + mov unused_lanes, _unused_lanes(state) + + movl $0xFFFFFFFF, _lens+4(state, idx, 8) + + vmovq _args_digest+0*32(state, idx, 8), %xmm0 + vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0 + vmovq _args_digest+2*32(state, idx, 8), %xmm1 + vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1 + vmovq _args_digest+4*32(state, idx, 8), %xmm2 + vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2 + vmovq _args_digest+6*32(state, idx, 8), %xmm3 + vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3 + + vmovdqu %xmm0, _result_digest(job_rax) + vmovdqu %xmm1, _result_digest+1*16(job_rax) + vmovdqu %xmm2, _result_digest+2*16(job_rax) + vmovdqu %xmm3, _result_digest+3*16(job_rax) + +return: + pop %rbx + FRAME_END + ret + +return_null: + xor job_rax, job_rax + jmp return +ENDPROC(sha512_mb_mgr_flush_avx2) +.align 16 + +ENTRY(sha512_mb_mgr_get_comp_job_avx2) + push %rbx + + mov _unused_lanes(state), unused_lanes + bt $(32+7), unused_lanes + jc .return_null + + # Find min length + mov _lens(state),lens0 + mov lens0,idx + mov _lens+1*8(state),lens1 + cmp idx,lens1 + cmovb lens1,idx + mov _lens+2*8(state),lens2 + cmp idx,lens2 + cmovb lens2,idx + mov _lens+3*8(state),lens3 + cmp idx,lens3 + cmovb lens3,idx + test $~0xF,idx + jnz .return_null + and $0xF,idx + + #process completed job "idx" + imul $_LANE_DATA_size, idx, lane_data + lea _ldata(state, lane_data), lane_data + + mov _job_in_lane(lane_data), job_rax + movq $0, _job_in_lane(lane_data) + movl $STS_COMPLETED, _status(job_rax) + mov _unused_lanes(state), unused_lanes + shl $8, unused_lanes + or idx, unused_lanes + mov unused_lanes, _unused_lanes(state) + + movl $0xFFFFFFFF, _lens+4(state, idx, 8) + + vmovq _args_digest(state, idx, 8), %xmm0 + vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0 + vmovq _args_digest+2*32(state, idx, 8), %xmm1 + vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1 + vmovq _args_digest+4*32(state, idx, 8), %xmm2 + vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2 + vmovq _args_digest+6*32(state, idx, 8), %xmm3 + vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3 + + vmovdqu %xmm0, _result_digest+0*16(job_rax) + vmovdqu %xmm1, _result_digest+1*16(job_rax) + vmovdqu %xmm2, _result_digest+2*16(job_rax) + vmovdqu %xmm3, _result_digest+3*16(job_rax) + + pop %rbx + + ret + +.return_null: + xor job_rax, job_rax + pop %rbx + ret +ENDPROC(sha512_mb_mgr_get_comp_job_avx2) +.data + +.align 16 +one: +.quad 1 +two: +.quad 2 +three: +.quad 3 diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c new file mode 100644 index 000000000000..36870b26067a --- /dev/null +++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c @@ -0,0 +1,67 @@ +/* + * Initialization code for multi buffer SHA256 algorithm for AVX2 + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "sha512_mb_mgr.h" + +void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state) +{ + unsigned int j; + + state->lens[0] = 0; + state->lens[1] = 1; + state->lens[2] = 2; + state->lens[3] = 3; + state->unused_lanes = 0xFF03020100; + for (j = 0; j < 4; j++) + state->ldata[j].job_in_lane = NULL; +} diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S new file mode 100644 index 000000000000..815f07bdd1f8 --- /dev/null +++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S @@ -0,0 +1,222 @@ +/* + * Buffer submit code for multi buffer SHA512 algorithm + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include "sha512_mb_mgr_datastruct.S" + +.extern sha512_x4_avx2 + +#define arg1 %rdi +#define arg2 %rsi + +#define idx %rdx +#define last_len %rdx + +#define size_offset %rcx +#define tmp2 %rcx + +# Common definitions +#define state arg1 +#define job arg2 +#define len2 arg2 +#define p2 arg2 + +#define p %r11 +#define start_offset %r11 + +#define unused_lanes %rbx + +#define job_rax %rax +#define len %rax + +#define lane %r12 +#define tmp3 %r12 +#define lens3 %r12 + +#define extra_blocks %r8 +#define lens0 %r8 + +#define tmp %r9 +#define lens1 %r9 + +#define lane_data %r10 +#define lens2 %r10 + +#define DWORD_len %eax + +# JOB* sha512_mb_mgr_submit_avx2(MB_MGR *state, JOB *job) +# arg 1 : rcx : state +# arg 2 : rdx : job +ENTRY(sha512_mb_mgr_submit_avx2) + FRAME_BEGIN + push %rbx + push %r12 + + mov _unused_lanes(state), unused_lanes + movzb %bl,lane + shr $8, unused_lanes + imul $_LANE_DATA_size, lane,lane_data + movl $STS_BEING_PROCESSED, _status(job) + lea _ldata(state, lane_data), lane_data + mov unused_lanes, _unused_lanes(state) + movl _len(job), DWORD_len + + mov job, _job_in_lane(lane_data) + movl DWORD_len,_lens+4(state , lane, 8) + + # Load digest words from result_digest + vmovdqu _result_digest+0*16(job), %xmm0 + vmovdqu _result_digest+1*16(job), %xmm1 + vmovdqu _result_digest+2*16(job), %xmm2 + vmovdqu _result_digest+3*16(job), %xmm3 + + vmovq %xmm0, _args_digest(state, lane, 8) + vpextrq $1, %xmm0, _args_digest+1*32(state , lane, 8) + vmovq %xmm1, _args_digest+2*32(state , lane, 8) + vpextrq $1, %xmm1, _args_digest+3*32(state , lane, 8) + vmovq %xmm2, _args_digest+4*32(state , lane, 8) + vpextrq $1, %xmm2, _args_digest+5*32(state , lane, 8) + vmovq %xmm3, _args_digest+6*32(state , lane, 8) + vpextrq $1, %xmm3, _args_digest+7*32(state , lane, 8) + + mov _buffer(job), p + mov p, _args_data_ptr(state, lane, 8) + + cmp $0xFF, unused_lanes + jne return_null + +start_loop: + + # Find min length + mov _lens+0*8(state),lens0 + mov lens0,idx + mov _lens+1*8(state),lens1 + cmp idx,lens1 + cmovb lens1, idx + mov _lens+2*8(state),lens2 + cmp idx,lens2 + cmovb lens2,idx + mov _lens+3*8(state),lens3 + cmp idx,lens3 + cmovb lens3,idx + mov idx,len2 + and $0xF,idx + and $~0xFF,len2 + jz len_is_0 + + sub len2,lens0 + sub len2,lens1 + sub len2,lens2 + sub len2,lens3 + shr $32,len2 + mov lens0, _lens + 0*8(state) + mov lens1, _lens + 1*8(state) + mov lens2, _lens + 2*8(state) + mov lens3, _lens + 3*8(state) + + # "state" and "args" are the same address, arg1 + # len is arg2 + call sha512_x4_avx2 + # state and idx are intact + +len_is_0: + + # process completed job "idx" + imul $_LANE_DATA_size, idx, lane_data + lea _ldata(state, lane_data), lane_data + + mov _job_in_lane(lane_data), job_rax + mov _unused_lanes(state), unused_lanes + movq $0, _job_in_lane(lane_data) + movl $STS_COMPLETED, _status(job_rax) + shl $8, unused_lanes + or idx, unused_lanes + mov unused_lanes, _unused_lanes(state) + + movl $0xFFFFFFFF,_lens+4(state,idx,8) + vmovq _args_digest+0*32(state , idx, 8), %xmm0 + vpinsrq $1, _args_digest+1*32(state , idx, 8), %xmm0, %xmm0 + vmovq _args_digest+2*32(state , idx, 8), %xmm1 + vpinsrq $1, _args_digest+3*32(state , idx, 8), %xmm1, %xmm1 + vmovq _args_digest+4*32(state , idx, 8), %xmm2 + vpinsrq $1, _args_digest+5*32(state , idx, 8), %xmm2, %xmm2 + vmovq _args_digest+6*32(state , idx, 8), %xmm3 + vpinsrq $1, _args_digest+7*32(state , idx, 8), %xmm3, %xmm3 + + vmovdqu %xmm0, _result_digest + 0*16(job_rax) + vmovdqu %xmm1, _result_digest + 1*16(job_rax) + vmovdqu %xmm2, _result_digest + 2*16(job_rax) + vmovdqu %xmm3, _result_digest + 3*16(job_rax) + +return: + pop %r12 + pop %rbx + FRAME_END + ret + +return_null: + xor job_rax, job_rax + jmp return +ENDPROC(sha512_mb_mgr_submit_avx2) +.data + +.align 16 +H0: .int 0x6a09e667 +H1: .int 0xbb67ae85 +H2: .int 0x3c6ef372 +H3: .int 0xa54ff53a +H4: .int 0x510e527f +H5: .int 0x9b05688c +H6: .int 0x1f83d9ab +H7: .int 0x5be0cd19 diff --git a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S new file mode 100644 index 000000000000..31ab1eff6413 --- /dev/null +++ b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S @@ -0,0 +1,529 @@ +/* + * Multi-buffer SHA512 algorithm hash compute routine + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2016 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Contact Information: + * Megha Dey + * + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +# code to compute quad SHA512 using AVX2 +# use YMMs to tackle the larger digest size +# outer calling routine takes care of save and restore of XMM registers +# Logic designed/laid out by JDG + +# Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; ymm0-15 +# Stack must be aligned to 32 bytes before call +# Linux clobbers: rax rbx rcx rsi r8 r9 r10 r11 r12 +# Linux preserves: rcx rdx rdi rbp r13 r14 r15 +# clobbers ymm0-15 + +#include +#include "sha512_mb_mgr_datastruct.S" + +arg1 = %rdi +arg2 = %rsi + +# Common definitions +STATE = arg1 +INP_SIZE = arg2 + +IDX = %rax +ROUND = %rbx +TBL = %r8 + +inp0 = %r9 +inp1 = %r10 +inp2 = %r11 +inp3 = %r12 + +a = %ymm0 +b = %ymm1 +c = %ymm2 +d = %ymm3 +e = %ymm4 +f = %ymm5 +g = %ymm6 +h = %ymm7 + +a0 = %ymm8 +a1 = %ymm9 +a2 = %ymm10 + +TT0 = %ymm14 +TT1 = %ymm13 +TT2 = %ymm12 +TT3 = %ymm11 +TT4 = %ymm10 +TT5 = %ymm9 + +T1 = %ymm14 +TMP = %ymm15 + +# Define stack usage +STACK_SPACE1 = SZ4*16 + NUM_SHA512_DIGEST_WORDS*SZ4 + 24 + +#define VMOVPD vmovupd +_digest = SZ4*16 + +# transpose r0, r1, r2, r3, t0, t1 +# "transpose" data in {r0..r3} using temps {t0..t3} +# Input looks like: {r0 r1 r2 r3} +# r0 = {a7 a6 a5 a4 a3 a2 a1 a0} +# r1 = {b7 b6 b5 b4 b3 b2 b1 b0} +# r2 = {c7 c6 c5 c4 c3 c2 c1 c0} +# r3 = {d7 d6 d5 d4 d3 d2 d1 d0} +# +# output looks like: {t0 r1 r0 r3} +# t0 = {d1 d0 c1 c0 b1 b0 a1 a0} +# r1 = {d3 d2 c3 c2 b3 b2 a3 a2} +# r0 = {d5 d4 c5 c4 b5 b4 a5 a4} +# r3 = {d7 d6 c7 c6 b7 b6 a7 a6} + +.macro TRANSPOSE r0 r1 r2 r3 t0 t1 + vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0} + vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2} + vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0} + vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2} + + vperm2f128 $0x20, \r2, \r0, \r1 # h6...a6 + vperm2f128 $0x31, \r2, \r0, \r3 # h2...a2 + vperm2f128 $0x31, \t1, \t0, \r0 # h5...a5 + vperm2f128 $0x20, \t1, \t0, \t0 # h1...a1 +.endm + +.macro ROTATE_ARGS +TMP_ = h +h = g +g = f +f = e +e = d +d = c +c = b +b = a +a = TMP_ +.endm + +# PRORQ reg, imm, tmp +# packed-rotate-right-double +# does a rotate by doing two shifts and an or +.macro _PRORQ reg imm tmp + vpsllq $(64-\imm),\reg,\tmp + vpsrlq $\imm,\reg, \reg + vpor \tmp,\reg, \reg +.endm + +# non-destructive +# PRORQ_nd reg, imm, tmp, src +.macro _PRORQ_nd reg imm tmp src + vpsllq $(64-\imm), \src, \tmp + vpsrlq $\imm, \src, \reg + vpor \tmp, \reg, \reg +.endm + +# PRORQ dst/src, amt +.macro PRORQ reg imm + _PRORQ \reg, \imm, TMP +.endm + +# PRORQ_nd dst, src, amt +.macro PRORQ_nd reg tmp imm + _PRORQ_nd \reg, \imm, TMP, \tmp +.endm + +#; arguments passed implicitly in preprocessor symbols i, a...h +.macro ROUND_00_15 _T1 i + PRORQ_nd a0, e, (18-14) # sig1: a0 = (e >> 4) + + vpxor g, f, a2 # ch: a2 = f^g + vpand e,a2, a2 # ch: a2 = (f^g)&e + vpxor g, a2, a2 # a2 = ch + + PRORQ_nd a1,e,41 # sig1: a1 = (e >> 25) + + offset = SZ4*(\i & 0xf) + vmovdqu \_T1,offset(%rsp) + vpaddq (TBL,ROUND,1), \_T1, \_T1 # T1 = W + K + vpxor e,a0, a0 # sig1: a0 = e ^ (e >> 5) + PRORQ a0, 14 # sig1: a0 = (e >> 6) ^ (e >> 11) + vpaddq a2, h, h # h = h + ch + PRORQ_nd a2,a,6 # sig0: a2 = (a >> 11) + vpaddq \_T1,h, h # h = h + ch + W + K + vpxor a1, a0, a0 # a0 = sigma1 + vmovdqu a,\_T1 + PRORQ_nd a1,a,39 # sig0: a1 = (a >> 22) + vpxor c, \_T1, \_T1 # maj: T1 = a^c + add $SZ4, ROUND # ROUND++ + vpand b, \_T1, \_T1 # maj: T1 = (a^c)&b + vpaddq a0, h, h + vpaddq h, d, d + vpxor a, a2, a2 # sig0: a2 = a ^ (a >> 11) + PRORQ a2,28 # sig0: a2 = (a >> 2) ^ (a >> 13) + vpxor a1, a2, a2 # a2 = sig0 + vpand c, a, a1 # maj: a1 = a&c + vpor \_T1, a1, a1 # a1 = maj + vpaddq a1, h, h # h = h + ch + W + K + maj + vpaddq a2, h, h # h = h + ch + W + K + maj + sigma0 + ROTATE_ARGS +.endm + + +#; arguments passed implicitly in preprocessor symbols i, a...h +.macro ROUND_16_XX _T1 i + vmovdqu SZ4*((\i-15)&0xf)(%rsp), \_T1 + vmovdqu SZ4*((\i-2)&0xf)(%rsp), a1 + vmovdqu \_T1, a0 + PRORQ \_T1,7 + vmovdqu a1, a2 + PRORQ a1,42 + vpxor a0, \_T1, \_T1 + PRORQ \_T1, 1 + vpxor a2, a1, a1 + PRORQ a1, 19 + vpsrlq $7, a0, a0 + vpxor a0, \_T1, \_T1 + vpsrlq $6, a2, a2 + vpxor a2, a1, a1 + vpaddq SZ4*((\i-16)&0xf)(%rsp), \_T1, \_T1 + vpaddq SZ4*((\i-7)&0xf)(%rsp), a1, a1 + vpaddq a1, \_T1, \_T1 + + ROUND_00_15 \_T1,\i +.endm + + +# void sha512_x4_avx2(void *STATE, const int INP_SIZE) +# arg 1 : STATE : pointer to input data +# arg 2 : INP_SIZE : size of data in blocks (assumed >= 1) +ENTRY(sha512_x4_avx2) + # general registers preserved in outer calling routine + # outer calling routine saves all the XMM registers + # save callee-saved clobbered registers to comply with C function ABI + push %r12 + push %r13 + push %r14 + push %r15 + + sub $STACK_SPACE1, %rsp + + # Load the pre-transposed incoming digest. + vmovdqu 0*SHA512_DIGEST_ROW_SIZE(STATE),a + vmovdqu 1*SHA512_DIGEST_ROW_SIZE(STATE),b + vmovdqu 2*SHA512_DIGEST_ROW_SIZE(STATE),c + vmovdqu 3*SHA512_DIGEST_ROW_SIZE(STATE),d + vmovdqu 4*SHA512_DIGEST_ROW_SIZE(STATE),e + vmovdqu 5*SHA512_DIGEST_ROW_SIZE(STATE),f + vmovdqu 6*SHA512_DIGEST_ROW_SIZE(STATE),g + vmovdqu 7*SHA512_DIGEST_ROW_SIZE(STATE),h + + lea K512_4(%rip),TBL + + # load the address of each of the 4 message lanes + # getting ready to transpose input onto stack + mov _data_ptr+0*PTR_SZ(STATE),inp0 + mov _data_ptr+1*PTR_SZ(STATE),inp1 + mov _data_ptr+2*PTR_SZ(STATE),inp2 + mov _data_ptr+3*PTR_SZ(STATE),inp3 + + xor IDX, IDX +lloop: + xor ROUND, ROUND + + # save old digest + vmovdqu a, _digest(%rsp) + vmovdqu b, _digest+1*SZ4(%rsp) + vmovdqu c, _digest+2*SZ4(%rsp) + vmovdqu d, _digest+3*SZ4(%rsp) + vmovdqu e, _digest+4*SZ4(%rsp) + vmovdqu f, _digest+5*SZ4(%rsp) + vmovdqu g, _digest+6*SZ4(%rsp) + vmovdqu h, _digest+7*SZ4(%rsp) + i = 0 +.rep 4 + vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP + VMOVPD i*32(inp0, IDX), TT2 + VMOVPD i*32(inp1, IDX), TT1 + VMOVPD i*32(inp2, IDX), TT4 + VMOVPD i*32(inp3, IDX), TT3 + TRANSPOSE TT2, TT1, TT4, TT3, TT0, TT5 + vpshufb TMP, TT0, TT0 + vpshufb TMP, TT1, TT1 + vpshufb TMP, TT2, TT2 + vpshufb TMP, TT3, TT3 + ROUND_00_15 TT0,(i*4+0) + ROUND_00_15 TT1,(i*4+1) + ROUND_00_15 TT2,(i*4+2) + ROUND_00_15 TT3,(i*4+3) + i = (i+1) +.endr + add $128, IDX + + i = (i*4) + + jmp Lrounds_16_xx +.align 16 +Lrounds_16_xx: +.rep 16 + ROUND_16_XX T1, i + i = (i+1) +.endr + cmp $0xa00,ROUND + jb Lrounds_16_xx + + # add old digest + vpaddq _digest(%rsp), a, a + vpaddq _digest+1*SZ4(%rsp), b, b + vpaddq _digest+2*SZ4(%rsp), c, c + vpaddq _digest+3*SZ4(%rsp), d, d + vpaddq _digest+4*SZ4(%rsp), e, e + vpaddq _digest+5*SZ4(%rsp), f, f + vpaddq _digest+6*SZ4(%rsp), g, g + vpaddq _digest+7*SZ4(%rsp), h, h + + sub $1, INP_SIZE # unit is blocks + jne lloop + + # write back to memory (state object) the transposed digest + vmovdqu a, 0*SHA512_DIGEST_ROW_SIZE(STATE) + vmovdqu b, 1*SHA512_DIGEST_ROW_SIZE(STATE) + vmovdqu c, 2*SHA512_DIGEST_ROW_SIZE(STATE) + vmovdqu d, 3*SHA512_DIGEST_ROW_SIZE(STATE) + vmovdqu e, 4*SHA512_DIGEST_ROW_SIZE(STATE) + vmovdqu f, 5*SHA512_DIGEST_ROW_SIZE(STATE) + vmovdqu g, 6*SHA512_DIGEST_ROW_SIZE(STATE) + vmovdqu h, 7*SHA512_DIGEST_ROW_SIZE(STATE) + + # update input data pointers + add IDX, inp0 + mov inp0, _data_ptr+0*PTR_SZ(STATE) + add IDX, inp1 + mov inp1, _data_ptr+1*PTR_SZ(STATE) + add IDX, inp2 + mov inp2, _data_ptr+2*PTR_SZ(STATE) + add IDX, inp3 + mov inp3, _data_ptr+3*PTR_SZ(STATE) + + #;;;;;;;;;;;;;;; + #; Postamble + add $STACK_SPACE1, %rsp + # restore callee-saved clobbered registers + + pop %r15 + pop %r14 + pop %r13 + pop %r12 + + # outer calling routine restores XMM and other GP registers + ret +ENDPROC(sha512_x4_avx2) + +.data +.align 64 +K512_4: + .octa 0x428a2f98d728ae22428a2f98d728ae22,\ + 0x428a2f98d728ae22428a2f98d728ae22 + .octa 0x7137449123ef65cd7137449123ef65cd,\ + 0x7137449123ef65cd7137449123ef65cd + .octa 0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f,\ + 0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f + .octa 0xe9b5dba58189dbbce9b5dba58189dbbc,\ + 0xe9b5dba58189dbbce9b5dba58189dbbc + .octa 0x3956c25bf348b5383956c25bf348b538,\ + 0x3956c25bf348b5383956c25bf348b538 + .octa 0x59f111f1b605d01959f111f1b605d019,\ + 0x59f111f1b605d01959f111f1b605d019 + .octa 0x923f82a4af194f9b923f82a4af194f9b,\ + 0x923f82a4af194f9b923f82a4af194f9b + .octa 0xab1c5ed5da6d8118ab1c5ed5da6d8118,\ + 0xab1c5ed5da6d8118ab1c5ed5da6d8118 + .octa 0xd807aa98a3030242d807aa98a3030242,\ + 0xd807aa98a3030242d807aa98a3030242 + .octa 0x12835b0145706fbe12835b0145706fbe,\ + 0x12835b0145706fbe12835b0145706fbe + .octa 0x243185be4ee4b28c243185be4ee4b28c,\ + 0x243185be4ee4b28c243185be4ee4b28c + .octa 0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2,\ + 0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2 + .octa 0x72be5d74f27b896f72be5d74f27b896f,\ + 0x72be5d74f27b896f72be5d74f27b896f + .octa 0x80deb1fe3b1696b180deb1fe3b1696b1,\ + 0x80deb1fe3b1696b180deb1fe3b1696b1 + .octa 0x9bdc06a725c712359bdc06a725c71235,\ + 0x9bdc06a725c712359bdc06a725c71235 + .octa 0xc19bf174cf692694c19bf174cf692694,\ + 0xc19bf174cf692694c19bf174cf692694 + .octa 0xe49b69c19ef14ad2e49b69c19ef14ad2,\ + 0xe49b69c19ef14ad2e49b69c19ef14ad2 + .octa 0xefbe4786384f25e3efbe4786384f25e3,\ + 0xefbe4786384f25e3efbe4786384f25e3 + .octa 0x0fc19dc68b8cd5b50fc19dc68b8cd5b5,\ + 0x0fc19dc68b8cd5b50fc19dc68b8cd5b5 + .octa 0x240ca1cc77ac9c65240ca1cc77ac9c65,\ + 0x240ca1cc77ac9c65240ca1cc77ac9c65 + .octa 0x2de92c6f592b02752de92c6f592b0275,\ + 0x2de92c6f592b02752de92c6f592b0275 + .octa 0x4a7484aa6ea6e4834a7484aa6ea6e483,\ + 0x4a7484aa6ea6e4834a7484aa6ea6e483 + .octa 0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4,\ + 0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4 + .octa 0x76f988da831153b576f988da831153b5,\ + 0x76f988da831153b576f988da831153b5 + .octa 0x983e5152ee66dfab983e5152ee66dfab,\ + 0x983e5152ee66dfab983e5152ee66dfab + .octa 0xa831c66d2db43210a831c66d2db43210,\ + 0xa831c66d2db43210a831c66d2db43210 + .octa 0xb00327c898fb213fb00327c898fb213f,\ + 0xb00327c898fb213fb00327c898fb213f + .octa 0xbf597fc7beef0ee4bf597fc7beef0ee4,\ + 0xbf597fc7beef0ee4bf597fc7beef0ee4 + .octa 0xc6e00bf33da88fc2c6e00bf33da88fc2,\ + 0xc6e00bf33da88fc2c6e00bf33da88fc2 + .octa 0xd5a79147930aa725d5a79147930aa725,\ + 0xd5a79147930aa725d5a79147930aa725 + .octa 0x06ca6351e003826f06ca6351e003826f,\ + 0x06ca6351e003826f06ca6351e003826f + .octa 0x142929670a0e6e70142929670a0e6e70,\ + 0x142929670a0e6e70142929670a0e6e70 + .octa 0x27b70a8546d22ffc27b70a8546d22ffc,\ + 0x27b70a8546d22ffc27b70a8546d22ffc + .octa 0x2e1b21385c26c9262e1b21385c26c926,\ + 0x2e1b21385c26c9262e1b21385c26c926 + .octa 0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed,\ + 0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed + .octa 0x53380d139d95b3df53380d139d95b3df,\ + 0x53380d139d95b3df53380d139d95b3df + .octa 0x650a73548baf63de650a73548baf63de,\ + 0x650a73548baf63de650a73548baf63de + .octa 0x766a0abb3c77b2a8766a0abb3c77b2a8,\ + 0x766a0abb3c77b2a8766a0abb3c77b2a8 + .octa 0x81c2c92e47edaee681c2c92e47edaee6,\ + 0x81c2c92e47edaee681c2c92e47edaee6 + .octa 0x92722c851482353b92722c851482353b,\ + 0x92722c851482353b92722c851482353b + .octa 0xa2bfe8a14cf10364a2bfe8a14cf10364,\ + 0xa2bfe8a14cf10364a2bfe8a14cf10364 + .octa 0xa81a664bbc423001a81a664bbc423001,\ + 0xa81a664bbc423001a81a664bbc423001 + .octa 0xc24b8b70d0f89791c24b8b70d0f89791,\ + 0xc24b8b70d0f89791c24b8b70d0f89791 + .octa 0xc76c51a30654be30c76c51a30654be30,\ + 0xc76c51a30654be30c76c51a30654be30 + .octa 0xd192e819d6ef5218d192e819d6ef5218,\ + 0xd192e819d6ef5218d192e819d6ef5218 + .octa 0xd69906245565a910d69906245565a910,\ + 0xd69906245565a910d69906245565a910 + .octa 0xf40e35855771202af40e35855771202a,\ + 0xf40e35855771202af40e35855771202a + .octa 0x106aa07032bbd1b8106aa07032bbd1b8,\ + 0x106aa07032bbd1b8106aa07032bbd1b8 + .octa 0x19a4c116b8d2d0c819a4c116b8d2d0c8,\ + 0x19a4c116b8d2d0c819a4c116b8d2d0c8 + .octa 0x1e376c085141ab531e376c085141ab53,\ + 0x1e376c085141ab531e376c085141ab53 + .octa 0x2748774cdf8eeb992748774cdf8eeb99,\ + 0x2748774cdf8eeb992748774cdf8eeb99 + .octa 0x34b0bcb5e19b48a834b0bcb5e19b48a8,\ + 0x34b0bcb5e19b48a834b0bcb5e19b48a8 + .octa 0x391c0cb3c5c95a63391c0cb3c5c95a63,\ + 0x391c0cb3c5c95a63391c0cb3c5c95a63 + .octa 0x4ed8aa4ae3418acb4ed8aa4ae3418acb,\ + 0x4ed8aa4ae3418acb4ed8aa4ae3418acb + .octa 0x5b9cca4f7763e3735b9cca4f7763e373,\ + 0x5b9cca4f7763e3735b9cca4f7763e373 + .octa 0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3,\ + 0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3 + .octa 0x748f82ee5defb2fc748f82ee5defb2fc,\ + 0x748f82ee5defb2fc748f82ee5defb2fc + .octa 0x78a5636f43172f6078a5636f43172f60,\ + 0x78a5636f43172f6078a5636f43172f60 + .octa 0x84c87814a1f0ab7284c87814a1f0ab72,\ + 0x84c87814a1f0ab7284c87814a1f0ab72 + .octa 0x8cc702081a6439ec8cc702081a6439ec,\ + 0x8cc702081a6439ec8cc702081a6439ec + .octa 0x90befffa23631e2890befffa23631e28,\ + 0x90befffa23631e2890befffa23631e28 + .octa 0xa4506cebde82bde9a4506cebde82bde9,\ + 0xa4506cebde82bde9a4506cebde82bde9 + .octa 0xbef9a3f7b2c67915bef9a3f7b2c67915,\ + 0xbef9a3f7b2c67915bef9a3f7b2c67915 + .octa 0xc67178f2e372532bc67178f2e372532b,\ + 0xc67178f2e372532bc67178f2e372532b + .octa 0xca273eceea26619cca273eceea26619c,\ + 0xca273eceea26619cca273eceea26619c + .octa 0xd186b8c721c0c207d186b8c721c0c207,\ + 0xd186b8c721c0c207d186b8c721c0c207 + .octa 0xeada7dd6cde0eb1eeada7dd6cde0eb1e,\ + 0xeada7dd6cde0eb1eeada7dd6cde0eb1e + .octa 0xf57d4f7fee6ed178f57d4f7fee6ed178,\ + 0xf57d4f7fee6ed178f57d4f7fee6ed178 + .octa 0x06f067aa72176fba06f067aa72176fba,\ + 0x06f067aa72176fba06f067aa72176fba + .octa 0x0a637dc5a2c898a60a637dc5a2c898a6,\ + 0x0a637dc5a2c898a60a637dc5a2c898a6 + .octa 0x113f9804bef90dae113f9804bef90dae,\ + 0x113f9804bef90dae113f9804bef90dae + .octa 0x1b710b35131c471b1b710b35131c471b,\ + 0x1b710b35131c471b1b710b35131c471b + .octa 0x28db77f523047d8428db77f523047d84,\ + 0x28db77f523047d8428db77f523047d84 + .octa 0x32caab7b40c7249332caab7b40c72493,\ + 0x32caab7b40c7249332caab7b40c72493 + .octa 0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc,\ + 0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc + .octa 0x431d67c49c100d4c431d67c49c100d4c,\ + 0x431d67c49c100d4c431d67c49c100d4c + .octa 0x4cc5d4becb3e42b64cc5d4becb3e42b6,\ + 0x4cc5d4becb3e42b64cc5d4becb3e42b6 + .octa 0x597f299cfc657e2a597f299cfc657e2a,\ + 0x597f299cfc657e2a597f299cfc657e2a + .octa 0x5fcb6fab3ad6faec5fcb6fab3ad6faec,\ + 0x5fcb6fab3ad6faec5fcb6fab3ad6faec + .octa 0x6c44198c4a4758176c44198c4a475817,\ + 0x6c44198c4a4758176c44198c4a475817 + +PSHUFFLE_BYTE_FLIP_MASK: .octa 0x08090a0b0c0d0e0f0001020304050607 + .octa 0x18191a1b1c1d1e1f1011121314151617 diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index 0b17c83d027d..2b0e2a6825f3 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -346,4 +346,10 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); MODULE_ALIAS_CRYPTO("sha512"); +MODULE_ALIAS_CRYPTO("sha512-ssse3"); +MODULE_ALIAS_CRYPTO("sha512-avx"); +MODULE_ALIAS_CRYPTO("sha512-avx2"); MODULE_ALIAS_CRYPTO("sha384"); +MODULE_ALIAS_CRYPTO("sha384-ssse3"); +MODULE_ALIAS_CRYPTO("sha384-avx"); +MODULE_ALIAS_CRYPTO("sha384-avx2"); diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index ec138e538c44..9e1e27d31c6d 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -40,10 +40,10 @@ static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs) #ifdef CONFIG_CONTEXT_TRACKING /* Called on entry from user mode with IRQs off. */ -__visible void enter_from_user_mode(void) +__visible inline void enter_from_user_mode(void) { CT_WARN_ON(ct_state() != CONTEXT_USER); - user_exit(); + user_exit_irqoff(); } #else static inline void enter_from_user_mode(void) {} @@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs) ti->status &= ~TS_COMPAT; #endif - user_enter(); + user_enter_irqoff(); } #define SYSCALL_EXIT_WORK_FLAGS \ diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index 983e5d3a0d27..0b56666e6039 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -1153,3 +1153,14 @@ ENTRY(async_page_fault) jmp error_code END(async_page_fault) #endif + +ENTRY(rewind_stack_do_exit) + /* Prevent any naive code from trying to unwind to our caller. */ + xorl %ebp, %ebp + + movl PER_CPU_VAR(cpu_current_top_of_stack), %esi + leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp + + call do_exit +1: jmp 1b +END(rewind_stack_do_exit) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 9ee0da1807ed..b846875aeea6 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1423,3 +1423,14 @@ ENTRY(ignore_sysret) mov $-ENOSYS, %eax sysret END(ignore_sysret) + +ENTRY(rewind_stack_do_exit) + /* Prevent any naive code from trying to unwind to our caller. */ + xorl %ebp, %ebp + + movq PER_CPU_VAR(cpu_current_top_of_stack), %rax + leaq -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%rax), %rsp + + call do_exit +1: jmp 1b +END(rewind_stack_do_exit) diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 555263e385c9..e9ce9c7c39b4 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -374,5 +374,5 @@ 543 x32 io_setup compat_sys_io_setup 544 x32 io_submit compat_sys_io_submit 545 x32 execveat compat_sys_execveat/ptregs -534 x32 preadv2 compat_sys_preadv2 -535 x32 pwritev2 compat_sys_pwritev2 +546 x32 preadv2 compat_sys_preadv64v2 +547 x32 pwritev2 compat_sys_pwritev64v2 diff --git a/arch/x86/entry/thunk_64.S b/arch/x86/entry/thunk_64.S index 027aec4a74df..627ecbcb2e62 100644 --- a/arch/x86/entry/thunk_64.S +++ b/arch/x86/entry/thunk_64.S @@ -33,7 +33,7 @@ .endif call \func - jmp restore + jmp .L_restore _ASM_NOKPROBE(\name) .endm @@ -54,7 +54,7 @@ #if defined(CONFIG_TRACE_IRQFLAGS) \ || defined(CONFIG_DEBUG_LOCK_ALLOC) \ || defined(CONFIG_PREEMPT) -restore: +.L_restore: popq %r11 popq %r10 popq %r9 @@ -66,5 +66,5 @@ restore: popq %rdi popq %rbp ret - _ASM_NOKPROBE(restore) + _ASM_NOKPROBE(.L_restore) #endif diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 253b72eaade6..6ba89a1ab0e5 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -55,7 +55,7 @@ VDSO_LDFLAGS_vdso.lds = -m64 -Wl,-soname=linux-vdso.so.1 \ $(obj)/vdso64.so.dbg: $(src)/vdso.lds $(vobjs) FORCE $(call if_changed,vdso) -HOST_EXTRACFLAGS += -I$(srctree)/tools/include -I$(srctree)/include/uapi -I$(srctree)/arch/x86/include/uapi +HOST_EXTRACFLAGS += -I$(srctree)/tools/include -I$(srctree)/include/uapi -I$(srctree)/arch/$(SUBARCH)/include/uapi hostprogs-y += vdso2c quiet_cmd_vdso2c = VDSO2C $@ @@ -134,7 +134,7 @@ VDSO_LDFLAGS_vdso32.lds = -m32 -Wl,-m,elf_i386 -Wl,-soname=linux-gate.so.1 override obj-dirs = $(dir $(obj)) $(obj)/vdso32/ targets += vdso32/vdso32.lds -targets += vdso32/note.o vdso32/vclock_gettime.o vdso32/system_call.o +targets += vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o targets += vdso32/vclock_gettime.o KBUILD_AFLAGS_32 := $(filter-out -m64,$(KBUILD_AFLAGS)) -DBUILD_VDSO @@ -156,7 +156,8 @@ $(obj)/vdso32.so.dbg: FORCE \ $(obj)/vdso32/vdso32.lds \ $(obj)/vdso32/vclock_gettime.o \ $(obj)/vdso32/note.o \ - $(obj)/vdso32/system_call.o + $(obj)/vdso32/system_call.o \ + $(obj)/vdso32/sigreturn.o $(call if_changed,vdso) # diff --git a/arch/x86/entry/vdso/vdso32/sigreturn.S b/arch/x86/entry/vdso/vdso32/sigreturn.S index d7ec4e251c0a..20633e026e82 100644 --- a/arch/x86/entry/vdso/vdso32/sigreturn.S +++ b/arch/x86/entry/vdso/vdso32/sigreturn.S @@ -1,11 +1,3 @@ -/* - * Common code for the sigreturn entry points in vDSO images. - * So far this code is the same for both int80 and sysenter versions. - * This file is #include'd by int80.S et al to define them first thing. - * The kernel assumes that the addresses of these routines are constant - * for all vDSO implementations. - */ - #include #include #include diff --git a/arch/x86/entry/vdso/vdso32/system_call.S b/arch/x86/entry/vdso/vdso32/system_call.S index 0109ac6cb79c..ed4bc9731cbb 100644 --- a/arch/x86/entry/vdso/vdso32/system_call.S +++ b/arch/x86/entry/vdso/vdso32/system_call.S @@ -2,16 +2,11 @@ * AT_SYSINFO entry point */ +#include #include #include #include -/* - * First get the common code for the sigreturn entry points. - * This must come first. - */ -#include "sigreturn.S" - .text .globl __kernel_vsyscall .type __kernel_vsyscall,@function diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index ab220ac9b3b9..3329844e3c43 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -97,10 +98,40 @@ static int vdso_fault(const struct vm_special_mapping *sm, return 0; } -static const struct vm_special_mapping text_mapping = { - .name = "[vdso]", - .fault = vdso_fault, -}; +static void vdso_fix_landing(const struct vdso_image *image, + struct vm_area_struct *new_vma) +{ +#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION + if (in_ia32_syscall() && image == &vdso_image_32) { + struct pt_regs *regs = current_pt_regs(); + unsigned long vdso_land = image->sym_int80_landing_pad; + unsigned long old_land_addr = vdso_land + + (unsigned long)current->mm->context.vdso; + + /* Fixing userspace landing - look at do_fast_syscall_32 */ + if (regs->ip == old_land_addr) + regs->ip = new_vma->vm_start + vdso_land; + } +#endif +} + +static int vdso_mremap(const struct vm_special_mapping *sm, + struct vm_area_struct *new_vma) +{ + unsigned long new_size = new_vma->vm_end - new_vma->vm_start; + const struct vdso_image *image = current->mm->context.vdso_image; + + if (image->size != new_size) + return -EINVAL; + + if (WARN_ON_ONCE(current->mm != new_vma->vm_mm)) + return -EFAULT; + + vdso_fix_landing(image, new_vma); + current->mm->context.vdso = (void __user *)new_vma->vm_start; + + return 0; +} static int vvar_fault(const struct vm_special_mapping *sm, struct vm_area_struct *vma, struct vm_fault *vmf) @@ -151,6 +182,12 @@ static int map_vdso(const struct vdso_image *image, bool calculate_addr) struct vm_area_struct *vma; unsigned long addr, text_start; int ret = 0; + + static const struct vm_special_mapping vdso_mapping = { + .name = "[vdso]", + .fault = vdso_fault, + .mremap = vdso_mremap, + }; static const struct vm_special_mapping vvar_mapping = { .name = "[vvar]", .fault = vvar_fault, @@ -185,7 +222,7 @@ static int map_vdso(const struct vdso_image *image, bool calculate_addr) image->size, VM_READ|VM_EXEC| VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - &text_mapping); + &vdso_mapping); if (IS_ERR(vma)) { ret = PTR_ERR(vma); diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index 174c2549939d..75fc719b7f31 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -96,7 +96,7 @@ static bool write_ok_or_segv(unsigned long ptr, size_t size) { /* * XXX: if access_ok, get_user, and put_user handled - * sig_on_uaccess_error, this could go away. + * sig_on_uaccess_err, this could go away. */ if (!access_ok(VERIFY_WRITE, (void __user *)ptr, size)) { @@ -125,7 +125,7 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address) struct task_struct *tsk; unsigned long caller; int vsyscall_nr, syscall_nr, tmp; - int prev_sig_on_uaccess_error; + int prev_sig_on_uaccess_err; long ret; /* @@ -221,8 +221,8 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address) * With a real vsyscall, page faults cause SIGSEGV. We want to * preserve that behavior to make writing exploits harder. */ - prev_sig_on_uaccess_error = current_thread_info()->sig_on_uaccess_error; - current_thread_info()->sig_on_uaccess_error = 1; + prev_sig_on_uaccess_err = current->thread.sig_on_uaccess_err; + current->thread.sig_on_uaccess_err = 1; ret = -EFAULT; switch (vsyscall_nr) { @@ -243,7 +243,7 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address) break; } - current_thread_info()->sig_on_uaccess_error = prev_sig_on_uaccess_error; + current->thread.sig_on_uaccess_err = prev_sig_on_uaccess_err; check_fault: if (ret == -EFAULT) { diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index feb90f6730e8..72dea2f40fc4 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -655,8 +655,12 @@ fail: } if (event->attr.sample_type & PERF_SAMPLE_RAW) { - raw.size = sizeof(u32) + ibs_data.size; - raw.data = ibs_data.data; + raw = (struct perf_raw_record){ + .frag = { + .size = sizeof(u32) + ibs_data.size, + .data = ibs_data.data, + }, + }; data.raw = &raw; } diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 33787ee817f0..dfebbde2a4cc 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -263,7 +263,7 @@ static bool check_hw_exists(void) msr_fail: pr_cont("Broken PMU hardware detected, using software events only.\n"); - pr_info("%sFailed to access perfctr msr (MSR %x is %Lx)\n", + printk("%sFailed to access perfctr msr (MSR %x is %Lx)\n", boot_cpu_has(X86_FEATURE_HYPERVISOR) ? KERN_INFO : KERN_ERR, reg, val_new); @@ -1622,6 +1622,29 @@ ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, cha } EXPORT_SYMBOL_GPL(events_sysfs_show); +ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, + char *page) +{ + struct perf_pmu_events_ht_attr *pmu_attr = + container_of(attr, struct perf_pmu_events_ht_attr, attr); + + /* + * Report conditional events depending on Hyper-Threading. + * + * This is overly conservative as usually the HT special + * handling is not needed if the other CPU thread is idle. + * + * Note this does not (and cannot) handle the case when thread + * siblings are invisible, for example with virtualization + * if they are owned by some other guest. The user tool + * has to re-read when a thread sibling gets onlined later. + */ + return sprintf(page, "%s", + topology_max_smt_threads() > 1 ? + pmu_attr->event_str_ht : + pmu_attr->event_str_noht); +} + EVENT_ATTR(cpu-cycles, CPU_CYCLES ); EVENT_ATTR(instructions, INSTRUCTIONS ); EVENT_ATTR(cache-references, CACHE_REFERENCES ); @@ -2319,7 +2342,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { struct stack_frame frame; - const void __user *fp; + const unsigned long __user *fp; if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { /* TODO: We don't support guest os callchain now */ @@ -2332,7 +2355,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) return; - fp = (void __user *)regs->bp; + fp = (unsigned long __user *)regs->bp; perf_callchain_store(entry, regs->ip); @@ -2345,16 +2368,17 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs pagefault_disable(); while (entry->nr < entry->max_stack) { unsigned long bytes; + frame.next_frame = NULL; frame.return_address = 0; - if (!access_ok(VERIFY_READ, fp, 16)) + if (!access_ok(VERIFY_READ, fp, sizeof(*fp) * 2)) break; - bytes = __copy_from_user_nmi(&frame.next_frame, fp, 8); + bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); if (bytes != 0) break; - bytes = __copy_from_user_nmi(&frame.return_address, fp+8, 8); + bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); if (bytes != 0) break; diff --git a/arch/x86/events/intel/Makefile b/arch/x86/events/intel/Makefile index 3660b2cf245a..06c2baa51814 100644 --- a/arch/x86/events/intel/Makefile +++ b/arch/x86/events/intel/Makefile @@ -1,8 +1,8 @@ obj-$(CONFIG_CPU_SUP_INTEL) += core.o bts.o cqm.o obj-$(CONFIG_CPU_SUP_INTEL) += ds.o knc.o obj-$(CONFIG_CPU_SUP_INTEL) += lbr.o p4.o p6.o pt.o -obj-$(CONFIG_PERF_EVENTS_INTEL_RAPL) += intel-rapl.o -intel-rapl-objs := rapl.o +obj-$(CONFIG_PERF_EVENTS_INTEL_RAPL) += intel-rapl-perf.o +intel-rapl-perf-objs := rapl.o obj-$(CONFIG_PERF_EVENTS_INTEL_UNCORE) += intel-uncore.o intel-uncore-objs := uncore.o uncore_nhmex.o uncore_snb.o uncore_snbep.o obj-$(CONFIG_PERF_EVENTS_INTEL_CSTATE) += intel-cstate.o diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 7c666958a625..0974ba11e954 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -16,6 +16,7 @@ #include #include +#include #include #include "../perf_event.h" @@ -115,6 +116,10 @@ static struct event_constraint intel_snb_event_constraints[] __read_mostly = INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */ INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ + /* + * When HT is off these events can only run on the bottom 4 counters + * When HT is on, they are impacted by the HT bug and require EXCL access + */ INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */ INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ @@ -139,6 +144,10 @@ static struct event_constraint intel_ivb_event_constraints[] __read_mostly = INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */ INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */ + /* + * When HT is off these events can only run on the bottom 4 counters + * When HT is on, they are impacted by the HT bug and require EXCL access + */ INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */ INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ @@ -177,19 +186,27 @@ static struct event_constraint intel_slm_event_constraints[] __read_mostly = EVENT_CONSTRAINT_END }; -struct event_constraint intel_skl_event_constraints[] = { +static struct event_constraint intel_skl_event_constraints[] = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ INTEL_UEVENT_CONSTRAINT(0x1c0, 0x2), /* INST_RETIRED.PREC_DIST */ + + /* + * when HT is off, these can only run on the bottom 4 counters + */ + INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_INST_RETIRED.* */ + INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_RETIRED.* */ + INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_L3_HIT_RETIRED.* */ + INTEL_EVENT_CONSTRAINT(0xcd, 0xf), /* MEM_TRANS_RETIRED.* */ + INTEL_EVENT_CONSTRAINT(0xc6, 0xf), /* FRONTEND_RETIRED.* */ + EVENT_CONSTRAINT_END }; static struct extra_reg intel_knl_extra_regs[] __read_mostly = { - INTEL_UEVENT_EXTRA_REG(0x01b7, - MSR_OFFCORE_RSP_0, 0x7f9ffbffffull, RSP_0), - INTEL_UEVENT_EXTRA_REG(0x02b7, - MSR_OFFCORE_RSP_1, 0x3f9ffbffffull, RSP_1), + INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x799ffbb6e7ull, RSP_0), + INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x399ffbffe7ull, RSP_1), EVENT_EXTRA_END }; @@ -225,14 +242,51 @@ EVENT_ATTR_STR(mem-loads, mem_ld_nhm, "event=0x0b,umask=0x10,ldlat=3"); EVENT_ATTR_STR(mem-loads, mem_ld_snb, "event=0xcd,umask=0x1,ldlat=3"); EVENT_ATTR_STR(mem-stores, mem_st_snb, "event=0xcd,umask=0x2"); -struct attribute *nhm_events_attrs[] = { +static struct attribute *nhm_events_attrs[] = { EVENT_PTR(mem_ld_nhm), NULL, }; -struct attribute *snb_events_attrs[] = { +/* + * topdown events for Intel Core CPUs. + * + * The events are all in slots, which is a free slot in a 4 wide + * pipeline. Some events are already reported in slots, for cycle + * events we multiply by the pipeline width (4). + * + * With Hyper Threading on, topdown metrics are either summed or averaged + * between the threads of a core: (count_t0 + count_t1). + * + * For the average case the metric is always scaled to pipeline width, + * so we use factor 2 ((count_t0 + count_t1) / 2 * 4) + */ + +EVENT_ATTR_STR_HT(topdown-total-slots, td_total_slots, + "event=0x3c,umask=0x0", /* cpu_clk_unhalted.thread */ + "event=0x3c,umask=0x0,any=1"); /* cpu_clk_unhalted.thread_any */ +EVENT_ATTR_STR_HT(topdown-total-slots.scale, td_total_slots_scale, "4", "2"); +EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued, + "event=0xe,umask=0x1"); /* uops_issued.any */ +EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired, + "event=0xc2,umask=0x2"); /* uops_retired.retire_slots */ +EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles, + "event=0x9c,umask=0x1"); /* idq_uops_not_delivered_core */ +EVENT_ATTR_STR_HT(topdown-recovery-bubbles, td_recovery_bubbles, + "event=0xd,umask=0x3,cmask=1", /* int_misc.recovery_cycles */ + "event=0xd,umask=0x3,cmask=1,any=1"); /* int_misc.recovery_cycles_any */ +EVENT_ATTR_STR_HT(topdown-recovery-bubbles.scale, td_recovery_bubbles_scale, + "4", "2"); + +static struct attribute *snb_events_attrs[] = { EVENT_PTR(mem_ld_snb), EVENT_PTR(mem_st_snb), + EVENT_PTR(td_slots_issued), + EVENT_PTR(td_slots_retired), + EVENT_PTR(td_fetch_bubbles), + EVENT_PTR(td_total_slots), + EVENT_PTR(td_total_slots_scale), + EVENT_PTR(td_recovery_bubbles), + EVENT_PTR(td_recovery_bubbles_scale), NULL, }; @@ -250,6 +304,10 @@ static struct event_constraint intel_hsw_event_constraints[] = { /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */ INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), + /* + * When HT is off these events can only run on the bottom 4 counters + * When HT is on, they are impacted by the HT bug and require EXCL access + */ INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */ INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ @@ -258,12 +316,19 @@ static struct event_constraint intel_hsw_event_constraints[] = { EVENT_CONSTRAINT_END }; -struct event_constraint intel_bdw_event_constraints[] = { +static struct event_constraint intel_bdw_event_constraints[] = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ INTEL_UEVENT_CONSTRAINT(0x148, 0x4), /* L1D_PEND_MISS.PENDING */ INTEL_UBIT_EVENT_CONSTRAINT(0x8a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_MISS */ + /* + * when HT is off, these can only run on the bottom 4 counters + */ + INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_INST_RETIRED.* */ + INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_RETIRED.* */ + INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_L3_HIT_RETIRED.* */ + INTEL_EVENT_CONSTRAINT(0xcd, 0xf), /* MEM_TRANS_RETIRED.* */ EVENT_CONSTRAINT_END }; @@ -1332,6 +1397,29 @@ static __initconst const u64 atom_hw_cache_event_ids }, }; +EVENT_ATTR_STR(topdown-total-slots, td_total_slots_slm, "event=0x3c"); +EVENT_ATTR_STR(topdown-total-slots.scale, td_total_slots_scale_slm, "2"); +/* no_alloc_cycles.not_delivered */ +EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles_slm, + "event=0xca,umask=0x50"); +EVENT_ATTR_STR(topdown-fetch-bubbles.scale, td_fetch_bubbles_scale_slm, "2"); +/* uops_retired.all */ +EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued_slm, + "event=0xc2,umask=0x10"); +/* uops_retired.all */ +EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired_slm, + "event=0xc2,umask=0x10"); + +static struct attribute *slm_events_attrs[] = { + EVENT_PTR(td_total_slots_slm), + EVENT_PTR(td_total_slots_scale_slm), + EVENT_PTR(td_fetch_bubbles_slm), + EVENT_PTR(td_fetch_bubbles_scale_slm), + EVENT_PTR(td_slots_issued_slm), + EVENT_PTR(td_slots_retired_slm), + NULL +}; + static struct extra_reg intel_slm_extra_regs[] __read_mostly = { /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ @@ -3261,11 +3349,11 @@ static int intel_snb_pebs_broken(int cpu) u32 rev = UINT_MAX; /* default to broken for unknown models */ switch (cpu_data(cpu).x86_model) { - case 42: /* SNB */ + case INTEL_FAM6_SANDYBRIDGE: rev = 0x28; break; - case 45: /* SNB-EP */ + case INTEL_FAM6_SANDYBRIDGE_X: switch (cpu_data(cpu).x86_mask) { case 6: rev = 0x618; break; case 7: rev = 0x70c; break; @@ -3302,6 +3390,13 @@ static void intel_snb_check_microcode(void) } } +static bool is_lbr_from(unsigned long msr) +{ + unsigned long lbr_from_nr = x86_pmu.lbr_from + x86_pmu.lbr_nr; + + return x86_pmu.lbr_from <= msr && msr < lbr_from_nr; +} + /* * Under certain circumstances, access certain MSR may cause #GP. * The function tests if the input MSR can be safely accessed. @@ -3322,13 +3417,24 @@ static bool check_msr(unsigned long msr, u64 mask) * Only change the bits which can be updated by wrmsrl. */ val_tmp = val_old ^ mask; + + if (is_lbr_from(msr)) + val_tmp = lbr_from_signext_quirk_wr(val_tmp); + if (wrmsrl_safe(msr, val_tmp) || rdmsrl_safe(msr, &val_new)) return false; + /* + * Quirk only affects validation in wrmsr(), so wrmsrl()'s value + * should equal rdmsrl()'s even with the quirk. + */ if (val_new != val_tmp) return false; + if (is_lbr_from(msr)) + val_old = lbr_from_signext_quirk_wr(val_old); + /* Here it's sure that the MSR can be safely accessed. * Restore the old value and return. */ @@ -3437,6 +3543,13 @@ static struct attribute *hsw_events_attrs[] = { EVENT_PTR(cycles_ct), EVENT_PTR(mem_ld_hsw), EVENT_PTR(mem_st_hsw), + EVENT_PTR(td_slots_issued), + EVENT_PTR(td_slots_retired), + EVENT_PTR(td_fetch_bubbles), + EVENT_PTR(td_total_slots), + EVENT_PTR(td_total_slots_scale), + EVENT_PTR(td_recovery_bubbles), + EVENT_PTR(td_recovery_bubbles_scale), NULL }; @@ -3508,15 +3621,15 @@ __init int intel_pmu_init(void) * Install the hw-cache-events table: */ switch (boot_cpu_data.x86_model) { - case 14: /* 65nm Core "Yonah" */ + case INTEL_FAM6_CORE_YONAH: pr_cont("Core events, "); break; - case 15: /* 65nm Core2 "Merom" */ + case INTEL_FAM6_CORE2_MEROM: x86_add_quirk(intel_clovertown_quirk); - case 22: /* 65nm Core2 "Merom-L" */ - case 23: /* 45nm Core2 "Penryn" */ - case 29: /* 45nm Core2 "Dunnington (MP) */ + case INTEL_FAM6_CORE2_MEROM_L: + case INTEL_FAM6_CORE2_PENRYN: + case INTEL_FAM6_CORE2_DUNNINGTON: memcpy(hw_cache_event_ids, core2_hw_cache_event_ids, sizeof(hw_cache_event_ids)); @@ -3527,9 +3640,9 @@ __init int intel_pmu_init(void) pr_cont("Core2 events, "); break; - case 30: /* 45nm Nehalem */ - case 26: /* 45nm Nehalem-EP */ - case 46: /* 45nm Nehalem-EX */ + case INTEL_FAM6_NEHALEM: + case INTEL_FAM6_NEHALEM_EP: + case INTEL_FAM6_NEHALEM_EX: memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids, sizeof(hw_cache_event_ids)); memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs, @@ -3557,11 +3670,11 @@ __init int intel_pmu_init(void) pr_cont("Nehalem events, "); break; - case 28: /* 45nm Atom "Pineview" */ - case 38: /* 45nm Atom "Lincroft" */ - case 39: /* 32nm Atom "Penwell" */ - case 53: /* 32nm Atom "Cloverview" */ - case 54: /* 32nm Atom "Cedarview" */ + case INTEL_FAM6_ATOM_PINEVIEW: + case INTEL_FAM6_ATOM_LINCROFT: + case INTEL_FAM6_ATOM_PENWELL: + case INTEL_FAM6_ATOM_CLOVERVIEW: + case INTEL_FAM6_ATOM_CEDARVIEW: memcpy(hw_cache_event_ids, atom_hw_cache_event_ids, sizeof(hw_cache_event_ids)); @@ -3573,9 +3686,9 @@ __init int intel_pmu_init(void) pr_cont("Atom events, "); break; - case 55: /* 22nm Atom "Silvermont" */ - case 76: /* 14nm Atom "Airmont" */ - case 77: /* 22nm Atom "Silvermont Avoton/Rangely" */ + case INTEL_FAM6_ATOM_SILVERMONT1: + case INTEL_FAM6_ATOM_SILVERMONT2: + case INTEL_FAM6_ATOM_AIRMONT: memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, sizeof(hw_cache_event_ids)); memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs, @@ -3587,11 +3700,12 @@ __init int intel_pmu_init(void) x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints; x86_pmu.extra_regs = intel_slm_extra_regs; x86_pmu.flags |= PMU_FL_HAS_RSP_1; + x86_pmu.cpu_events = slm_events_attrs; pr_cont("Silvermont events, "); break; - case 92: /* 14nm Atom "Goldmont" */ - case 95: /* 14nm Atom "Goldmont Denverton" */ + case INTEL_FAM6_ATOM_GOLDMONT: + case INTEL_FAM6_ATOM_DENVERTON: memcpy(hw_cache_event_ids, glm_hw_cache_event_ids, sizeof(hw_cache_event_ids)); memcpy(hw_cache_extra_regs, glm_hw_cache_extra_regs, @@ -3614,9 +3728,9 @@ __init int intel_pmu_init(void) pr_cont("Goldmont events, "); break; - case 37: /* 32nm Westmere */ - case 44: /* 32nm Westmere-EP */ - case 47: /* 32nm Westmere-EX */ + case INTEL_FAM6_WESTMERE: + case INTEL_FAM6_WESTMERE_EP: + case INTEL_FAM6_WESTMERE_EX: memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids, sizeof(hw_cache_event_ids)); memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs, @@ -3643,8 +3757,8 @@ __init int intel_pmu_init(void) pr_cont("Westmere events, "); break; - case 42: /* 32nm SandyBridge */ - case 45: /* 32nm SandyBridge-E/EN/EP */ + case INTEL_FAM6_SANDYBRIDGE: + case INTEL_FAM6_SANDYBRIDGE_X: x86_add_quirk(intel_sandybridge_quirk); x86_add_quirk(intel_ht_bug); memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, @@ -3657,7 +3771,7 @@ __init int intel_pmu_init(void) x86_pmu.event_constraints = intel_snb_event_constraints; x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints; x86_pmu.pebs_aliases = intel_pebs_aliases_snb; - if (boot_cpu_data.x86_model == 45) + if (boot_cpu_data.x86_model == INTEL_FAM6_SANDYBRIDGE_X) x86_pmu.extra_regs = intel_snbep_extra_regs; else x86_pmu.extra_regs = intel_snb_extra_regs; @@ -3679,8 +3793,8 @@ __init int intel_pmu_init(void) pr_cont("SandyBridge events, "); break; - case 58: /* 22nm IvyBridge */ - case 62: /* 22nm IvyBridge-EP/EX */ + case INTEL_FAM6_IVYBRIDGE: + case INTEL_FAM6_IVYBRIDGE_X: x86_add_quirk(intel_ht_bug); memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, sizeof(hw_cache_event_ids)); @@ -3696,7 +3810,7 @@ __init int intel_pmu_init(void) x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints; x86_pmu.pebs_aliases = intel_pebs_aliases_ivb; x86_pmu.pebs_prec_dist = true; - if (boot_cpu_data.x86_model == 62) + if (boot_cpu_data.x86_model == INTEL_FAM6_IVYBRIDGE_X) x86_pmu.extra_regs = intel_snbep_extra_regs; else x86_pmu.extra_regs = intel_snb_extra_regs; @@ -3714,10 +3828,10 @@ __init int intel_pmu_init(void) break; - case 60: /* 22nm Haswell Core */ - case 63: /* 22nm Haswell Server */ - case 69: /* 22nm Haswell ULT */ - case 70: /* 22nm Haswell + GT3e (Intel Iris Pro graphics) */ + case INTEL_FAM6_HASWELL_CORE: + case INTEL_FAM6_HASWELL_X: + case INTEL_FAM6_HASWELL_ULT: + case INTEL_FAM6_HASWELL_GT3E: x86_add_quirk(intel_ht_bug); x86_pmu.late_ack = true; memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids)); @@ -3741,10 +3855,10 @@ __init int intel_pmu_init(void) pr_cont("Haswell events, "); break; - case 61: /* 14nm Broadwell Core-M */ - case 86: /* 14nm Broadwell Xeon D */ - case 71: /* 14nm Broadwell + GT3e (Intel Iris Pro graphics) */ - case 79: /* 14nm Broadwell Server */ + case INTEL_FAM6_BROADWELL_CORE: + case INTEL_FAM6_BROADWELL_XEON_D: + case INTEL_FAM6_BROADWELL_GT3E: + case INTEL_FAM6_BROADWELL_X: x86_pmu.late_ack = true; memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids)); memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); @@ -3777,7 +3891,7 @@ __init int intel_pmu_init(void) pr_cont("Broadwell events, "); break; - case 87: /* Knights Landing Xeon Phi */ + case INTEL_FAM6_XEON_PHI_KNL: memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, sizeof(hw_cache_event_ids)); memcpy(hw_cache_extra_regs, @@ -3795,16 +3909,22 @@ __init int intel_pmu_init(void) pr_cont("Knights Landing events, "); break; - case 142: /* 14nm Kabylake Mobile */ - case 158: /* 14nm Kabylake Desktop */ - case 78: /* 14nm Skylake Mobile */ - case 94: /* 14nm Skylake Desktop */ - case 85: /* 14nm Skylake Server */ + case INTEL_FAM6_SKYLAKE_MOBILE: + case INTEL_FAM6_SKYLAKE_DESKTOP: + case INTEL_FAM6_SKYLAKE_X: + case INTEL_FAM6_KABYLAKE_MOBILE: + case INTEL_FAM6_KABYLAKE_DESKTOP: x86_pmu.late_ack = true; memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids)); memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); intel_pmu_lbr_init_skl(); + /* INT_MISC.RECOVERY_CYCLES has umask 1 in Skylake */ + event_attr_td_recovery_bubbles.event_str_noht = + "event=0xd,umask=0x1,cmask=1"; + event_attr_td_recovery_bubbles.event_str_ht = + "event=0xd,umask=0x1,cmask=1,any=1"; + x86_pmu.event_constraints = intel_skl_event_constraints; x86_pmu.pebs_constraints = intel_skl_pebs_event_constraints; x86_pmu.extra_regs = intel_skl_extra_regs; @@ -3885,6 +4005,8 @@ __init int intel_pmu_init(void) x86_pmu.lbr_nr = 0; } + if (x86_pmu.lbr_nr) + pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr); /* * Access extra MSR may cause #GP under certain circumstances. * E.g. KVM doesn't support offcore event @@ -3917,16 +4039,14 @@ __init int intel_pmu_init(void) */ static __init int fixup_ht_bug(void) { - int cpu = smp_processor_id(); - int w, c; + int c; /* * problem not present on this CPU model, nothing to do */ if (!(x86_pmu.flags & PMU_FL_EXCL_ENABLED)) return 0; - w = cpumask_weight(topology_sibling_cpumask(cpu)); - if (w > 1) { + if (topology_max_smt_threads() > 1) { pr_info("PMU erratum BJ122, BV98, HSD29 worked around, HT is on\n"); return 0; } diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 9ba4e4136a15..4c7638b91fa5 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -89,6 +89,7 @@ #include #include #include +#include #include "../perf_event.h" MODULE_LICENSE("GPL"); @@ -511,37 +512,37 @@ static const struct cstate_model slm_cstates __initconst = { { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long) &(states) } static const struct x86_cpu_id intel_cstates_match[] __initconst = { - X86_CSTATES_MODEL(30, nhm_cstates), /* 45nm Nehalem */ - X86_CSTATES_MODEL(26, nhm_cstates), /* 45nm Nehalem-EP */ - X86_CSTATES_MODEL(46, nhm_cstates), /* 45nm Nehalem-EX */ + X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM, nhm_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM_EP, nhm_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM_EX, nhm_cstates), - X86_CSTATES_MODEL(37, nhm_cstates), /* 32nm Westmere */ - X86_CSTATES_MODEL(44, nhm_cstates), /* 32nm Westmere-EP */ - X86_CSTATES_MODEL(47, nhm_cstates), /* 32nm Westmere-EX */ + X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE, nhm_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE_EP, nhm_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE_EX, nhm_cstates), - X86_CSTATES_MODEL(42, snb_cstates), /* 32nm SandyBridge */ - X86_CSTATES_MODEL(45, snb_cstates), /* 32nm SandyBridge-E/EN/EP */ + X86_CSTATES_MODEL(INTEL_FAM6_SANDYBRIDGE, snb_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_SANDYBRIDGE_X, snb_cstates), - X86_CSTATES_MODEL(58, snb_cstates), /* 22nm IvyBridge */ - X86_CSTATES_MODEL(62, snb_cstates), /* 22nm IvyBridge-EP/EX */ + X86_CSTATES_MODEL(INTEL_FAM6_IVYBRIDGE, snb_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_IVYBRIDGE_X, snb_cstates), - X86_CSTATES_MODEL(60, snb_cstates), /* 22nm Haswell Core */ - X86_CSTATES_MODEL(63, snb_cstates), /* 22nm Haswell Server */ - X86_CSTATES_MODEL(70, snb_cstates), /* 22nm Haswell + GT3e */ + X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_CORE, snb_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_X, snb_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_GT3E, snb_cstates), - X86_CSTATES_MODEL(69, hswult_cstates), /* 22nm Haswell ULT */ + X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_ULT, hswult_cstates), - X86_CSTATES_MODEL(55, slm_cstates), /* 22nm Atom Silvermont */ - X86_CSTATES_MODEL(77, slm_cstates), /* 22nm Atom Avoton/Rangely */ - X86_CSTATES_MODEL(76, slm_cstates), /* 22nm Atom Airmont */ + X86_CSTATES_MODEL(INTEL_FAM6_ATOM_SILVERMONT1, slm_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_ATOM_SILVERMONT2, slm_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_ATOM_AIRMONT, slm_cstates), - X86_CSTATES_MODEL(61, snb_cstates), /* 14nm Broadwell Core-M */ - X86_CSTATES_MODEL(86, snb_cstates), /* 14nm Broadwell Xeon D */ - X86_CSTATES_MODEL(71, snb_cstates), /* 14nm Broadwell + GT3e */ - X86_CSTATES_MODEL(79, snb_cstates), /* 14nm Broadwell Server */ + X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_CORE, snb_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_XEON_D, snb_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_GT3E, snb_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_X, snb_cstates), - X86_CSTATES_MODEL(78, snb_cstates), /* 14nm Skylake Mobile */ - X86_CSTATES_MODEL(94, snb_cstates), /* 14nm Skylake Desktop */ + X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE_MOBILE, snb_cstates), + X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE_DESKTOP, snb_cstates), { }, }; MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 9e2b40cdb05f..707d358e0dff 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -77,9 +77,11 @@ static enum { LBR_IND_JMP |\ LBR_FAR) -#define LBR_FROM_FLAG_MISPRED (1ULL << 63) -#define LBR_FROM_FLAG_IN_TX (1ULL << 62) -#define LBR_FROM_FLAG_ABORT (1ULL << 61) +#define LBR_FROM_FLAG_MISPRED BIT_ULL(63) +#define LBR_FROM_FLAG_IN_TX BIT_ULL(62) +#define LBR_FROM_FLAG_ABORT BIT_ULL(61) + +#define LBR_FROM_SIGNEXT_2MSB (BIT_ULL(60) | BIT_ULL(59)) /* * x86control flow change classification @@ -235,6 +237,97 @@ enum { LBR_VALID, }; +/* + * For formats with LBR_TSX flags (e.g. LBR_FORMAT_EIP_FLAGS2), bits 61:62 in + * MSR_LAST_BRANCH_FROM_x are the TSX flags when TSX is supported, but when + * TSX is not supported they have no consistent behavior: + * + * - For wrmsr(), bits 61:62 are considered part of the sign extension. + * - For HW updates (branch captures) bits 61:62 are always OFF and are not + * part of the sign extension. + * + * Therefore, if: + * + * 1) LBR has TSX format + * 2) CPU has no TSX support enabled + * + * ... then any value passed to wrmsr() must be sign extended to 63 bits and any + * value from rdmsr() must be converted to have a 61 bits sign extension, + * ignoring the TSX flags. + */ +static inline bool lbr_from_signext_quirk_needed(void) +{ + int lbr_format = x86_pmu.intel_cap.lbr_format; + bool tsx_support = boot_cpu_has(X86_FEATURE_HLE) || + boot_cpu_has(X86_FEATURE_RTM); + + return !tsx_support && (lbr_desc[lbr_format] & LBR_TSX); +} + +DEFINE_STATIC_KEY_FALSE(lbr_from_quirk_key); + +/* If quirk is enabled, ensure sign extension is 63 bits: */ +inline u64 lbr_from_signext_quirk_wr(u64 val) +{ + if (static_branch_unlikely(&lbr_from_quirk_key)) { + /* + * Sign extend into bits 61:62 while preserving bit 63. + * + * Quirk is enabled when TSX is disabled. Therefore TSX bits + * in val are always OFF and must be changed to be sign + * extension bits. Since bits 59:60 are guaranteed to be + * part of the sign extension bits, we can just copy them + * to 61:62. + */ + val |= (LBR_FROM_SIGNEXT_2MSB & val) << 2; + } + return val; +} + +/* + * If quirk is needed, ensure sign extension is 61 bits: + */ +u64 lbr_from_signext_quirk_rd(u64 val) +{ + if (static_branch_unlikely(&lbr_from_quirk_key)) { + /* + * Quirk is on when TSX is not enabled. Therefore TSX + * flags must be read as OFF. + */ + val &= ~(LBR_FROM_FLAG_IN_TX | LBR_FROM_FLAG_ABORT); + } + return val; +} + +static inline void wrlbr_from(unsigned int idx, u64 val) +{ + val = lbr_from_signext_quirk_wr(val); + wrmsrl(x86_pmu.lbr_from + idx, val); +} + +static inline void wrlbr_to(unsigned int idx, u64 val) +{ + wrmsrl(x86_pmu.lbr_to + idx, val); +} + +static inline u64 rdlbr_from(unsigned int idx) +{ + u64 val; + + rdmsrl(x86_pmu.lbr_from + idx, val); + + return lbr_from_signext_quirk_rd(val); +} + +static inline u64 rdlbr_to(unsigned int idx) +{ + u64 val; + + rdmsrl(x86_pmu.lbr_to + idx, val); + + return val; +} + static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) { int i; @@ -251,8 +344,9 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) tos = task_ctx->tos; for (i = 0; i < tos; i++) { lbr_idx = (tos - i) & mask; - wrmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]); - wrmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]); + wrlbr_from(lbr_idx, task_ctx->lbr_from[i]); + wrlbr_to (lbr_idx, task_ctx->lbr_to[i]); + if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) wrmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); } @@ -262,9 +356,9 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx) { - int i; unsigned lbr_idx, mask; u64 tos; + int i; if (task_ctx->lbr_callstack_users == 0) { task_ctx->lbr_stack_state = LBR_NONE; @@ -275,8 +369,8 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx) tos = intel_pmu_lbr_tos(); for (i = 0; i < tos; i++) { lbr_idx = (tos - i) & mask; - rdmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]); - rdmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]); + task_ctx->lbr_from[i] = rdlbr_from(lbr_idx); + task_ctx->lbr_to[i] = rdlbr_to(lbr_idx); if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) rdmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); } @@ -452,8 +546,8 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) u16 cycles = 0; int lbr_flags = lbr_desc[lbr_format]; - rdmsrl(x86_pmu.lbr_from + lbr_idx, from); - rdmsrl(x86_pmu.lbr_to + lbr_idx, to); + from = rdlbr_from(lbr_idx); + to = rdlbr_to(lbr_idx); if (lbr_format == LBR_FORMAT_INFO && need_info) { u64 info; @@ -956,7 +1050,6 @@ void __init intel_pmu_lbr_init_core(void) * SW branch filter usage: * - compensate for lack of HW filter */ - pr_cont("4-deep LBR, "); } /* nehalem/westmere */ @@ -977,7 +1070,6 @@ void __init intel_pmu_lbr_init_nhm(void) * That requires LBR_FAR but that means far * jmp need to be filtered out */ - pr_cont("16-deep LBR, "); } /* sandy bridge */ @@ -997,7 +1089,6 @@ void __init intel_pmu_lbr_init_snb(void) * That requires LBR_FAR but that means far * jmp need to be filtered out */ - pr_cont("16-deep LBR, "); } /* haswell */ @@ -1011,7 +1102,8 @@ void intel_pmu_lbr_init_hsw(void) x86_pmu.lbr_sel_mask = LBR_SEL_MASK; x86_pmu.lbr_sel_map = hsw_lbr_sel_map; - pr_cont("16-deep LBR, "); + if (lbr_from_signext_quirk_needed()) + static_branch_enable(&lbr_from_quirk_key); } /* skylake */ @@ -1031,7 +1123,6 @@ __init void intel_pmu_lbr_init_skl(void) * That requires LBR_FAR but that means far * jmp need to be filtered out */ - pr_cont("32-deep LBR, "); } /* atom */ @@ -1057,7 +1148,6 @@ void __init intel_pmu_lbr_init_atom(void) * SW branch filter usage: * - compensate for lack of HW filter */ - pr_cont("8-deep LBR, "); } /* slm */ @@ -1088,6 +1178,4 @@ void intel_pmu_lbr_init_knl(void) x86_pmu.lbr_sel_mask = LBR_SEL_MASK; x86_pmu.lbr_sel_map = snb_lbr_sel_map; - - pr_cont("8-deep LBR, "); } diff --git a/arch/x86/events/intel/rapl.c b/arch/x86/events/intel/rapl.c index e30eef4f29a6..d0c58b35155f 100644 --- a/arch/x86/events/intel/rapl.c +++ b/arch/x86/events/intel/rapl.c @@ -55,6 +55,7 @@ #include #include #include +#include #include "../perf_event.h" MODULE_LICENSE("GPL"); @@ -786,26 +787,27 @@ static const struct intel_rapl_init_fun skl_rapl_init __initconst = { }; static const struct x86_cpu_id rapl_cpu_match[] __initconst = { - X86_RAPL_MODEL_MATCH(42, snb_rapl_init), /* Sandy Bridge */ - X86_RAPL_MODEL_MATCH(45, snbep_rapl_init), /* Sandy Bridge-EP */ + X86_RAPL_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE, snb_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE_X, snbep_rapl_init), - X86_RAPL_MODEL_MATCH(58, snb_rapl_init), /* Ivy Bridge */ - X86_RAPL_MODEL_MATCH(62, snbep_rapl_init), /* IvyTown */ + X86_RAPL_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE, snb_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE_X, snbep_rapl_init), - X86_RAPL_MODEL_MATCH(60, hsw_rapl_init), /* Haswell */ - X86_RAPL_MODEL_MATCH(63, hsx_rapl_init), /* Haswell-Server */ - X86_RAPL_MODEL_MATCH(69, hsw_rapl_init), /* Haswell-Celeron */ - X86_RAPL_MODEL_MATCH(70, hsw_rapl_init), /* Haswell GT3e */ + X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_CORE, hsw_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_X, hsw_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_ULT, hsw_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_GT3E, hsw_rapl_init), - X86_RAPL_MODEL_MATCH(61, hsw_rapl_init), /* Broadwell */ - X86_RAPL_MODEL_MATCH(71, hsw_rapl_init), /* Broadwell-H */ - X86_RAPL_MODEL_MATCH(79, hsx_rapl_init), /* Broadwell-Server */ - X86_RAPL_MODEL_MATCH(86, hsx_rapl_init), /* Broadwell Xeon D */ + X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_CORE, hsw_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_GT3E, hsw_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_X, hsw_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_XEON_D, hsw_rapl_init), - X86_RAPL_MODEL_MATCH(87, knl_rapl_init), /* Knights Landing */ + X86_RAPL_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNL, knl_rapl_init), - X86_RAPL_MODEL_MATCH(78, skl_rapl_init), /* Skylake */ - X86_RAPL_MODEL_MATCH(94, skl_rapl_init), /* Skylake H/S */ + X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE_MOBILE, skl_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE_DESKTOP, skl_rapl_init), + X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE_X, hsx_rapl_init), {}, }; diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index fce74062d981..59b4974c697f 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -1,4 +1,5 @@ #include +#include #include "uncore.h" static struct intel_uncore_type *empty_uncore[] = { NULL, }; @@ -882,7 +883,7 @@ uncore_types_init(struct intel_uncore_type **types, bool setid) static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct intel_uncore_type *type; - struct intel_uncore_pmu *pmu; + struct intel_uncore_pmu *pmu = NULL; struct intel_uncore_box *box; int phys_id, pkg, ret; @@ -903,20 +904,37 @@ static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id } type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)]; + /* - * for performance monitoring unit with multiple boxes, - * each box has a different function id. - */ - pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)]; - /* Knights Landing uses a common PCI device ID for multiple instances of - * an uncore PMU device type. There is only one entry per device type in - * the knl_uncore_pci_ids table inspite of multiple devices present for - * some device types. Hence PCI device idx would be 0 for all devices. - * So increment pmu pointer to point to an unused array element. + * Some platforms, e.g. Knights Landing, use a common PCI device ID + * for multiple instances of an uncore PMU device type. We should check + * PCI slot and func to indicate the uncore box. */ - if (boot_cpu_data.x86_model == 87) { - while (pmu->func_id >= 0) - pmu++; + if (id->driver_data & ~0xffff) { + struct pci_driver *pci_drv = pdev->driver; + const struct pci_device_id *ids = pci_drv->id_table; + unsigned int devfn; + + while (ids && ids->vendor) { + if ((ids->vendor == pdev->vendor) && + (ids->device == pdev->device)) { + devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data), + UNCORE_PCI_DEV_FUNC(ids->driver_data)); + if (devfn == pdev->devfn) { + pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)]; + break; + } + } + ids++; + } + if (pmu == NULL) + return -ENODEV; + } else { + /* + * for performance monitoring unit with multiple boxes, + * each box has a different function id. + */ + pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)]; } if (WARN_ON_ONCE(pmu->boxes[pkg] != NULL)) @@ -956,7 +974,7 @@ static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id static void uncore_pci_remove(struct pci_dev *pdev) { - struct intel_uncore_box *box = pci_get_drvdata(pdev); + struct intel_uncore_box *box; struct intel_uncore_pmu *pmu; int i, phys_id, pkg; @@ -1361,30 +1379,32 @@ static const struct intel_uncore_init_fun knl_uncore_init __initconst = { }; static const struct intel_uncore_init_fun skl_uncore_init __initconst = { + .cpu_init = skl_uncore_cpu_init, .pci_init = skl_uncore_pci_init, }; static const struct x86_cpu_id intel_uncore_match[] __initconst = { - X86_UNCORE_MODEL_MATCH(26, nhm_uncore_init), /* Nehalem */ - X86_UNCORE_MODEL_MATCH(30, nhm_uncore_init), - X86_UNCORE_MODEL_MATCH(37, nhm_uncore_init), /* Westmere */ - X86_UNCORE_MODEL_MATCH(44, nhm_uncore_init), - X86_UNCORE_MODEL_MATCH(42, snb_uncore_init), /* Sandy Bridge */ - X86_UNCORE_MODEL_MATCH(58, ivb_uncore_init), /* Ivy Bridge */ - X86_UNCORE_MODEL_MATCH(60, hsw_uncore_init), /* Haswell */ - X86_UNCORE_MODEL_MATCH(69, hsw_uncore_init), /* Haswell Celeron */ - X86_UNCORE_MODEL_MATCH(70, hsw_uncore_init), /* Haswell */ - X86_UNCORE_MODEL_MATCH(61, bdw_uncore_init), /* Broadwell */ - X86_UNCORE_MODEL_MATCH(71, bdw_uncore_init), /* Broadwell */ - X86_UNCORE_MODEL_MATCH(45, snbep_uncore_init), /* Sandy Bridge-EP */ - X86_UNCORE_MODEL_MATCH(46, nhmex_uncore_init), /* Nehalem-EX */ - X86_UNCORE_MODEL_MATCH(47, nhmex_uncore_init), /* Westmere-EX aka. Xeon E7 */ - X86_UNCORE_MODEL_MATCH(62, ivbep_uncore_init), /* Ivy Bridge-EP */ - X86_UNCORE_MODEL_MATCH(63, hswep_uncore_init), /* Haswell-EP */ - X86_UNCORE_MODEL_MATCH(79, bdx_uncore_init), /* BDX-EP */ - X86_UNCORE_MODEL_MATCH(86, bdx_uncore_init), /* BDX-DE */ - X86_UNCORE_MODEL_MATCH(87, knl_uncore_init), /* Knights Landing */ - X86_UNCORE_MODEL_MATCH(94, skl_uncore_init), /* SkyLake */ + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EP, nhm_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM, nhm_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE, nhm_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EP, nhm_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE, snb_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE, ivb_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_CORE, hsw_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_ULT, hsw_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_GT3E, hsw_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_CORE, bdw_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_GT3E, bdw_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE_X, snbep_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EX, nhmex_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EX, nhmex_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE_X, ivbep_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_X, hswep_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_X, bdx_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_XEON_D, bdx_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNL, knl_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_DESKTOP,skl_uncore_init), + X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_MOBILE, skl_uncore_init), {}, }; diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index 79766b9a3580..d6063e438158 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -15,7 +15,11 @@ #define UNCORE_PMC_IDX_FIXED UNCORE_PMC_IDX_MAX_GENERIC #define UNCORE_PMC_IDX_MAX (UNCORE_PMC_IDX_FIXED + 1) +#define UNCORE_PCI_DEV_FULL_DATA(dev, func, type, idx) \ + ((dev << 24) | (func << 16) | (type << 8) | idx) #define UNCORE_PCI_DEV_DATA(type, idx) ((type << 8) | idx) +#define UNCORE_PCI_DEV_DEV(data) ((data >> 24) & 0xff) +#define UNCORE_PCI_DEV_FUNC(data) ((data >> 16) & 0xff) #define UNCORE_PCI_DEV_TYPE(data) ((data >> 8) & 0xff) #define UNCORE_PCI_DEV_IDX(data) (data & 0xff) #define UNCORE_EXTRA_PCI_DEV 0xff @@ -360,6 +364,7 @@ int bdw_uncore_pci_init(void); int skl_uncore_pci_init(void); void snb_uncore_cpu_init(void); void nhm_uncore_cpu_init(void); +void skl_uncore_cpu_init(void); int snb_pci2phy_map_init(int devid); /* perf_event_intel_uncore_snbep.c */ diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index 96531d2b843f..97a69dbba649 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -1,4 +1,4 @@ -/* Nehalem/SandBridge/Haswell uncore support */ +/* Nehalem/SandBridge/Haswell/Broadwell/Skylake uncore support */ #include "uncore.h" /* Uncore IMC PCI IDs */ @@ -9,6 +9,7 @@ #define PCI_DEVICE_ID_INTEL_HSW_U_IMC 0x0a04 #define PCI_DEVICE_ID_INTEL_BDW_IMC 0x1604 #define PCI_DEVICE_ID_INTEL_SKL_IMC 0x191f +#define PCI_DEVICE_ID_INTEL_SKL_U_IMC 0x190c /* SNB event control */ #define SNB_UNC_CTL_EV_SEL_MASK 0x000000ff @@ -64,6 +65,10 @@ #define NHM_UNC_PERFEVTSEL0 0x3c0 #define NHM_UNC_UNCORE_PMC0 0x3b0 +/* SKL uncore global control */ +#define SKL_UNC_PERF_GLOBAL_CTL 0xe01 +#define SKL_UNC_GLOBAL_CTL_CORE_ALL ((1 << 5) - 1) + DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18"); @@ -179,6 +184,60 @@ void snb_uncore_cpu_init(void) snb_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores; } +static void skl_uncore_msr_init_box(struct intel_uncore_box *box) +{ + if (box->pmu->pmu_idx == 0) { + wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, + SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL); + } +} + +static void skl_uncore_msr_exit_box(struct intel_uncore_box *box) +{ + if (box->pmu->pmu_idx == 0) + wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, 0); +} + +static struct intel_uncore_ops skl_uncore_msr_ops = { + .init_box = skl_uncore_msr_init_box, + .exit_box = skl_uncore_msr_exit_box, + .disable_event = snb_uncore_msr_disable_event, + .enable_event = snb_uncore_msr_enable_event, + .read_counter = uncore_msr_read_counter, +}; + +static struct intel_uncore_type skl_uncore_cbox = { + .name = "cbox", + .num_counters = 4, + .num_boxes = 5, + .perf_ctr_bits = 44, + .fixed_ctr_bits = 48, + .perf_ctr = SNB_UNC_CBO_0_PER_CTR0, + .event_ctl = SNB_UNC_CBO_0_PERFEVTSEL0, + .fixed_ctr = SNB_UNC_FIXED_CTR, + .fixed_ctl = SNB_UNC_FIXED_CTR_CTRL, + .single_fixed = 1, + .event_mask = SNB_UNC_RAW_EVENT_MASK, + .msr_offset = SNB_UNC_CBO_MSR_OFFSET, + .ops = &skl_uncore_msr_ops, + .format_group = &snb_uncore_format_group, + .event_descs = snb_uncore_events, +}; + +static struct intel_uncore_type *skl_msr_uncores[] = { + &skl_uncore_cbox, + &snb_uncore_arb, + NULL, +}; + +void skl_uncore_cpu_init(void) +{ + uncore_msr_uncores = skl_msr_uncores; + if (skl_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores) + skl_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores; + snb_uncore_arb.ops = &skl_uncore_msr_ops; +} + enum { SNB_PCI_UNCORE_IMC, }; @@ -544,6 +603,11 @@ static const struct pci_device_id skl_uncore_pci_ids[] = { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_IMC), .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_U_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* end: all zeroes */ }, }; @@ -587,6 +651,7 @@ static const struct imc_uncore_pci_dev desktop_imc_pci_ids[] = { IMC_DEV(HSW_U_IMC, &hsw_uncore_pci_driver), /* 4th Gen Core ULT Mobile Processor */ IMC_DEV(BDW_IMC, &bdw_uncore_pci_driver), /* 5th Gen Core U */ IMC_DEV(SKL_IMC, &skl_uncore_pci_driver), /* 6th Gen Core */ + IMC_DEV(SKL_U_IMC, &skl_uncore_pci_driver), /* 6th Gen Core U */ { /* end marker */ } }; diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 874e8bd64d1d..824e54086e07 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -2164,21 +2164,101 @@ static struct intel_uncore_type *knl_pci_uncores[] = { */ static const struct pci_device_id knl_uncore_pci_ids[] = { - { /* MC UClk */ + { /* MC0 UClk */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7841), - .driver_data = UNCORE_PCI_DEV_DATA(KNL_PCI_UNCORE_MC_UCLK, 0), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(10, 0, KNL_PCI_UNCORE_MC_UCLK, 0), }, - { /* MC DClk Channel */ + { /* MC1 UClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7841), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(11, 0, KNL_PCI_UNCORE_MC_UCLK, 1), + }, + { /* MC0 DClk CH 0 */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(8, 2, KNL_PCI_UNCORE_MC_DCLK, 0), + }, + { /* MC0 DClk CH 1 */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(8, 3, KNL_PCI_UNCORE_MC_DCLK, 1), + }, + { /* MC0 DClk CH 2 */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(8, 4, KNL_PCI_UNCORE_MC_DCLK, 2), + }, + { /* MC1 DClk CH 0 */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(9, 2, KNL_PCI_UNCORE_MC_DCLK, 3), + }, + { /* MC1 DClk CH 1 */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843), - .driver_data = UNCORE_PCI_DEV_DATA(KNL_PCI_UNCORE_MC_DCLK, 0), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(9, 3, KNL_PCI_UNCORE_MC_DCLK, 4), + }, + { /* MC1 DClk CH 2 */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(9, 4, KNL_PCI_UNCORE_MC_DCLK, 5), + }, + { /* EDC0 UClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(15, 0, KNL_PCI_UNCORE_EDC_UCLK, 0), + }, + { /* EDC1 UClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(16, 0, KNL_PCI_UNCORE_EDC_UCLK, 1), + }, + { /* EDC2 UClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(17, 0, KNL_PCI_UNCORE_EDC_UCLK, 2), + }, + { /* EDC3 UClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(18, 0, KNL_PCI_UNCORE_EDC_UCLK, 3), }, - { /* EDC UClk */ + { /* EDC4 UClk */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833), - .driver_data = UNCORE_PCI_DEV_DATA(KNL_PCI_UNCORE_EDC_UCLK, 0), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(19, 0, KNL_PCI_UNCORE_EDC_UCLK, 4), + }, + { /* EDC5 UClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(20, 0, KNL_PCI_UNCORE_EDC_UCLK, 5), + }, + { /* EDC6 UClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(21, 0, KNL_PCI_UNCORE_EDC_UCLK, 6), + }, + { /* EDC7 UClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(22, 0, KNL_PCI_UNCORE_EDC_UCLK, 7), + }, + { /* EDC0 EClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(24, 2, KNL_PCI_UNCORE_EDC_ECLK, 0), + }, + { /* EDC1 EClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(25, 2, KNL_PCI_UNCORE_EDC_ECLK, 1), + }, + { /* EDC2 EClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(26, 2, KNL_PCI_UNCORE_EDC_ECLK, 2), + }, + { /* EDC3 EClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(27, 2, KNL_PCI_UNCORE_EDC_ECLK, 3), + }, + { /* EDC4 EClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(28, 2, KNL_PCI_UNCORE_EDC_ECLK, 4), + }, + { /* EDC5 EClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(29, 2, KNL_PCI_UNCORE_EDC_ECLK, 5), + }, + { /* EDC6 EClk */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(30, 2, KNL_PCI_UNCORE_EDC_ECLK, 6), }, - { /* EDC EClk */ + { /* EDC7 EClk */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835), - .driver_data = UNCORE_PCI_DEV_DATA(KNL_PCI_UNCORE_EDC_ECLK, 0), + .driver_data = UNCORE_PCI_DEV_FULL_DATA(31, 2, KNL_PCI_UNCORE_EDC_ECLK, 7), }, { /* M2PCIe */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7817), diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c index 85ef3c2e80e0..50b3a056f96b 100644 --- a/arch/x86/events/msr.c +++ b/arch/x86/events/msr.c @@ -1,4 +1,5 @@ #include +#include enum perf_msr_id { PERF_MSR_TSC = 0, @@ -34,39 +35,43 @@ static bool test_intel(int idx) return false; switch (boot_cpu_data.x86_model) { - case 30: /* 45nm Nehalem */ - case 26: /* 45nm Nehalem-EP */ - case 46: /* 45nm Nehalem-EX */ - - case 37: /* 32nm Westmere */ - case 44: /* 32nm Westmere-EP */ - case 47: /* 32nm Westmere-EX */ - - case 42: /* 32nm SandyBridge */ - case 45: /* 32nm SandyBridge-E/EN/EP */ - - case 58: /* 22nm IvyBridge */ - case 62: /* 22nm IvyBridge-EP/EX */ - - case 60: /* 22nm Haswell Core */ - case 63: /* 22nm Haswell Server */ - case 69: /* 22nm Haswell ULT */ - case 70: /* 22nm Haswell + GT3e (Intel Iris Pro graphics) */ - - case 61: /* 14nm Broadwell Core-M */ - case 86: /* 14nm Broadwell Xeon D */ - case 71: /* 14nm Broadwell + GT3e (Intel Iris Pro graphics) */ - case 79: /* 14nm Broadwell Server */ - - case 55: /* 22nm Atom "Silvermont" */ - case 77: /* 22nm Atom "Silvermont Avoton/Rangely" */ - case 76: /* 14nm Atom "Airmont" */ + case INTEL_FAM6_NEHALEM: + case INTEL_FAM6_NEHALEM_EP: + case INTEL_FAM6_NEHALEM_EX: + + case INTEL_FAM6_WESTMERE: + case INTEL_FAM6_WESTMERE2: + case INTEL_FAM6_WESTMERE_EP: + case INTEL_FAM6_WESTMERE_EX: + + case INTEL_FAM6_SANDYBRIDGE: + case INTEL_FAM6_SANDYBRIDGE_X: + + case INTEL_FAM6_IVYBRIDGE: + case INTEL_FAM6_IVYBRIDGE_X: + + case INTEL_FAM6_HASWELL_CORE: + case INTEL_FAM6_HASWELL_X: + case INTEL_FAM6_HASWELL_ULT: + case INTEL_FAM6_HASWELL_GT3E: + + case INTEL_FAM6_BROADWELL_CORE: + case INTEL_FAM6_BROADWELL_XEON_D: + case INTEL_FAM6_BROADWELL_GT3E: + case INTEL_FAM6_BROADWELL_X: + + case INTEL_FAM6_ATOM_SILVERMONT1: + case INTEL_FAM6_ATOM_SILVERMONT2: + case INTEL_FAM6_ATOM_AIRMONT: if (idx == PERF_MSR_SMI) return true; break; - case 78: /* 14nm Skylake Mobile */ - case 94: /* 14nm Skylake Desktop */ + case INTEL_FAM6_SKYLAKE_MOBILE: + case INTEL_FAM6_SKYLAKE_DESKTOP: + case INTEL_FAM6_SKYLAKE_X: + case INTEL_FAM6_KABYLAKE_MOBILE: + case INTEL_FAM6_KABYLAKE_DESKTOP: if (idx == PERF_MSR_SMI || idx == PERF_MSR_PPERF) return true; break; diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 8bd764df815d..8c4a47706296 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -668,6 +668,14 @@ static struct perf_pmu_events_attr event_attr_##v = { \ .event_str = str, \ }; +#define EVENT_ATTR_STR_HT(_name, v, noht, ht) \ +static struct perf_pmu_events_ht_attr event_attr_##v = { \ + .attr = __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\ + .id = 0, \ + .event_str_noht = noht, \ + .event_str_ht = ht, \ +} + extern struct x86_pmu x86_pmu __read_mostly; static inline bool x86_pmu_has_lbr_callstack(void) @@ -803,6 +811,8 @@ struct attribute **merge_attr(struct attribute **a, struct attribute **b); ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page); +ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, + char *page); #ifdef CONFIG_CPU_SUP_AMD @@ -892,6 +902,8 @@ void intel_ds_init(void); void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in); +u64 lbr_from_signext_quirk_wr(u64 val); + void intel_pmu_lbr_reset(void); void intel_pmu_lbr_enable(struct perf_event *event); diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild index aeac434c9feb..2cfed174e3c9 100644 --- a/arch/x86/include/asm/Kbuild +++ b/arch/x86/include/asm/Kbuild @@ -1,5 +1,11 @@ +generated-y += syscalls_32.h +generated-y += syscalls_64.h +generated-y += unistd_32_ia32.h +generated-y += unistd_64_x32.h +generated-y += xen-hypercalls.h + genhdr-y += unistd_32.h genhdr-y += unistd_64.h genhdr-y += unistd_x32.h diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h index 94c18ebfd68c..5391b0ae7cc3 100644 --- a/arch/x86/include/asm/acpi.h +++ b/arch/x86/include/asm/acpi.h @@ -145,7 +145,6 @@ static inline void disable_acpi(void) { } #define ARCH_HAS_POWER_INIT 1 #ifdef CONFIG_ACPI_NUMA -extern int acpi_numa; extern int x86_acpi_numa_init(void); #endif /* CONFIG_ACPI_NUMA */ @@ -170,4 +169,6 @@ static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr) } #endif +#define ACPI_TABLE_UPGRADE_MAX_PHYS (max_low_pfn_mapped << PAGE_SHIFT) + #endif /* _ASM_X86_ACPI_H */ diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index bc27611fa58f..f5befd4945f2 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -300,7 +300,6 @@ struct apic { unsigned int (*get_apic_id)(unsigned long x); unsigned long (*set_apic_id)(unsigned int id); - unsigned long apic_id_mask; int (*cpu_mask_to_apicid_and)(const struct cpumask *cpumask, const struct cpumask *andmask, diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h index 20370c6db74b..93eebc636c76 100644 --- a/arch/x86/include/asm/apm.h +++ b/arch/x86/include/asm/apm.h @@ -45,11 +45,11 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in, : "memory", "cc"); } -static inline u8 apm_bios_call_simple_asm(u32 func, u32 ebx_in, - u32 ecx_in, u32 *eax) +static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in, + u32 ecx_in, u32 *eax) { int cx, dx, si; - u8 error; + bool error; /* * N.B. We do NOT need a cld after the BIOS call diff --git a/arch/x86/include/asm/arch_hweight.h b/arch/x86/include/asm/arch_hweight.h index 02e799fa43d1..e7cd63175de4 100644 --- a/arch/x86/include/asm/arch_hweight.h +++ b/arch/x86/include/asm/arch_hweight.h @@ -4,8 +4,8 @@ #include #ifdef CONFIG_64BIT -/* popcnt %edi, %eax -- redundant REX prefix for alignment */ -#define POPCNT32 ".byte 0xf3,0x40,0x0f,0xb8,0xc7" +/* popcnt %edi, %eax */ +#define POPCNT32 ".byte 0xf3,0x0f,0xb8,0xc7" /* popcnt %rdi, %rax */ #define POPCNT64 ".byte 0xf3,0x48,0x0f,0xb8,0xc7" #define REG_IN "D" @@ -17,19 +17,15 @@ #define REG_OUT "a" #endif -/* - * __sw_hweightXX are called from within the alternatives below - * and callee-clobbered registers need to be taken care of. See - * ARCH_HWEIGHT_CFLAGS in for the respective - * compiler switches. - */ +#define __HAVE_ARCH_SW_HWEIGHT + static __always_inline unsigned int __arch_hweight32(unsigned int w) { - unsigned int res = 0; + unsigned int res; asm (ALTERNATIVE("call __sw_hweight32", POPCNT32, X86_FEATURE_POPCNT) - : "="REG_OUT (res) - : REG_IN (w)); + : "="REG_OUT (res) + : REG_IN (w)); return res; } @@ -53,11 +49,11 @@ static inline unsigned long __arch_hweight64(__u64 w) #else static __always_inline unsigned long __arch_hweight64(__u64 w) { - unsigned long res = 0; + unsigned long res; asm (ALTERNATIVE("call __sw_hweight64", POPCNT64, X86_FEATURE_POPCNT) - : "="REG_OUT (res) - : REG_IN (w)); + : "="REG_OUT (res) + : REG_IN (w)); return res; } diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h index 69f1366f1aa3..5b0579abb398 100644 --- a/arch/x86/include/asm/archrandom.h +++ b/arch/x86/include/asm/archrandom.h @@ -25,8 +25,6 @@ #include #include -#include -#include #define RDRAND_RETRY_LOOPS 10 @@ -40,97 +38,91 @@ # define RDSEED_LONG RDSEED_INT #endif -#ifdef CONFIG_ARCH_RANDOM +/* Unconditional execution of RDRAND and RDSEED */ -/* Instead of arch_get_random_long() when alternatives haven't run. */ -static inline int rdrand_long(unsigned long *v) +static inline bool rdrand_long(unsigned long *v) { - int ok; - asm volatile("1: " RDRAND_LONG "\n\t" - "jc 2f\n\t" - "decl %0\n\t" - "jnz 1b\n\t" - "2:" - : "=r" (ok), "=a" (*v) - : "0" (RDRAND_RETRY_LOOPS)); - return ok; + bool ok; + unsigned int retry = RDRAND_RETRY_LOOPS; + do { + asm volatile(RDRAND_LONG "\n\t" + CC_SET(c) + : CC_OUT(c) (ok), "=a" (*v)); + if (ok) + return true; + } while (--retry); + return false; +} + +static inline bool rdrand_int(unsigned int *v) +{ + bool ok; + unsigned int retry = RDRAND_RETRY_LOOPS; + do { + asm volatile(RDRAND_INT "\n\t" + CC_SET(c) + : CC_OUT(c) (ok), "=a" (*v)); + if (ok) + return true; + } while (--retry); + return false; } -/* A single attempt at RDSEED */ static inline bool rdseed_long(unsigned long *v) { - unsigned char ok; + bool ok; asm volatile(RDSEED_LONG "\n\t" - "setc %0" - : "=qm" (ok), "=a" (*v)); + CC_SET(c) + : CC_OUT(c) (ok), "=a" (*v)); return ok; } -#define GET_RANDOM(name, type, rdrand, nop) \ -static inline int name(type *v) \ -{ \ - int ok; \ - alternative_io("movl $0, %0\n\t" \ - nop, \ - "\n1: " rdrand "\n\t" \ - "jc 2f\n\t" \ - "decl %0\n\t" \ - "jnz 1b\n\t" \ - "2:", \ - X86_FEATURE_RDRAND, \ - ASM_OUTPUT2("=r" (ok), "=a" (*v)), \ - "0" (RDRAND_RETRY_LOOPS)); \ - return ok; \ -} - -#define GET_SEED(name, type, rdseed, nop) \ -static inline int name(type *v) \ -{ \ - unsigned char ok; \ - alternative_io("movb $0, %0\n\t" \ - nop, \ - rdseed "\n\t" \ - "setc %0", \ - X86_FEATURE_RDSEED, \ - ASM_OUTPUT2("=q" (ok), "=a" (*v))); \ - return ok; \ +static inline bool rdseed_int(unsigned int *v) +{ + bool ok; + asm volatile(RDSEED_INT "\n\t" + CC_SET(c) + : CC_OUT(c) (ok), "=a" (*v)); + return ok; } -#ifdef CONFIG_X86_64 - -GET_RANDOM(arch_get_random_long, unsigned long, RDRAND_LONG, ASM_NOP5); -GET_RANDOM(arch_get_random_int, unsigned int, RDRAND_INT, ASM_NOP4); - -GET_SEED(arch_get_random_seed_long, unsigned long, RDSEED_LONG, ASM_NOP5); -GET_SEED(arch_get_random_seed_int, unsigned int, RDSEED_INT, ASM_NOP4); - -#else - -GET_RANDOM(arch_get_random_long, unsigned long, RDRAND_LONG, ASM_NOP3); -GET_RANDOM(arch_get_random_int, unsigned int, RDRAND_INT, ASM_NOP3); - -GET_SEED(arch_get_random_seed_long, unsigned long, RDSEED_LONG, ASM_NOP4); -GET_SEED(arch_get_random_seed_int, unsigned int, RDSEED_INT, ASM_NOP4); - -#endif /* CONFIG_X86_64 */ - +/* Conditional execution based on CPU type */ #define arch_has_random() static_cpu_has(X86_FEATURE_RDRAND) #define arch_has_random_seed() static_cpu_has(X86_FEATURE_RDSEED) -#else +/* + * These are the generic interfaces; they must not be declared if the + * stubs in are to be invoked, + * i.e. CONFIG_ARCH_RANDOM is not defined. + */ +#ifdef CONFIG_ARCH_RANDOM -static inline int rdrand_long(unsigned long *v) +static inline bool arch_get_random_long(unsigned long *v) { - return 0; + return arch_has_random() ? rdrand_long(v) : false; } -static inline bool rdseed_long(unsigned long *v) +static inline bool arch_get_random_int(unsigned int *v) { - return 0; + return arch_has_random() ? rdrand_int(v) : false; } -#endif /* CONFIG_ARCH_RANDOM */ +static inline bool arch_get_random_seed_long(unsigned long *v) +{ + return arch_has_random_seed() ? rdseed_long(v) : false; +} + +static inline bool arch_get_random_seed_int(unsigned int *v) +{ + return arch_has_random_seed() ? rdseed_int(v) : false; +} extern void x86_init_rdrand(struct cpuinfo_x86 *c); +#else /* !CONFIG_ARCH_RANDOM */ + +static inline void x86_init_rdrand(struct cpuinfo_x86 *c) { } + +#endif /* !CONFIG_ARCH_RANDOM */ + #endif /* ASM_X86_ARCHRANDOM_H */ diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index f5063b6659eb..7acb51c49fec 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -42,6 +42,18 @@ #define _ASM_SI __ASM_REG(si) #define _ASM_DI __ASM_REG(di) +/* + * Macros to generate condition code outputs from inline assembly, + * The output operand must be type "bool". + */ +#ifdef __GCC_ASM_FLAG_OUTPUTS__ +# define CC_SET(c) "\n\t/* output condition code " #c "*/\n" +# define CC_OUT(c) "=@cc" #c +#else +# define CC_SET(c) "\n\tset" #c " %[_cc_" #c "]\n" +# define CC_OUT(c) [_cc_ ## c] "=qm" +#endif + /* Exception table entry */ #ifdef __ASSEMBLY__ # define _ASM_EXTABLE_HANDLE(from, to, handler) \ diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index 3e8674288198..14635c5ea025 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -75,9 +75,9 @@ static __always_inline void atomic_sub(int i, atomic_t *v) * true if the result is zero, or false for all * other cases. */ -static __always_inline int atomic_sub_and_test(int i, atomic_t *v) +static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) { - GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", "e"); + GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", e); } /** @@ -112,9 +112,9 @@ static __always_inline void atomic_dec(atomic_t *v) * returns true if the result is 0, or false for all other * cases. */ -static __always_inline int atomic_dec_and_test(atomic_t *v) +static __always_inline bool atomic_dec_and_test(atomic_t *v) { - GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", "e"); + GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", e); } /** @@ -125,9 +125,9 @@ static __always_inline int atomic_dec_and_test(atomic_t *v) * and returns true if the result is zero, or false for all * other cases. */ -static __always_inline int atomic_inc_and_test(atomic_t *v) +static __always_inline bool atomic_inc_and_test(atomic_t *v) { - GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", "e"); + GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", e); } /** @@ -139,9 +139,9 @@ static __always_inline int atomic_inc_and_test(atomic_t *v) * if the result is negative, or false when * result is greater than or equal to zero. */ -static __always_inline int atomic_add_negative(int i, atomic_t *v) +static __always_inline bool atomic_add_negative(int i, atomic_t *v) { - GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", "s"); + GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", s); } /** @@ -171,6 +171,16 @@ static __always_inline int atomic_sub_return(int i, atomic_t *v) #define atomic_inc_return(v) (atomic_add_return(1, v)) #define atomic_dec_return(v) (atomic_sub_return(1, v)) +static __always_inline int atomic_fetch_add(int i, atomic_t *v) +{ + return xadd(&v->counter, i); +} + +static __always_inline int atomic_fetch_sub(int i, atomic_t *v) +{ + return xadd(&v->counter, -i); +} + static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new) { return cmpxchg(&v->counter, old, new); @@ -190,10 +200,29 @@ static inline void atomic_##op(int i, atomic_t *v) \ : "memory"); \ } -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#define ATOMIC_FETCH_OP(op, c_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int old, val = atomic_read(v); \ + for (;;) { \ + old = atomic_cmpxchg(v, val, val c_op i); \ + if (old == val) \ + break; \ + val = old; \ + } \ + return old; \ +} + +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op) \ + ATOMIC_FETCH_OP(op, c_op) + +ATOMIC_OPS(and, &) +ATOMIC_OPS(or , |) +ATOMIC_OPS(xor, ^) +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP /** diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h index a984111135b1..71d7705fb303 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -320,10 +320,29 @@ static inline void atomic64_##op(long long i, atomic64_t *v) \ c = old; \ } -ATOMIC64_OP(and, &) -ATOMIC64_OP(or, |) -ATOMIC64_OP(xor, ^) +#define ATOMIC64_FETCH_OP(op, c_op) \ +static inline long long atomic64_fetch_##op(long long i, atomic64_t *v) \ +{ \ + long long old, c = 0; \ + while ((old = atomic64_cmpxchg(v, c, c c_op i)) != c) \ + c = old; \ + return old; \ +} + +ATOMIC64_FETCH_OP(add, +) + +#define atomic64_fetch_sub(i, v) atomic64_fetch_add(-(i), (v)) + +#define ATOMIC64_OPS(op, c_op) \ + ATOMIC64_OP(op, c_op) \ + ATOMIC64_FETCH_OP(op, c_op) + +ATOMIC64_OPS(and, &) +ATOMIC64_OPS(or, |) +ATOMIC64_OPS(xor, ^) +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP #endif /* _ASM_X86_ATOMIC64_32_H */ diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h index 037351022f54..89ed2f6ae2f7 100644 --- a/arch/x86/include/asm/atomic64_64.h +++ b/arch/x86/include/asm/atomic64_64.h @@ -70,9 +70,9 @@ static inline void atomic64_sub(long i, atomic64_t *v) * true if the result is zero, or false for all * other cases. */ -static inline int atomic64_sub_and_test(long i, atomic64_t *v) +static inline bool atomic64_sub_and_test(long i, atomic64_t *v) { - GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", "e"); + GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", e); } /** @@ -109,9 +109,9 @@ static __always_inline void atomic64_dec(atomic64_t *v) * returns true if the result is 0, or false for all other * cases. */ -static inline int atomic64_dec_and_test(atomic64_t *v) +static inline bool atomic64_dec_and_test(atomic64_t *v) { - GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", "e"); + GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", e); } /** @@ -122,9 +122,9 @@ static inline int atomic64_dec_and_test(atomic64_t *v) * and returns true if the result is zero, or false for all * other cases. */ -static inline int atomic64_inc_and_test(atomic64_t *v) +static inline bool atomic64_inc_and_test(atomic64_t *v) { - GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", "e"); + GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", e); } /** @@ -136,9 +136,9 @@ static inline int atomic64_inc_and_test(atomic64_t *v) * if the result is negative, or false when * result is greater than or equal to zero. */ -static inline int atomic64_add_negative(long i, atomic64_t *v) +static inline bool atomic64_add_negative(long i, atomic64_t *v) { - GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", "s"); + GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", s); } /** @@ -158,6 +158,16 @@ static inline long atomic64_sub_return(long i, atomic64_t *v) return atomic64_add_return(-i, v); } +static inline long atomic64_fetch_add(long i, atomic64_t *v) +{ + return xadd(&v->counter, i); +} + +static inline long atomic64_fetch_sub(long i, atomic64_t *v) +{ + return xadd(&v->counter, -i); +} + #define atomic64_inc_return(v) (atomic64_add_return(1, (v))) #define atomic64_dec_return(v) (atomic64_sub_return(1, (v))) @@ -180,7 +190,7 @@ static inline long atomic64_xchg(atomic64_t *v, long new) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static inline int atomic64_add_unless(atomic64_t *v, long a, long u) +static inline bool atomic64_add_unless(atomic64_t *v, long a, long u) { long c, old; c = atomic64_read(v); @@ -229,10 +239,29 @@ static inline void atomic64_##op(long i, atomic64_t *v) \ : "memory"); \ } -ATOMIC64_OP(and) -ATOMIC64_OP(or) -ATOMIC64_OP(xor) +#define ATOMIC64_FETCH_OP(op, c_op) \ +static inline long atomic64_fetch_##op(long i, atomic64_t *v) \ +{ \ + long old, val = atomic64_read(v); \ + for (;;) { \ + old = atomic64_cmpxchg(v, val, val c_op i); \ + if (old == val) \ + break; \ + val = old; \ + } \ + return old; \ +} + +#define ATOMIC64_OPS(op, c_op) \ + ATOMIC64_OP(op) \ + ATOMIC64_FETCH_OP(op, c_op) + +ATOMIC64_OPS(and, &) +ATOMIC64_OPS(or, |) +ATOMIC64_OPS(xor, ^) +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP #endif /* _ASM_X86_ATOMIC64_64_H */ diff --git a/arch/x86/include/asm/bios_ebda.h b/arch/x86/include/asm/bios_ebda.h index 2b00c776f223..4b7b8e71607e 100644 --- a/arch/x86/include/asm/bios_ebda.h +++ b/arch/x86/include/asm/bios_ebda.h @@ -17,7 +17,7 @@ static inline unsigned int get_bios_ebda(void) return address; /* 0 means none */ } -void reserve_ebda_region(void); +void reserve_bios_regions(void); #ifdef CONFIG_X86_CHECK_BIOS_CORRUPTION /* diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 7766d1cf096e..68557f52b961 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -201,9 +201,9 @@ static __always_inline void change_bit(long nr, volatile unsigned long *addr) * This operation is atomic and cannot be reordered. * It also implies a memory barrier. */ -static __always_inline int test_and_set_bit(long nr, volatile unsigned long *addr) +static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr) { - GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c"); + GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", c); } /** @@ -213,7 +213,7 @@ static __always_inline int test_and_set_bit(long nr, volatile unsigned long *add * * This is the same as test_and_set_bit on x86. */ -static __always_inline int +static __always_inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr) { return test_and_set_bit(nr, addr); @@ -228,13 +228,13 @@ test_and_set_bit_lock(long nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static __always_inline int __test_and_set_bit(long nr, volatile unsigned long *addr) +static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) { - int oldbit; + bool oldbit; asm("bts %2,%1\n\t" - "sbb %0,%0" - : "=r" (oldbit), ADDR + CC_SET(c) + : CC_OUT(c) (oldbit), ADDR : "Ir" (nr)); return oldbit; } @@ -247,9 +247,9 @@ static __always_inline int __test_and_set_bit(long nr, volatile unsigned long *a * This operation is atomic and cannot be reordered. * It also implies a memory barrier. */ -static __always_inline int test_and_clear_bit(long nr, volatile unsigned long *addr) +static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) { - GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c"); + GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", c); } /** @@ -268,25 +268,25 @@ static __always_inline int test_and_clear_bit(long nr, volatile unsigned long *a * accessed from a hypervisor on the same CPU if running in a VM: don't change * this without also updating arch/x86/kernel/kvm.c */ -static __always_inline int __test_and_clear_bit(long nr, volatile unsigned long *addr) +static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) { - int oldbit; + bool oldbit; asm volatile("btr %2,%1\n\t" - "sbb %0,%0" - : "=r" (oldbit), ADDR + CC_SET(c) + : CC_OUT(c) (oldbit), ADDR : "Ir" (nr)); return oldbit; } /* WARNING: non atomic and it can be reordered! */ -static __always_inline int __test_and_change_bit(long nr, volatile unsigned long *addr) +static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) { - int oldbit; + bool oldbit; asm volatile("btc %2,%1\n\t" - "sbb %0,%0" - : "=r" (oldbit), ADDR + CC_SET(c) + : CC_OUT(c) (oldbit), ADDR : "Ir" (nr) : "memory"); return oldbit; @@ -300,24 +300,24 @@ static __always_inline int __test_and_change_bit(long nr, volatile unsigned long * This operation is atomic and cannot be reordered. * It also implies a memory barrier. */ -static __always_inline int test_and_change_bit(long nr, volatile unsigned long *addr) +static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr) { - GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c"); + GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", c); } -static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr) +static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr) { return ((1UL << (nr & (BITS_PER_LONG-1))) & (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; } -static __always_inline int variable_test_bit(long nr, volatile const unsigned long *addr) +static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr) { - int oldbit; + bool oldbit; asm volatile("bt %2,%1\n\t" - "sbb %0,%0" - : "=r" (oldbit) + CC_SET(c) + : CC_OUT(c) (oldbit) : "m" (*(unsigned long *)addr), "Ir" (nr)); return oldbit; @@ -329,7 +329,7 @@ static __always_inline int variable_test_bit(long nr, volatile const unsigned lo * @nr: bit number to test * @addr: Address to start counting from */ -static int test_bit(int nr, const volatile unsigned long *addr); +static bool test_bit(int nr, const volatile unsigned long *addr); #endif #define test_bit(nr, addr) \ diff --git a/arch/x86/include/asm/checksum_32.h b/arch/x86/include/asm/checksum_32.h index 532f85e6651f..7b53743ed267 100644 --- a/arch/x86/include/asm/checksum_32.h +++ b/arch/x86/include/asm/checksum_32.h @@ -2,8 +2,7 @@ #define _ASM_X86_CHECKSUM_32_H #include - -#include +#include /* * computes the checksum of a memory block at buff, length len, diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h index 5a3b2c119ed0..a18806165fe4 100644 --- a/arch/x86/include/asm/compat.h +++ b/arch/x86/include/asm/compat.h @@ -40,6 +40,7 @@ typedef s32 compat_long_t; typedef s64 __attribute__((aligned(4))) compat_s64; typedef u32 compat_uint_t; typedef u32 compat_ulong_t; +typedef u32 compat_u32; typedef u64 __attribute__((aligned(4))) compat_u64; typedef u32 compat_uptr_t; @@ -181,6 +182,16 @@ typedef struct compat_siginfo { /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ struct { unsigned int _addr; /* faulting insn/memory ref. */ + short int _addr_lsb; /* Valid LSB of the reported address. */ + union { + /* used when si_code=SEGV_BNDERR */ + struct { + compat_uptr_t _lower; + compat_uptr_t _upper; + } _addr_bnd; + /* used when si_code=SEGV_PKUERR */ + compat_u32 _pkey; + }; } _sigfault; /* SIGPOLL */ diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h index 678637ad7476..9b7fa6313f1a 100644 --- a/arch/x86/include/asm/cpu.h +++ b/arch/x86/include/asm/cpu.h @@ -16,8 +16,8 @@ extern void prefill_possible_map(void); static inline void prefill_possible_map(void) {} #define cpu_physical_id(cpu) boot_cpu_physical_apicid +#define cpu_acpi_id(cpu) 0 #define safe_smp_processor_id() 0 -#define stack_smp_processor_id() 0 #endif /* CONFIG_SMP */ diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 4a413485f9eb..c64b1e9c5d1a 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -301,10 +301,6 @@ #define X86_BUG_FXSAVE_LEAK X86_BUG(6) /* FXSAVE leaks FOP/FIP/FOP */ #define X86_BUG_CLFLUSH_MONITOR X86_BUG(7) /* AAI65, CLFLUSH required before MONITOR */ #define X86_BUG_SYSRET_SS_ATTRS X86_BUG(8) /* SYSRET doesn't fix up SS attrs */ -#define X86_BUG_NULL_SEG X86_BUG(9) /* Nulling a selector preserves the base */ -#define X86_BUG_SWAPGS_FENCE X86_BUG(10) /* SWAPGS without input dep on GS */ - - #ifdef CONFIG_X86_32 /* * 64-bit kernels don't use X86_BUG_ESPFIX. Make the define conditional @@ -312,5 +308,7 @@ */ #define X86_BUG_ESPFIX X86_BUG(9) /* "" IRET to 16-bit SS corrupts ESP/RSP high bits */ #endif +#define X86_BUG_NULL_SEG X86_BUG(10) /* Nulling a selector preserves the base */ +#define X86_BUG_SWAPGS_FENCE X86_BUG(11) /* SWAPGS without input dep on GS */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index 78d1e7467eae..d0bb76d81402 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -41,10 +41,9 @@ extern unsigned long asmlinkage efi_call_phys(void *, ...); /* * Wrap all the virtual calls in a way that forces the parameters on the stack. */ -#define arch_efi_call_virt(f, args...) \ +#define arch_efi_call_virt(p, f, args...) \ ({ \ - ((efi_##f##_t __attribute__((regparm(0)))*) \ - efi.systab->runtime->f)(args); \ + ((efi_##f##_t __attribute__((regparm(0)))*) p->f)(args); \ }) #define efi_ioremap(addr, size, type, attr) ioremap_cache(addr, size) @@ -81,8 +80,8 @@ struct efi_scratch { } \ }) -#define arch_efi_call_virt(f, args...) \ - efi_call((void *)efi.systab->runtime->f, args) \ +#define arch_efi_call_virt(p, f, args...) \ + efi_call((void *)p->f, args) \ #define arch_efi_call_virt_teardown() \ ({ \ @@ -125,7 +124,6 @@ extern void __init efi_map_region_fixed(efi_memory_desc_t *md); extern void efi_sync_low_kernel_mappings(void); extern int __init efi_alloc_page_tables(void); extern int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages); -extern void __init efi_cleanup_page_tables(unsigned long pa_memmap, unsigned num_pages); extern void __init old_map_region(efi_memory_desc_t *md); extern void __init runtime_code_page_mkexec(void); extern void __init efi_runtime_update_mappings(void); diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index 31ac8e6d9f36..116b58347501 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -18,6 +18,7 @@ #include #include #include +#include /* * High level FPU state handling functions: @@ -524,6 +525,7 @@ static inline void __fpregs_deactivate(struct fpu *fpu) fpu->fpregs_active = 0; this_cpu_write(fpu_fpregs_owner_ctx, NULL); + trace_x86_fpu_regs_deactivated(fpu); } /* Must be paired with a 'clts' (fpregs_activate_hw()) before! */ @@ -533,6 +535,7 @@ static inline void __fpregs_activate(struct fpu *fpu) fpu->fpregs_active = 1; this_cpu_write(fpu_fpregs_owner_ctx, fpu); + trace_x86_fpu_regs_activated(fpu); } /* @@ -604,11 +607,13 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu) /* But leave fpu_fpregs_owner_ctx! */ old_fpu->fpregs_active = 0; + trace_x86_fpu_regs_deactivated(old_fpu); /* Don't change CR0.TS if we just switch! */ if (fpu.preload) { new_fpu->counter++; __fpregs_activate(new_fpu); + trace_x86_fpu_regs_activated(new_fpu); prefetch(&new_fpu->state); } else { __fpregs_deactivate_hw(); diff --git a/arch/x86/include/asm/fpu/types.h b/arch/x86/include/asm/fpu/types.h index 36b90bbfc69f..48df486b02f9 100644 --- a/arch/x86/include/asm/fpu/types.h +++ b/arch/x86/include/asm/fpu/types.h @@ -122,6 +122,7 @@ enum xfeature { #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) +#define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR) #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) @@ -230,6 +231,12 @@ struct xstate_header { u64 reserved[6]; } __attribute__((packed)); +/* + * xstate_header.xcomp_bv[63] indicates that the extended_state_area + * is in compacted format. + */ +#define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63) + /* * This is our most modern FPU state format, as saved by the XSAVE * and restored by the XRSTOR instructions. diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h index 38951b0fcc5a..ae55a43e09c0 100644 --- a/arch/x86/include/asm/fpu/xstate.h +++ b/arch/x86/include/asm/fpu/xstate.h @@ -18,6 +18,9 @@ #define XSAVE_YMM_SIZE 256 #define XSAVE_YMM_OFFSET (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET) +/* Supervisor features */ +#define XFEATURE_MASK_SUPERVISOR (XFEATURE_MASK_PT) + /* Supported features which support lazy state saving */ #define XFEATURE_MASK_LAZY (XFEATURE_MASK_FP | \ XFEATURE_MASK_SSE | \ @@ -39,7 +42,6 @@ #define REX_PREFIX #endif -extern unsigned int xstate_size; extern u64 xfeatures_mask; extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS]; @@ -48,5 +50,9 @@ extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); void fpu__xstate_clear_all_cpu_caps(void); void *get_xsave_addr(struct xregs_state *xsave, int xstate); const void *get_xsave_field_ptr(int xstate_field); - +int using_compacted_format(void); +int copyout_from_xsaves(unsigned int pos, unsigned int count, void *kbuf, + void __user *ubuf, struct xregs_state *xsave); +int copyin_to_xsaves(const void *kbuf, const void __user *ubuf, + struct xregs_state *xsave); #endif diff --git a/arch/x86/include/asm/inat.h b/arch/x86/include/asm/inat.h index 74a2e312e8a2..02aff0867211 100644 --- a/arch/x86/include/asm/inat.h +++ b/arch/x86/include/asm/inat.h @@ -48,6 +48,7 @@ /* AVX VEX prefixes */ #define INAT_PFX_VEX2 13 /* 2-bytes VEX prefix */ #define INAT_PFX_VEX3 14 /* 3-bytes VEX prefix */ +#define INAT_PFX_EVEX 15 /* EVEX prefix */ #define INAT_LSTPFX_MAX 3 #define INAT_LGCPFX_MAX 11 @@ -89,6 +90,7 @@ #define INAT_VARIANT (1 << (INAT_FLAG_OFFS + 4)) #define INAT_VEXOK (1 << (INAT_FLAG_OFFS + 5)) #define INAT_VEXONLY (1 << (INAT_FLAG_OFFS + 6)) +#define INAT_EVEXONLY (1 << (INAT_FLAG_OFFS + 7)) /* Attribute making macros for attribute tables */ #define INAT_MAKE_PREFIX(pfx) (pfx << INAT_PFX_OFFS) #define INAT_MAKE_ESCAPE(esc) (esc << INAT_ESC_OFFS) @@ -141,7 +143,13 @@ static inline int inat_last_prefix_id(insn_attr_t attr) static inline int inat_is_vex_prefix(insn_attr_t attr) { attr &= INAT_PFX_MASK; - return attr == INAT_PFX_VEX2 || attr == INAT_PFX_VEX3; + return attr == INAT_PFX_VEX2 || attr == INAT_PFX_VEX3 || + attr == INAT_PFX_EVEX; +} + +static inline int inat_is_evex_prefix(insn_attr_t attr) +{ + return (attr & INAT_PFX_MASK) == INAT_PFX_EVEX; } static inline int inat_is_vex3_prefix(insn_attr_t attr) @@ -216,6 +224,11 @@ static inline int inat_accept_vex(insn_attr_t attr) static inline int inat_must_vex(insn_attr_t attr) { - return attr & INAT_VEXONLY; + return attr & (INAT_VEXONLY | INAT_EVEXONLY); +} + +static inline int inat_must_evex(insn_attr_t attr) +{ + return attr & INAT_EVEXONLY; } #endif diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h index e7814b74caf8..b3e32b010ab1 100644 --- a/arch/x86/include/asm/insn.h +++ b/arch/x86/include/asm/insn.h @@ -91,6 +91,7 @@ struct insn { #define X86_VEX_B(vex) ((vex) & 0x20) /* VEX3 Byte1 */ #define X86_VEX_L(vex) ((vex) & 0x04) /* VEX3 Byte2, VEX2 Byte1 */ /* VEX bit fields */ +#define X86_EVEX_M(vex) ((vex) & 0x03) /* EVEX Byte1 */ #define X86_VEX3_M(vex) ((vex) & 0x1f) /* VEX3 Byte1 */ #define X86_VEX2_M 1 /* VEX2.M always 1 */ #define X86_VEX_V(vex) (((vex) & 0x78) >> 3) /* VEX3 Byte2, VEX2 Byte1 */ @@ -133,6 +134,13 @@ static inline int insn_is_avx(struct insn *insn) return (insn->vex_prefix.value != 0); } +static inline int insn_is_evex(struct insn *insn) +{ + if (!insn->prefixes.got) + insn_get_prefixes(insn); + return (insn->vex_prefix.nbytes == 4); +} + /* Ensure this instruction is decoded completely */ static inline int insn_complete(struct insn *insn) { @@ -144,8 +152,10 @@ static inline insn_byte_t insn_vex_m_bits(struct insn *insn) { if (insn->vex_prefix.nbytes == 2) /* 2 bytes VEX */ return X86_VEX2_M; - else + else if (insn->vex_prefix.nbytes == 3) /* 3 bytes VEX */ return X86_VEX3_M(insn->vex_prefix.bytes[1]); + else /* EVEX */ + return X86_EVEX_M(insn->vex_prefix.bytes[1]); } static inline insn_byte_t insn_vex_p_bits(struct insn *insn) diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index 7c5af123bdbd..9d6b097aa73d 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -12,9 +12,17 @@ #define _ASM_X86_INTEL_MID_H #include +#include #include extern int intel_mid_pci_init(void); +extern int intel_mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state); + +#define INTEL_MID_PWR_LSS_OFFSET 4 +#define INTEL_MID_PWR_LSS_TYPE (1 << 7) + +extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev); + extern int get_gpio_by_name(const char *name); extern void intel_scu_device_register(struct platform_device *pdev); extern int __init sfi_parse_mrtc(struct sfi_table_header *table); @@ -34,13 +42,28 @@ struct devs_id { void *(*get_platform_data)(void *info); /* Custom handler for devices */ void (*device_handler)(struct sfi_device_table_entry *pentry, - struct devs_id *dev); + struct devs_id *dev); }; -#define sfi_device(i) \ - static const struct devs_id *const __intel_mid_sfi_##i##_dev __used \ +#define sfi_device(i) \ + static const struct devs_id *const __intel_mid_sfi_##i##_dev __used \ __attribute__((__section__(".x86_intel_mid_dev.init"))) = &i +/** +* struct mid_sd_board_info - template for SD device creation +* @name: identifies the driver +* @bus_num: board-specific identifier for a given SD controller +* @max_clk: the maximum frequency device supports +* @platform_data: the particular data stored there is driver-specific +*/ +struct mid_sd_board_info { + char name[SFI_NAME_LEN]; + int bus_num; + unsigned short addr; + u32 max_clk; + void *platform_data; +}; + /* * Medfield is the follow-up of Moorestown, it combines two chip solution into * one. Other than that it also added always-on and constant tsc and lapic @@ -60,7 +83,7 @@ extern enum intel_mid_cpu_type __intel_mid_cpu_chip; /** * struct intel_mid_ops - Interface between intel-mid & sub archs * @arch_setup: arch_setup function to re-initialize platform - * structures (x86_init, x86_platform_init) + * structures (x86_init, x86_platform_init) * * This structure can be extended if any new interface is required * between intel-mid & its sub arch files. @@ -70,20 +93,20 @@ struct intel_mid_ops { }; /* Helper API's for INTEL_MID_OPS_INIT */ -#define DECLARE_INTEL_MID_OPS_INIT(cpuname, cpuid) \ - [cpuid] = get_##cpuname##_ops +#define DECLARE_INTEL_MID_OPS_INIT(cpuname, cpuid) \ + [cpuid] = get_##cpuname##_ops /* Maximum number of CPU ops */ -#define MAX_CPU_OPS(a) (sizeof(a)/sizeof(void *)) +#define MAX_CPU_OPS(a) (sizeof(a)/sizeof(void *)) /* * For every new cpu addition, a weak get__ops() function needs be * declared in arch/x86/platform/intel_mid/intel_mid_weak_decls.h. */ -#define INTEL_MID_OPS_INIT {\ - DECLARE_INTEL_MID_OPS_INIT(penwell, INTEL_MID_CPU_CHIP_PENWELL), \ - DECLARE_INTEL_MID_OPS_INIT(cloverview, INTEL_MID_CPU_CHIP_CLOVERVIEW), \ - DECLARE_INTEL_MID_OPS_INIT(tangier, INTEL_MID_CPU_CHIP_TANGIER) \ +#define INTEL_MID_OPS_INIT { \ + DECLARE_INTEL_MID_OPS_INIT(penwell, INTEL_MID_CPU_CHIP_PENWELL), \ + DECLARE_INTEL_MID_OPS_INIT(cloverview, INTEL_MID_CPU_CHIP_CLOVERVIEW), \ + DECLARE_INTEL_MID_OPS_INIT(tangier, INTEL_MID_CPU_CHIP_TANGIER) \ }; #ifdef CONFIG_X86_INTEL_MID @@ -100,8 +123,8 @@ static inline bool intel_mid_has_msic(void) #else /* !CONFIG_X86_INTEL_MID */ -#define intel_mid_identify_cpu() (0) -#define intel_mid_has_msic() (0) +#define intel_mid_identify_cpu() 0 +#define intel_mid_has_msic() 0 #endif /* !CONFIG_X86_INTEL_MID */ @@ -117,35 +140,38 @@ extern enum intel_mid_timer_options intel_mid_timer_options; * Penwell uses spread spectrum clock, so the freq number is not exactly * the same as reported by MSR based on SDM. */ -#define FSB_FREQ_83SKU 83200 -#define FSB_FREQ_100SKU 99840 -#define FSB_FREQ_133SKU 133000 +#define FSB_FREQ_83SKU 83200 +#define FSB_FREQ_100SKU 99840 +#define FSB_FREQ_133SKU 133000 -#define FSB_FREQ_167SKU 167000 -#define FSB_FREQ_200SKU 200000 -#define FSB_FREQ_267SKU 267000 -#define FSB_FREQ_333SKU 333000 -#define FSB_FREQ_400SKU 400000 +#define FSB_FREQ_167SKU 167000 +#define FSB_FREQ_200SKU 200000 +#define FSB_FREQ_267SKU 267000 +#define FSB_FREQ_333SKU 333000 +#define FSB_FREQ_400SKU 400000 /* Bus Select SoC Fuse value */ -#define BSEL_SOC_FUSE_MASK 0x7 -#define BSEL_SOC_FUSE_001 0x1 /* FSB 133MHz */ -#define BSEL_SOC_FUSE_101 0x5 /* FSB 100MHz */ -#define BSEL_SOC_FUSE_111 0x7 /* FSB 83MHz */ +#define BSEL_SOC_FUSE_MASK 0x7 +/* FSB 133MHz */ +#define BSEL_SOC_FUSE_001 0x1 +/* FSB 100MHz */ +#define BSEL_SOC_FUSE_101 0x5 +/* FSB 83MHz */ +#define BSEL_SOC_FUSE_111 0x7 -#define SFI_MTMR_MAX_NUM 8 -#define SFI_MRTC_MAX 8 +#define SFI_MTMR_MAX_NUM 8 +#define SFI_MRTC_MAX 8 extern void intel_scu_devices_create(void); extern void intel_scu_devices_destroy(void); /* VRTC timer */ -#define MRST_VRTC_MAP_SZ (1024) -/*#define MRST_VRTC_PGOFFSET (0xc00) */ +#define MRST_VRTC_MAP_SZ 1024 +/* #define MRST_VRTC_PGOFFSET 0xc00 */ extern void intel_mid_rtc_init(void); -/* the offset for the mapping of global gpio pin to irq */ -#define INTEL_MID_IRQ_OFFSET 0x100 +/* The offset for the mapping of global gpio pin to irq */ +#define INTEL_MID_IRQ_OFFSET 0x100 #endif /* _ASM_X86_INTEL_MID_H */ diff --git a/arch/x86/include/asm/kaslr.h b/arch/x86/include/asm/kaslr.h new file mode 100644 index 000000000000..2674ee3de748 --- /dev/null +++ b/arch/x86/include/asm/kaslr.h @@ -0,0 +1,15 @@ +#ifndef _ASM_KASLR_H_ +#define _ASM_KASLR_H_ + +unsigned long kaslr_get_random_long(const char *purpose); + +#ifdef CONFIG_RANDOMIZE_MEMORY +extern unsigned long page_offset_base; +extern unsigned long vmalloc_base; + +void kernel_randomize_memory(void); +#else +static inline void kernel_randomize_memory(void) { } +#endif /* CONFIG_RANDOMIZE_MEMORY */ + +#endif diff --git a/arch/x86/include/asm/kdebug.h b/arch/x86/include/asm/kdebug.h index e5f5dc9787d5..1ef9d581b5d9 100644 --- a/arch/x86/include/asm/kdebug.h +++ b/arch/x86/include/asm/kdebug.h @@ -26,6 +26,7 @@ extern void die(const char *, struct pt_regs *,long); extern int __must_check __die(const char *, struct pt_regs *, long); extern void show_trace(struct task_struct *t, struct pt_regs *regs, unsigned long *sp, unsigned long bp); +extern void show_stack_regs(struct pt_regs *regs); extern void __show_regs(struct pt_regs *regs, int all); extern unsigned long oops_begin(void); extern void oops_end(unsigned long, struct pt_regs *, int signr); diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h index 4421b5da409d..d1d1e5094c28 100644 --- a/arch/x86/include/asm/kprobes.h +++ b/arch/x86/include/asm/kprobes.h @@ -38,12 +38,11 @@ typedef u8 kprobe_opcode_t; #define RELATIVECALL_OPCODE 0xe8 #define RELATIVE_ADDR_SIZE 4 #define MAX_STACK_SIZE 64 -#define MIN_STACK_SIZE(ADDR) \ - (((MAX_STACK_SIZE) < (((unsigned long)current_thread_info()) + \ - THREAD_SIZE - (unsigned long)(ADDR))) \ - ? (MAX_STACK_SIZE) \ - : (((unsigned long)current_thread_info()) + \ - THREAD_SIZE - (unsigned long)(ADDR))) +#define CUR_STACK_SIZE(ADDR) \ + (current_top_of_stack() - (unsigned long)(ADDR)) +#define MIN_STACK_SIZE(ADDR) \ + (MAX_STACK_SIZE < CUR_STACK_SIZE(ADDR) ? \ + MAX_STACK_SIZE : CUR_STACK_SIZE(ADDR)) #define flush_insn_slot(p) do { } while (0) diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h index 4ad6560847b1..7511978093eb 100644 --- a/arch/x86/include/asm/local.h +++ b/arch/x86/include/asm/local.h @@ -50,9 +50,9 @@ static inline void local_sub(long i, local_t *l) * true if the result is zero, or false for all * other cases. */ -static inline int local_sub_and_test(long i, local_t *l) +static inline bool local_sub_and_test(long i, local_t *l) { - GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, "er", i, "%0", "e"); + GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, "er", i, "%0", e); } /** @@ -63,9 +63,9 @@ static inline int local_sub_and_test(long i, local_t *l) * returns true if the result is 0, or false for all other * cases. */ -static inline int local_dec_and_test(local_t *l) +static inline bool local_dec_and_test(local_t *l) { - GEN_UNARY_RMWcc(_ASM_DEC, l->a.counter, "%0", "e"); + GEN_UNARY_RMWcc(_ASM_DEC, l->a.counter, "%0", e); } /** @@ -76,9 +76,9 @@ static inline int local_dec_and_test(local_t *l) * and returns true if the result is zero, or false for all * other cases. */ -static inline int local_inc_and_test(local_t *l) +static inline bool local_inc_and_test(local_t *l) { - GEN_UNARY_RMWcc(_ASM_INC, l->a.counter, "%0", "e"); + GEN_UNARY_RMWcc(_ASM_INC, l->a.counter, "%0", e); } /** @@ -90,9 +90,9 @@ static inline int local_inc_and_test(local_t *l) * if the result is negative, or false when * result is greater than or equal to zero. */ -static inline int local_add_negative(long i, local_t *l) +static inline bool local_add_negative(long i, local_t *l) { - GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, "er", i, "%0", "s"); + GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, "er", i, "%0", s); } /** diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 5a73a9c62c39..56f4c6676b29 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -64,8 +64,6 @@ #define MSR_OFFCORE_RSP_0 0x000001a6 #define MSR_OFFCORE_RSP_1 0x000001a7 -#define MSR_NHM_TURBO_RATIO_LIMIT 0x000001ad -#define MSR_IVT_TURBO_RATIO_LIMIT 0x000001ae #define MSR_TURBO_RATIO_LIMIT 0x000001ad #define MSR_TURBO_RATIO_LIMIT1 0x000001ae #define MSR_TURBO_RATIO_LIMIT2 0x000001af diff --git a/arch/x86/include/asm/mutex_32.h b/arch/x86/include/asm/mutex_32.h index 85e6cda45a02..e9355a84fc67 100644 --- a/arch/x86/include/asm/mutex_32.h +++ b/arch/x86/include/asm/mutex_32.h @@ -101,7 +101,7 @@ static inline int __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) { /* cmpxchg because it never induces a false contention state. */ - if (likely(atomic_cmpxchg(count, 1, 0) == 1)) + if (likely(atomic_read(count) == 1 && atomic_cmpxchg(count, 1, 0) == 1)) return 1; return 0; diff --git a/arch/x86/include/asm/mutex_64.h b/arch/x86/include/asm/mutex_64.h index 07537a44216e..d9850758464e 100644 --- a/arch/x86/include/asm/mutex_64.h +++ b/arch/x86/include/asm/mutex_64.h @@ -118,10 +118,10 @@ do { \ static inline int __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) { - if (likely(atomic_cmpxchg(count, 1, 0) == 1)) + if (likely(atomic_read(count) == 1 && atomic_cmpxchg(count, 1, 0) == 1)) return 1; - else - return 0; + + return 0; } #endif /* _ASM_X86_MUTEX_64_H */ diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h index d5c2f8b40faa..9215e0527647 100644 --- a/arch/x86/include/asm/page_64_types.h +++ b/arch/x86/include/asm/page_64_types.h @@ -1,6 +1,10 @@ #ifndef _ASM_X86_PAGE_64_DEFS_H #define _ASM_X86_PAGE_64_DEFS_H +#ifndef __ASSEMBLY__ +#include +#endif + #ifdef CONFIG_KASAN #define KASAN_STACK_ORDER 1 #else @@ -32,7 +36,12 @@ * hypervisor to fit. Choosing 16 slots here is arbitrary, but it's * what Xen requires. */ -#define __PAGE_OFFSET _AC(0xffff880000000000, UL) +#define __PAGE_OFFSET_BASE _AC(0xffff880000000000, UL) +#ifdef CONFIG_RANDOMIZE_MEMORY +#define __PAGE_OFFSET page_offset_base +#else +#define __PAGE_OFFSET __PAGE_OFFSET_BASE +#endif /* CONFIG_RANDOMIZE_MEMORY */ #define __START_KERNEL_map _AC(0xffffffff80000000, UL) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index e0ba66ca68c6..e02e3f80d363 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -510,14 +510,15 @@ do { \ /* This is not atomic against other CPUs -- CPU preemption needs to be off */ #define x86_test_and_clear_bit_percpu(bit, var) \ ({ \ - int old__; \ - asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \ - : "=r" (old__), "+m" (var) \ + bool old__; \ + asm volatile("btr %2,"__percpu_arg(1)"\n\t" \ + CC_SET(c) \ + : CC_OUT(c) (old__), "+m" (var) \ : "dIr" (bit)); \ old__; \ }) -static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, +static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr, const unsigned long __percpu *addr) { unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG; @@ -529,14 +530,14 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, #endif } -static inline int x86_this_cpu_variable_test_bit(int nr, +static inline bool x86_this_cpu_variable_test_bit(int nr, const unsigned long __percpu *addr) { - int oldbit; + bool oldbit; asm volatile("bt "__percpu_arg(2)",%1\n\t" - "sbb %0,%0" - : "=r" (oldbit) + CC_SET(c) + : CC_OUT(c) (oldbit) : "m" (*(unsigned long *)addr), "Ir" (nr)); return oldbit; diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h index bf7f8b55b0f9..b6d425999f99 100644 --- a/arch/x86/include/asm/pgalloc.h +++ b/arch/x86/include/asm/pgalloc.h @@ -81,7 +81,11 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { struct page *page; - page = alloc_pages(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO, 0); + gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO; + + if (mm == &init_mm) + gfp &= ~__GFP_ACCOUNT; + page = alloc_pages(gfp, 0); if (!page) return NULL; if (!pgtable_pmd_page_ctor(page)) { @@ -125,7 +129,11 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return (pud_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); + gfp_t gfp = GFP_KERNEL_ACCOUNT; + + if (mm == &init_mm) + gfp &= ~__GFP_ACCOUNT; + return (pud_t *)get_zeroed_page(gfp); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 1a27396b6ea0..437feb436efa 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -480,7 +480,7 @@ pte_t *populate_extra_pte(unsigned long vaddr); static inline int pte_none(pte_t pte) { - return !pte.pte; + return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK)); } #define __HAVE_ARCH_PTE_SAME @@ -552,7 +552,8 @@ static inline int pmd_none(pmd_t pmd) { /* Only check low word on 32-bit platforms, since it might be out of sync with upper half. */ - return (unsigned long)native_pmd_val(pmd) == 0; + unsigned long val = native_pmd_val(pmd); + return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0; } static inline unsigned long pmd_page_vaddr(pmd_t pmd) @@ -616,7 +617,7 @@ static inline unsigned long pages_to_mb(unsigned long npg) #if CONFIG_PGTABLE_LEVELS > 2 static inline int pud_none(pud_t pud) { - return native_pud_val(pud) == 0; + return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0; } static inline int pud_present(pud_t pud) @@ -694,6 +695,12 @@ static inline int pgd_bad(pgd_t pgd) static inline int pgd_none(pgd_t pgd) { + /* + * There is no need to do a workaround for the KNL stray + * A/D bit erratum here. PGDs only point to page tables + * except on 32-bit non-PAE which is not supported on + * KNL. + */ return !native_pgd_val(pgd); } #endif /* CONFIG_PGTABLE_LEVELS > 3 */ @@ -729,6 +736,23 @@ extern int direct_gbpages; void init_mem_mapping(void); void early_alloc_pgt_buf(void); +#ifdef CONFIG_X86_64 +/* Realmode trampoline initialization. */ +extern pgd_t trampoline_pgd_entry; +static inline void __meminit init_trampoline_default(void) +{ + /* Default trampoline pgd value */ + trampoline_pgd_entry = init_level4_pgt[pgd_index(__PAGE_OFFSET)]; +} +# ifdef CONFIG_RANDOMIZE_MEMORY +void __meminit init_trampoline(void); +# else +# define init_trampoline init_trampoline_default +# endif +#else +static inline void init_trampoline(void) { } +#endif + /* local pte updates need not use xchg for locking */ static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) { diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index 2ee781114d34..7e8ec7ae10fa 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -140,18 +140,32 @@ static inline int pgd_large(pgd_t pgd) { return 0; } #define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) #define pte_unmap(pte) ((void)(pte))/* NOP */ -/* Encode and de-code a swap entry */ +/* + * Encode and de-code a swap entry + * + * | ... | 11| 10| 9|8|7|6|5| 4| 3|2|1|0| <- bit number + * | ... |SW3|SW2|SW1|G|L|D|A|CD|WT|U|W|P| <- bit names + * | OFFSET (14->63) | TYPE (10-13) |0|X|X|X| X| X|X|X|0| <- swp entry + * + * G (8) is aliased and used as a PROT_NONE indicator for + * !present ptes. We need to start storing swap entries above + * there. We also need to avoid using A and D because of an + * erratum where they can be incorrectly set by hardware on + * non-present PTEs. + */ +#define SWP_TYPE_FIRST_BIT (_PAGE_BIT_PROTNONE + 1) #define SWP_TYPE_BITS 5 -#define SWP_OFFSET_SHIFT (_PAGE_BIT_PROTNONE + 1) +/* Place the offset above the type: */ +#define SWP_OFFSET_FIRST_BIT (SWP_TYPE_FIRST_BIT + SWP_TYPE_BITS + 1) #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS) -#define __swp_type(x) (((x).val >> (_PAGE_BIT_PRESENT + 1)) \ +#define __swp_type(x) (((x).val >> (SWP_TYPE_FIRST_BIT)) \ & ((1U << SWP_TYPE_BITS) - 1)) -#define __swp_offset(x) ((x).val >> SWP_OFFSET_SHIFT) +#define __swp_offset(x) ((x).val >> SWP_OFFSET_FIRST_BIT) #define __swp_entry(type, offset) ((swp_entry_t) { \ - ((type) << (_PAGE_BIT_PRESENT + 1)) \ - | ((offset) << SWP_OFFSET_SHIFT) }) + ((type) << (SWP_TYPE_FIRST_BIT)) \ + | ((offset) << SWP_OFFSET_FIRST_BIT) }) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) }) #define __swp_entry_to_pte(x) ((pte_t) { .pte = (x).val }) diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h index e6844dfb4471..6fdef9eef2d5 100644 --- a/arch/x86/include/asm/pgtable_64_types.h +++ b/arch/x86/include/asm/pgtable_64_types.h @@ -5,6 +5,7 @@ #ifndef __ASSEMBLY__ #include +#include /* * These are used to make use of C type-checking.. @@ -53,10 +54,16 @@ typedef struct { pteval_t pte; } pte_t; #define PGDIR_MASK (~(PGDIR_SIZE - 1)) /* See Documentation/x86/x86_64/mm.txt for a description of the memory map. */ -#define MAXMEM _AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL) -#define VMALLOC_START _AC(0xffffc90000000000, UL) -#define VMALLOC_END _AC(0xffffe8ffffffffff, UL) -#define VMEMMAP_START _AC(0xffffea0000000000, UL) +#define MAXMEM _AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL) +#define VMALLOC_SIZE_TB _AC(32, UL) +#define __VMALLOC_BASE _AC(0xffffc90000000000, UL) +#define VMEMMAP_START _AC(0xffffea0000000000, UL) +#ifdef CONFIG_RANDOMIZE_MEMORY +#define VMALLOC_START vmalloc_base +#else +#define VMALLOC_START __VMALLOC_BASE +#endif /* CONFIG_RANDOMIZE_MEMORY */ +#define VMALLOC_END (VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL)) #define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE) #define MODULES_END _AC(0xffffffffff000000, UL) #define MODULES_LEN (MODULES_END - MODULES_VADDR) diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 7b5efe264eff..f1218f512f62 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -70,6 +70,12 @@ _PAGE_PKEY_BIT2 | \ _PAGE_PKEY_BIT3) +#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) +#define _PAGE_KNL_ERRATUM_MASK (_PAGE_DIRTY | _PAGE_ACCESSED) +#else +#define _PAGE_KNL_ERRATUM_MASK 0 +#endif + #ifdef CONFIG_KMEMCHECK #define _PAGE_HIDDEN (_AT(pteval_t, 1) << _PAGE_BIT_HIDDEN) #else @@ -475,8 +481,6 @@ extern pmd_t *lookup_pmd_address(unsigned long address); extern phys_addr_t slow_virt_to_phys(void *__address); extern int kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address, unsigned numpages, unsigned long page_flags); -void kernel_unmap_pages_in_pgd(pgd_t *root, unsigned long address, - unsigned numpages); #endif /* !__ASSEMBLY__ */ #endif /* _ASM_X86_PGTABLE_DEFS_H */ diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index d397deb58146..17f218645701 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -81,7 +81,7 @@ static __always_inline void __preempt_count_sub(int val) */ static __always_inline bool __preempt_count_dec_and_test(void) { - GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), "e"); + GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), e); } /* diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 62c6cc3cc5d3..63def9537a2d 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -367,10 +367,15 @@ DECLARE_PER_CPU(struct irq_stack *, hardirq_stack); DECLARE_PER_CPU(struct irq_stack *, softirq_stack); #endif /* X86_64 */ -extern unsigned int xstate_size; +extern unsigned int fpu_kernel_xstate_size; +extern unsigned int fpu_user_xstate_size; struct perf_event; +typedef struct { + unsigned long seg; +} mm_segment_t; + struct thread_struct { /* Cached TLS descriptors: */ struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES]; @@ -419,6 +424,11 @@ struct thread_struct { /* Max allowed port in the bitmap, in bytes: */ unsigned io_bitmap_max; + mm_segment_t addr_limit; + + unsigned int sig_on_uaccess_err:1; + unsigned int uaccess_err:1; /* uaccess failed */ + /* Floating point and extended processor state */ struct fpu fpu; /* @@ -490,11 +500,6 @@ static inline void load_sp0(struct tss_struct *tss, #define set_iopl_mask native_set_iopl_mask #endif /* CONFIG_PARAVIRT */ -typedef struct { - unsigned long seg; -} mm_segment_t; - - /* Free all resources held by a thread. */ extern void release_thread(struct task_struct *); @@ -716,6 +721,7 @@ static inline void spin_lock_prefetch(const void *x) .sp0 = TOP_OF_INIT_STACK, \ .sysenter_cs = __KERNEL_CS, \ .io_bitmap_ptr = NULL, \ + .addr_limit = KERNEL_DS, \ } extern unsigned long thread_saved_pc(struct task_struct *tsk); @@ -765,8 +771,9 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); #define STACK_TOP TASK_SIZE #define STACK_TOP_MAX TASK_SIZE_MAX -#define INIT_THREAD { \ - .sp0 = TOP_OF_INIT_STACK \ +#define INIT_THREAD { \ + .sp0 = TOP_OF_INIT_STACK, \ + .addr_limit = KERNEL_DS, \ } /* diff --git a/arch/x86/include/asm/pvclock.h b/arch/x86/include/asm/pvclock.h index fdcc04020636..7c1c89598688 100644 --- a/arch/x86/include/asm/pvclock.h +++ b/arch/x86/include/asm/pvclock.h @@ -68,30 +68,23 @@ static inline u64 pvclock_scale_delta(u64 delta, u32 mul_frac, int shift) return product; } -static __always_inline -u64 pvclock_get_nsec_offset(const struct pvclock_vcpu_time_info *src) -{ - u64 delta = rdtsc_ordered() - src->tsc_timestamp; - return pvclock_scale_delta(delta, src->tsc_to_system_mul, - src->tsc_shift); -} - static __always_inline unsigned __pvclock_read_cycles(const struct pvclock_vcpu_time_info *src, cycle_t *cycles, u8 *flags) { unsigned version; - cycle_t ret, offset; - u8 ret_flags; + cycle_t offset; + u64 delta; version = src->version; + /* Make the latest version visible */ + smp_rmb(); - offset = pvclock_get_nsec_offset(src); - ret = src->system_time + offset; - ret_flags = src->flags; - - *cycles = ret; - *flags = ret_flags; + delta = rdtsc_ordered() - src->tsc_timestamp; + offset = pvclock_scale_delta(delta, src->tsc_to_system_mul, + src->tsc_shift); + *cycles = src->system_time + offset; + *flags = src->flags; return version; } diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h index 8f7866a5b9a4..661dd305694a 100644 --- a/arch/x86/include/asm/rmwcc.h +++ b/arch/x86/include/asm/rmwcc.h @@ -1,11 +1,13 @@ #ifndef _ASM_X86_RMWcc #define _ASM_X86_RMWcc -#ifdef CC_HAVE_ASM_GOTO +#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO) + +/* Use asm goto */ #define __GEN_RMWcc(fullop, var, cc, ...) \ do { \ - asm_volatile_goto (fullop "; j" cc " %l[cc_label]" \ + asm_volatile_goto (fullop "; j" #cc " %l[cc_label]" \ : : "m" (var), ## __VA_ARGS__ \ : "memory" : cc_label); \ return 0; \ @@ -19,15 +21,17 @@ cc_label: \ #define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ __GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val)) -#else /* !CC_HAVE_ASM_GOTO */ +#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ + +/* Use flags output or a set instruction */ #define __GEN_RMWcc(fullop, var, cc, ...) \ do { \ - char c; \ - asm volatile (fullop "; set" cc " %1" \ - : "+m" (var), "=qm" (c) \ + bool c; \ + asm volatile (fullop ";" CC_SET(cc) \ + : "+m" (var), CC_OUT(cc) (c) \ : __VA_ARGS__ : "memory"); \ - return c != 0; \ + return c; \ } while (0) #define GEN_UNARY_RMWcc(op, var, arg0, cc) \ @@ -36,6 +40,6 @@ do { \ #define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ __GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val)) -#endif /* CC_HAVE_ASM_GOTO */ +#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ #endif /* _ASM_X86_RMWcc */ diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h index 453744c1d347..8dbc762ad132 100644 --- a/arch/x86/include/asm/rwsem.h +++ b/arch/x86/include/asm/rwsem.h @@ -77,7 +77,7 @@ static inline void __down_read(struct rw_semaphore *sem) /* * trylock for reading -- returns 1 if successful, 0 if contention */ -static inline int __down_read_trylock(struct rw_semaphore *sem) +static inline bool __down_read_trylock(struct rw_semaphore *sem) { long result, tmp; asm volatile("# beginning __down_read_trylock\n\t" @@ -93,7 +93,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) : "+m" (sem->count), "=&a" (result), "=&r" (tmp) : "i" (RWSEM_ACTIVE_READ_BIAS) : "memory", "cc"); - return result >= 0 ? 1 : 0; + return result >= 0; } /* @@ -134,9 +134,10 @@ static inline int __down_write_killable(struct rw_semaphore *sem) /* * trylock for writing -- returns 1 if successful, 0 if contention */ -static inline int __down_write_trylock(struct rw_semaphore *sem) +static inline bool __down_write_trylock(struct rw_semaphore *sem) { - long result, tmp; + bool result; + long tmp0, tmp1; asm volatile("# beginning __down_write_trylock\n\t" " mov %0,%1\n\t" "1:\n\t" @@ -144,14 +145,14 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) /* was the active mask 0 before? */ " jnz 2f\n\t" " mov %1,%2\n\t" - " add %3,%2\n\t" + " add %4,%2\n\t" LOCK_PREFIX " cmpxchg %2,%0\n\t" " jnz 1b\n\t" "2:\n\t" - " sete %b1\n\t" - " movzbl %b1, %k1\n\t" + CC_SET(e) "# ending __down_write_trylock\n\t" - : "+m" (sem->count), "=&a" (result), "=&r" (tmp) + : "+m" (sem->count), "=&a" (tmp0), "=&r" (tmp1), + CC_OUT(e) (result) : "er" (RWSEM_ACTIVE_WRITE_BIAS) : "memory", "cc"); return result; @@ -213,23 +214,5 @@ static inline void __downgrade_write(struct rw_semaphore *sem) : "memory", "cc"); } -/* - * implement atomic add functionality - */ -static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) -{ - asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0" - : "+m" (sem->count) - : "er" (delta)); -} - -/* - * implement exchange and add functionality - */ -static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) -{ - return delta + xadd(&sem->count, delta); -} - #endif /* __KERNEL__ */ #endif /* _ASM_X86_RWSEM_H */ diff --git a/arch/x86/include/asm/signal.h b/arch/x86/include/asm/signal.h index 2138c9ae19ee..dd1e7d6387ab 100644 --- a/arch/x86/include/asm/signal.h +++ b/arch/x86/include/asm/signal.h @@ -81,9 +81,9 @@ static inline int __const_sigismember(sigset_t *set, int _sig) static inline int __gen_sigismember(sigset_t *set, int _sig) { - int ret; - asm("btl %2,%1\n\tsbbl %0,%0" - : "=r"(ret) : "m"(*set), "Ir"(_sig-1) : "cc"); + unsigned char ret; + asm("btl %2,%1\n\tsetc %0" + : "=qm"(ret) : "m"(*set), "Ir"(_sig-1) : "cc"); return ret; } diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 66b057306f40..ebd0c164cd4e 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -33,6 +33,7 @@ static inline struct cpumask *cpu_llc_shared_mask(int cpu) } DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid); +DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid); #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32) DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid); @@ -135,6 +136,7 @@ int native_cpu_up(unsigned int cpunum, struct task_struct *tidle); int native_cpu_disable(void); int common_cpu_die(unsigned int cpu); void native_cpu_die(unsigned int cpu); +void hlt_play_dead(void); void native_play_dead(void); void play_dead_common(void); void wbinvd_on_cpu(int cpu); @@ -147,6 +149,7 @@ void x86_idle_thread_init(unsigned int cpu, struct task_struct *idle); void smp_store_boot_cpu_info(void); void smp_store_cpu_info(int id); #define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu) +#define cpu_acpi_id(cpu) per_cpu(x86_cpu_to_acpiid, cpu) #else /* !CONFIG_SMP */ #define wbinvd_on_cpu(cpu) wbinvd() @@ -172,12 +175,6 @@ extern int safe_smp_processor_id(void); #elif defined(CONFIG_X86_64_SMP) #define raw_smp_processor_id() (this_cpu_read(cpu_number)) -#define stack_smp_processor_id() \ -({ \ - struct thread_info *ti; \ - __asm__("andq %%rsp,%0; ":"=r" (ti) : "0" (CURRENT_MASK)); \ - ti->cpu; \ -}) #define safe_smp_processor_id() smp_processor_id() #endif diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h index 7c247e7404be..0944218af9e2 100644 --- a/arch/x86/include/asm/stacktrace.h +++ b/arch/x86/include/asm/stacktrace.h @@ -14,7 +14,7 @@ extern int kstack_depth_to_print; struct thread_info; struct stacktrace_ops; -typedef unsigned long (*walk_stack_t)(struct thread_info *tinfo, +typedef unsigned long (*walk_stack_t)(struct task_struct *task, unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, @@ -23,13 +23,13 @@ typedef unsigned long (*walk_stack_t)(struct thread_info *tinfo, int *graph); extern unsigned long -print_context_stack(struct thread_info *tinfo, +print_context_stack(struct task_struct *task, unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data, unsigned long *end, int *graph); extern unsigned long -print_context_stack_bp(struct thread_info *tinfo, +print_context_stack_bp(struct task_struct *task, unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data, unsigned long *end, int *graph); diff --git a/arch/x86/include/asm/sync_bitops.h b/arch/x86/include/asm/sync_bitops.h index f28a24b51dc7..cbf8847d02a0 100644 --- a/arch/x86/include/asm/sync_bitops.h +++ b/arch/x86/include/asm/sync_bitops.h @@ -79,10 +79,10 @@ static inline void sync_change_bit(long nr, volatile unsigned long *addr) */ static inline int sync_test_and_set_bit(long nr, volatile unsigned long *addr) { - int oldbit; + unsigned char oldbit; - asm volatile("lock; bts %2,%1\n\tsbbl %0,%0" - : "=r" (oldbit), "+m" (ADDR) + asm volatile("lock; bts %2,%1\n\tsetc %0" + : "=qm" (oldbit), "+m" (ADDR) : "Ir" (nr) : "memory"); return oldbit; } @@ -97,10 +97,10 @@ static inline int sync_test_and_set_bit(long nr, volatile unsigned long *addr) */ static inline int sync_test_and_clear_bit(long nr, volatile unsigned long *addr) { - int oldbit; + unsigned char oldbit; - asm volatile("lock; btr %2,%1\n\tsbbl %0,%0" - : "=r" (oldbit), "+m" (ADDR) + asm volatile("lock; btr %2,%1\n\tsetc %0" + : "=qm" (oldbit), "+m" (ADDR) : "Ir" (nr) : "memory"); return oldbit; } @@ -115,10 +115,10 @@ static inline int sync_test_and_clear_bit(long nr, volatile unsigned long *addr) */ static inline int sync_test_and_change_bit(long nr, volatile unsigned long *addr) { - int oldbit; + unsigned char oldbit; - asm volatile("lock; btc %2,%1\n\tsbbl %0,%0" - : "=r" (oldbit), "+m" (ADDR) + asm volatile("lock; btc %2,%1\n\tsetc %0" + : "=qm" (oldbit), "+m" (ADDR) : "Ir" (nr) : "memory"); return oldbit; } diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index 30c133ac05cd..89bff044a6f5 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -57,9 +57,6 @@ struct thread_info { __u32 flags; /* low level flags */ __u32 status; /* thread synchronous flags */ __u32 cpu; /* current CPU */ - mm_segment_t addr_limit; - unsigned int sig_on_uaccess_error:1; - unsigned int uaccess_err:1; /* uaccess failed */ }; #define INIT_THREAD_INFO(tsk) \ @@ -67,7 +64,6 @@ struct thread_info { .task = &tsk, \ .flags = 0, \ .cpu = 0, \ - .addr_limit = KERNEL_DS, \ } #define init_thread_info (init_thread_union.thread_info) @@ -186,11 +182,6 @@ static inline unsigned long current_stack_pointer(void) # define cpu_current_top_of_stack (cpu_tss + TSS_sp0) #endif -/* Load thread_info address into "reg" */ -#define GET_THREAD_INFO(reg) \ - _ASM_MOV PER_CPU_VAR(cpu_current_top_of_stack),reg ; \ - _ASM_SUB $(THREAD_SIZE),reg ; - /* * ASM operand which evaluates to a 'thread_info' address of * the current task, if it is known that "reg" is exactly "off" diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 7f991bd5031b..43e87a3dd95c 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -25,16 +25,6 @@ #ifndef _ASM_X86_TOPOLOGY_H #define _ASM_X86_TOPOLOGY_H -#ifdef CONFIG_X86_32 -# ifdef CONFIG_SMP -# define ENABLE_TOPO_DEFINES -# endif -#else -# ifdef CONFIG_SMP -# define ENABLE_TOPO_DEFINES -# endif -#endif - /* * to preserve the visibility of NUMA_NO_NODE definition, * moved to there from here. May be used independent of @@ -123,12 +113,20 @@ extern const struct cpumask *cpu_coregroup_mask(int cpu); #define topology_physical_package_id(cpu) (cpu_data(cpu).phys_proc_id) #define topology_core_id(cpu) (cpu_data(cpu).cpu_core_id) -#ifdef ENABLE_TOPO_DEFINES +#ifdef CONFIG_SMP #define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu)) #define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu)) extern unsigned int __max_logical_packages; #define topology_max_packages() (__max_logical_packages) + +extern int __max_smt_threads; + +static inline int topology_max_smt_threads(void) +{ + return __max_smt_threads; +} + int topology_update_package_map(unsigned int apicid, unsigned int cpu); extern int topology_phys_to_logical_pkg(unsigned int pkg); #else @@ -136,6 +134,7 @@ extern int topology_phys_to_logical_pkg(unsigned int pkg); static inline int topology_update_package_map(unsigned int apicid, unsigned int cpu) { return 0; } static inline int topology_phys_to_logical_pkg(unsigned int pkg) { return 0; } +static inline int topology_max_smt_threads(void) { return 1; } #endif static inline void arch_fix_phys_package_id(int num, u32 slot) diff --git a/arch/x86/include/asm/trace/fpu.h b/arch/x86/include/asm/trace/fpu.h new file mode 100644 index 000000000000..9217ab1f5bf6 --- /dev/null +++ b/arch/x86/include/asm/trace/fpu.h @@ -0,0 +1,119 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM x86_fpu + +#if !defined(_TRACE_FPU_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_FPU_H + +#include + +DECLARE_EVENT_CLASS(x86_fpu, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu), + + TP_STRUCT__entry( + __field(struct fpu *, fpu) + __field(bool, fpregs_active) + __field(bool, fpstate_active) + __field(int, counter) + __field(u64, xfeatures) + __field(u64, xcomp_bv) + ), + + TP_fast_assign( + __entry->fpu = fpu; + __entry->fpregs_active = fpu->fpregs_active; + __entry->fpstate_active = fpu->fpstate_active; + __entry->counter = fpu->counter; + if (boot_cpu_has(X86_FEATURE_OSXSAVE)) { + __entry->xfeatures = fpu->state.xsave.header.xfeatures; + __entry->xcomp_bv = fpu->state.xsave.header.xcomp_bv; + } + ), + TP_printk("x86/fpu: %p fpregs_active: %d fpstate_active: %d counter: %d xfeatures: %llx xcomp_bv: %llx", + __entry->fpu, + __entry->fpregs_active, + __entry->fpstate_active, + __entry->counter, + __entry->xfeatures, + __entry->xcomp_bv + ) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_state, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_before_save, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_after_save, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_before_restore, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_after_restore, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_regs_activated, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_regs_deactivated, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_activate_state, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_deactivate_state, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_init_state, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_dropped, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_copy_src, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_copy_dst, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +DEFINE_EVENT(x86_fpu, x86_fpu_xstate_check_failed, + TP_PROTO(struct fpu *fpu), + TP_ARGS(fpu) +); + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH asm/trace/ +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE fpu +#endif /* _TRACE_FPU_H */ + +/* This part must be outside protection */ +#include diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h index 7428697c5b8d..33b6365c22fe 100644 --- a/arch/x86/include/asm/tsc.h +++ b/arch/x86/include/asm/tsc.h @@ -35,7 +35,7 @@ extern void tsc_init(void); extern void mark_tsc_unstable(char *reason); extern int unsynchronized_tsc(void); extern int check_tsc_unstable(void); -extern int check_tsc_disabled(void); +extern unsigned long native_calibrate_cpu(void); extern unsigned long native_calibrate_tsc(void); extern unsigned long long native_sched_clock_from_tsc(u64 tsc); @@ -52,7 +52,6 @@ extern int notsc_setup(char *); extern void tsc_save_sched_clock_state(void); extern void tsc_restore_sched_clock_state(void); -/* MSR based TSC calibration for Intel Atom SoC platforms */ -unsigned long try_msr_calibrate_tsc(void); +unsigned long cpu_khz_from_msr(void); #endif /* _ASM_X86_TSC_H */ diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 2982387ba817..c03bfb68c503 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -29,12 +29,12 @@ #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX) #define get_ds() (KERNEL_DS) -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) +#define get_fs() (current->thread.addr_limit) +#define set_fs(x) (current->thread.addr_limit = (x)) #define segment_eq(a, b) ((a).seg == (b).seg) -#define user_addr_max() (current_thread_info()->addr_limit.seg) +#define user_addr_max() (current->thread.addr_limit.seg) #define __addr_ok(addr) \ ((unsigned long __force)(addr) < user_addr_max()) @@ -342,7 +342,26 @@ do { \ } while (0) #ifdef CONFIG_X86_32 -#define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad() +#define __get_user_asm_u64(x, ptr, retval, errret) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + asm volatile(ASM_STAC "\n" \ + "1: movl %2,%%eax\n" \ + "2: movl %3,%%edx\n" \ + "3: " ASM_CLAC "\n" \ + ".section .fixup,\"ax\"\n" \ + "4: mov %4,%0\n" \ + " xorl %%eax,%%eax\n" \ + " xorl %%edx,%%edx\n" \ + " jmp 3b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ + : "=r" (retval), "=A"(x) \ + : "m" (__m(__ptr)), "m" __m(((u32 *)(__ptr)) + 1), \ + "i" (errret), "0" (retval)); \ +}) + #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad() #else #define __get_user_asm_u64(x, ptr, retval, errret) \ @@ -429,7 +448,7 @@ do { \ #define __get_user_nocheck(x, ptr, size) \ ({ \ int __gu_err; \ - unsigned long __gu_val; \ + __inttype(*(ptr)) __gu_val; \ __uaccess_begin(); \ __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ __uaccess_end(); \ @@ -468,13 +487,13 @@ struct __large_struct { unsigned long buf[100]; }; * uaccess_try and catch */ #define uaccess_try do { \ - current_thread_info()->uaccess_err = 0; \ + current->thread.uaccess_err = 0; \ __uaccess_begin(); \ barrier(); #define uaccess_catch(err) \ __uaccess_end(); \ - (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \ + (err) |= (current->thread.uaccess_err ? -EFAULT : 0); \ } while (0) /** diff --git a/arch/x86/include/asm/unistd.h b/arch/x86/include/asm/unistd.h index 2b19caa4081c..32712a925f26 100644 --- a/arch/x86/include/asm/unistd.h +++ b/arch/x86/include/asm/unistd.h @@ -26,6 +26,8 @@ # define __ARCH_WANT_COMPAT_SYS_GETDENTS64 # define __ARCH_WANT_COMPAT_SYS_PREADV64 # define __ARCH_WANT_COMPAT_SYS_PWRITEV64 +# define __ARCH_WANT_COMPAT_SYS_PREADV64V2 +# define __ARCH_WANT_COMPAT_SYS_PWRITEV64V2 # endif diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 4dcdf74dfed8..6ba793178441 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -168,20 +168,22 @@ struct x86_legacy_devices { * struct x86_legacy_features - legacy x86 features * * @rtc: this device has a CMOS real-time clock present - * @ebda_search: it's safe to search for the EBDA signature in the hardware's - * low RAM + * @reserve_bios_regions: boot code will search for the EBDA address and the + * start of the 640k - 1M BIOS region. If false, the platform must + * ensure that its memory map correctly reserves sub-1MB regions as needed. * @devices: legacy x86 devices, refer to struct x86_legacy_devices * documentation for further details. */ struct x86_legacy_features { int rtc; - int ebda_search; + int reserve_bios_regions; struct x86_legacy_devices devices; }; /** * struct x86_platform_ops - platform specific runtime functions - * @calibrate_tsc: calibrate TSC + * @calibrate_cpu: calibrate CPU + * @calibrate_tsc: calibrate TSC, if different from CPU * @get_wallclock: get time from HW clock like RTC etc. * @set_wallclock: set time back to HW clock * @is_untracked_pat_range exclude from PAT logic @@ -200,6 +202,7 @@ struct x86_legacy_features { * semantics. */ struct x86_platform_ops { + unsigned long (*calibrate_cpu)(void); unsigned long (*calibrate_tsc)(void); void (*get_wallclock)(struct timespec *ts); int (*set_wallclock)(const struct timespec *ts); diff --git a/arch/x86/include/asm/xen/cpuid.h b/arch/x86/include/asm/xen/cpuid.h index 0d809e9fc975..3bdd10d71223 100644 --- a/arch/x86/include/asm/xen/cpuid.h +++ b/arch/x86/include/asm/xen/cpuid.h @@ -76,15 +76,18 @@ /* * Leaf 5 (0x40000x04) * HVM-specific features + * EAX: Features + * EBX: vcpu id (iff EAX has XEN_HVM_CPUID_VCPU_ID_PRESENT flag) */ -/* EAX Features */ /* Virtualized APIC registers */ #define XEN_HVM_CPUID_APIC_ACCESS_VIRT (1u << 0) /* Virtualized x2APIC accesses */ #define XEN_HVM_CPUID_X2APIC_VIRT (1u << 1) /* Memory mapped from other domains has valid IOMMU entries */ #define XEN_HVM_CPUID_IOMMU_MAPPINGS (1u << 2) +/* vcpu id is present in EBX */ +#define XEN_HVM_CPUID_VCPU_ID_PRESENT (1u << 3) #define XEN_CPUID_MAX_NUM_LEAVES 4 diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 9414f84584e4..6738e5c82cca 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -161,13 +161,15 @@ static int __init acpi_parse_madt(struct acpi_table_header *table) /** * acpi_register_lapic - register a local apic and generates a logic cpu number * @id: local apic id to register + * @acpiid: ACPI id to register * @enabled: this cpu is enabled or not * * Returns the logic cpu number which maps to the local apic */ -static int acpi_register_lapic(int id, u8 enabled) +static int acpi_register_lapic(int id, u32 acpiid, u8 enabled) { unsigned int ver = 0; + int cpu; if (id >= MAX_LOCAL_APIC) { printk(KERN_INFO PREFIX "skipped apicid that is too big\n"); @@ -182,7 +184,11 @@ static int acpi_register_lapic(int id, u8 enabled) if (boot_cpu_physical_apicid != -1U) ver = apic_version[boot_cpu_physical_apicid]; - return generic_processor_info(id, ver); + cpu = generic_processor_info(id, ver); + if (cpu >= 0) + early_per_cpu(x86_cpu_to_acpiid, cpu) = acpiid; + + return cpu; } static int __init @@ -212,7 +218,7 @@ acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end) if (!apic->apic_id_valid(apic_id) && enabled) printk(KERN_WARNING PREFIX "x2apic entry ignored\n"); else - acpi_register_lapic(apic_id, enabled); + acpi_register_lapic(apic_id, processor->uid, enabled); #else printk(KERN_WARNING PREFIX "x2apic entry ignored\n"); #endif @@ -240,6 +246,7 @@ acpi_parse_lapic(struct acpi_subtable_header * header, const unsigned long end) * when we use CPU hotplug. */ acpi_register_lapic(processor->id, /* APIC ID */ + processor->processor_id, /* ACPI ID */ processor->lapic_flags & ACPI_MADT_ENABLED); return 0; @@ -258,6 +265,7 @@ acpi_parse_sapic(struct acpi_subtable_header *header, const unsigned long end) acpi_table_print_madt_entry(header); acpi_register_lapic((processor->id << 8) | processor->eid,/* APIC ID */ + processor->processor_id, /* ACPI ID */ processor->lapic_flags & ACPI_MADT_ENABLED); return 0; @@ -714,7 +722,7 @@ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, int *pcpu) { int cpu; - cpu = acpi_register_lapic(physid, ACPI_MADT_ENABLED); + cpu = acpi_register_lapic(physid, U32_MAX, ACPI_MADT_ENABLED); if (cpu < 0) { pr_info(PREFIX "Unable to map lapic to logical cpu number\n"); return cpu; diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index a147e676fc7b..e991d5c8bb3a 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -71,8 +71,8 @@ int amd_cache_northbridges(void) while ((misc = next_northbridge(misc, amd_nb_misc_ids)) != NULL) i++; - if (i == 0) - return 0; + if (!i) + return -ENODEV; nb = kzalloc(i * sizeof(struct amd_northbridge), GFP_KERNEL); if (!nb) diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 60078a67d7e3..ac8d8ad8b009 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -92,8 +92,10 @@ static int apic_extnmi = APIC_EXTNMI_BSP; */ DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID); DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID); +DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX); EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); +EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid); #ifdef CONFIG_X86_32 @@ -2045,7 +2047,7 @@ int generic_processor_info(int apicid, int version) int thiscpu = max + disabled_cpus - 1; pr_warning( - "ACPI: NR_CPUS/possible_cpus limit of %i almost" + "APIC: NR_CPUS/possible_cpus limit of %i almost" " reached. Keeping one slot for boot cpu." " Processor %d/0x%x ignored.\n", max, thiscpu, apicid); @@ -2057,7 +2059,7 @@ int generic_processor_info(int apicid, int version) int thiscpu = max + disabled_cpus; pr_warning( - "ACPI: NR_CPUS/possible_cpus limit of %i reached." + "APIC: NR_CPUS/possible_cpus limit of %i reached." " Processor %d/0x%x ignored.\n", max, thiscpu, apicid); disabled_cpus++; @@ -2085,7 +2087,7 @@ int generic_processor_info(int apicid, int version) if (topology_update_package_map(apicid, cpu) < 0) { int thiscpu = max + disabled_cpus; - pr_warning("ACPI: Package limit reached. Processor %d/0x%x ignored.\n", + pr_warning("APIC: Package limit reached. Processor %d/0x%x ignored.\n", thiscpu, apicid); disabled_cpus++; return -ENOSPC; diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 76f89e2b245a..048747778d37 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -181,7 +181,6 @@ static struct apic apic_flat = { .get_apic_id = flat_get_apic_id, .set_apic_id = set_apic_id, - .apic_id_mask = 0xFFu << 24, .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, @@ -278,7 +277,6 @@ static struct apic apic_physflat = { .get_apic_id = flat_get_apic_id, .set_apic_id = set_apic_id, - .apic_id_mask = 0xFFu << 24, .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index 13d19ed58514..2cebf59092d8 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -141,7 +141,6 @@ struct apic apic_noop = { .get_apic_id = noop_get_apic_id, .set_apic_id = NULL, - .apic_id_mask = 0x0F << 24, .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index ab5c2c685a3c..714d4fda0d52 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -269,7 +269,6 @@ static const struct apic apic_numachip1 __refconst = { .get_apic_id = numachip1_get_apic_id, .set_apic_id = numachip1_set_apic_id, - .apic_id_mask = 0xffU << 24, .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, @@ -321,7 +320,6 @@ static const struct apic apic_numachip2 __refconst = { .get_apic_id = numachip2_get_apic_id, .set_apic_id = numachip2_set_apic_id, - .apic_id_mask = 0xffU << 24, .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index cf9bd896c12d..06dbaa458bfe 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -171,7 +171,6 @@ static struct apic apic_bigsmp = { .get_apic_id = bigsmp_get_apic_id, .set_apic_id = NULL, - .apic_id_mask = 0xFF << 24, .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 446702ed99dc..f072b9572634 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -981,7 +981,7 @@ static int alloc_irq_from_domain(struct irq_domain *domain, int ioapic, u32 gsi, return __irq_domain_alloc_irqs(domain, irq, 1, ioapic_alloc_attr_node(info), - info, legacy); + info, legacy, NULL); } /* @@ -1014,7 +1014,8 @@ static int alloc_isa_irq_from_domain(struct irq_domain *domain, info->ioapic_pin)) return -ENOMEM; } else { - irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true); + irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true, + NULL); if (irq >= 0) { irq_data = irq_domain_get_irq_data(domain, irq); data = irq_data->chip_data; @@ -2567,29 +2568,25 @@ static struct resource * __init ioapic_setup_resources(void) unsigned long n; struct resource *res; char *mem; - int i, num = 0; + int i; - for_each_ioapic(i) - num++; - if (num == 0) + if (nr_ioapics == 0) return NULL; n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource); - n *= num; + n *= nr_ioapics; mem = alloc_bootmem(n); res = (void *)mem; - mem += sizeof(struct resource) * num; + mem += sizeof(struct resource) * nr_ioapics; - num = 0; for_each_ioapic(i) { - res[num].name = mem; - res[num].flags = IORESOURCE_MEM | IORESOURCE_BUSY; + res[i].name = mem; + res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY; snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i); mem += IOAPIC_RESOURCE_NAME_SIZE; - ioapics[i].iomem_res = &res[num]; - num++; + ioapics[i].iomem_res = &res[i]; } ioapic_resources = res; diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index f316e34abb42..93edfa01b408 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -101,7 +101,6 @@ static struct apic apic_default = { .get_apic_id = default_get_apic_id, .set_apic_id = NULL, - .apic_id_mask = 0x0F << 24, .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index aca8b75c1552..24170d0809ba 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -270,7 +270,6 @@ static struct apic apic_x2apic_cluster = { .get_apic_id = x2apic_get_apic_id, .set_apic_id = x2apic_set_apic_id, - .apic_id_mask = 0xFFFFFFFFu, .cpu_mask_to_apicid_and = x2apic_cpu_mask_to_apicid_and, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index a1242e2c12e6..4f13f54f1b1f 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -126,7 +126,6 @@ static struct apic apic_x2apic_phys = { .get_apic_id = x2apic_get_apic_id, .set_apic_id = x2apic_set_apic_id, - .apic_id_mask = 0xFFFFFFFFu, .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 29003154fafd..64dd38fbf218 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -582,7 +582,6 @@ static struct apic __refdata apic_x2apic_uv_x = { .get_apic_id = x2apic_get_apic_id, .set_apic_id = set_apic_id, - .apic_id_mask = 0xFFFFFFFFu, .cpu_mask_to_apicid_and = uv_cpu_mask_to_apicid_and, @@ -919,7 +918,7 @@ static void uv_heartbeat(unsigned long ignored) uv_set_scir_bits(bits); /* enable next timer period */ - mod_timer_pinned(timer, jiffies + SCIR_CPU_HB_INTERVAL); + mod_timer(timer, jiffies + SCIR_CPU_HB_INTERVAL); } static void uv_heartbeat_enable(int cpu) @@ -928,7 +927,7 @@ static void uv_heartbeat_enable(int cpu) struct timer_list *timer = &uv_cpu_scir_info(cpu)->timer; uv_set_cpu_scir_bits(cpu, SCIR_CPU_HEARTBEAT|SCIR_CPU_ACTIVITY); - setup_timer(timer, uv_heartbeat, cpu); + setup_pinned_timer(timer, uv_heartbeat, cpu); timer->expires = jiffies + SCIR_CPU_HB_INTERVAL; add_timer_on(timer, cpu); uv_cpu_scir_info(cpu)->enabled = 1; diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 674134e9f5e5..2bd5c6ff7ee7 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -31,7 +31,9 @@ void common(void) { BLANK(); OFFSET(TI_flags, thread_info, flags); OFFSET(TI_status, thread_info, status); - OFFSET(TI_addr_limit, thread_info, addr_limit); + + BLANK(); + OFFSET(TASK_addr_limit, task_struct, thread.addr_limit); BLANK(); OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 0fe6953f421c..d22a7b9c4f0e 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1452,7 +1452,7 @@ void cpu_init(void) struct task_struct *me; struct tss_struct *t; unsigned long v; - int cpu = stack_smp_processor_id(); + int cpu = raw_smp_processor_id(); int i; wait_for_master_cpu(cpu); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 6e2ffbebbcdb..c1a89bc026ac 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -300,15 +300,14 @@ static void intel_workarounds(struct cpuinfo_x86 *c) } /* - * P4 Xeon errata 037 workaround. + * P4 Xeon erratum 037 workaround. * Hardware prefetcher may cause stale data to be loaded into the cache. */ if ((c->x86 == 15) && (c->x86_model == 1) && (c->x86_mask == 1)) { if (msr_set_bit(MSR_IA32_MISC_ENABLE, - MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE_BIT) - > 0) { + MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE_BIT) > 0) { pr_info("CPU: C0 stepping P4 Xeon detected.\n"); - pr_info("CPU: Disabling hardware prefetching (Errata 037)\n"); + pr_info("CPU: Disabling hardware prefetching (Erratum 037)\n"); } } diff --git a/arch/x86/kernel/cpu/mcheck/mce-apei.c b/arch/x86/kernel/cpu/mcheck/mce-apei.c index 34c89a3e8260..83f1a98d37db 100644 --- a/arch/x86/kernel/cpu/mcheck/mce-apei.c +++ b/arch/x86/kernel/cpu/mcheck/mce-apei.c @@ -46,7 +46,7 @@ void apei_mce_report_mem_error(int severity, struct cper_sec_mem_err *mem_err) return; mce_setup(&m); - m.bank = 1; + m.bank = -1; /* Fake a memory read error with unknown channel */ m.status = MCI_STATUS_VAL | MCI_STATUS_EN | MCI_STATUS_ADDRV | 0x9f; diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 92e5e37d97bf..79d8ec849468 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -425,7 +425,7 @@ static u64 mce_rdmsrl(u32 msr) } if (rdmsrl_safe(msr, &v)) { - WARN_ONCE(1, "mce: Unable to read msr %d!\n", msr); + WARN_ONCE(1, "mce: Unable to read MSR 0x%x!\n", msr); /* * Return zero in case the access faulted. This should * not happen normally but can happen if the CPU does @@ -1309,7 +1309,7 @@ static void __restart_timer(struct timer_list *t, unsigned long interval) if (timer_pending(t)) { if (time_before(when, t->expires)) - mod_timer_pinned(t, when); + mod_timer(t, when); } else { t->expires = round_jiffies(when); add_timer_on(t, smp_processor_id()); @@ -1735,7 +1735,7 @@ static void __mcheck_cpu_init_timer(void) struct timer_list *t = this_cpu_ptr(&mce_timer); unsigned int cpu = smp_processor_id(); - setup_timer(t, mce_timer_fn, cpu); + setup_pinned_timer(t, mce_timer_fn, cpu); mce_start_timer(cpu, t); } diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index 10b0661651e0..7b7f3be783d4 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -93,7 +93,7 @@ const char * const amd_df_mcablock_names[] = { EXPORT_SYMBOL_GPL(amd_df_mcablock_names); static DEFINE_PER_CPU(struct threshold_bank **, threshold_banks); -static DEFINE_PER_CPU(unsigned char, bank_map); /* see which banks are on */ +static DEFINE_PER_CPU(unsigned int, bank_map); /* see which banks are on */ static void amd_threshold_interrupt(void); static void amd_deferred_error_interrupt(void); diff --git a/arch/x86/kernel/cpu/rdrand.c b/arch/x86/kernel/cpu/rdrand.c index f6f50c4ceaec..cfa97ff67bda 100644 --- a/arch/x86/kernel/cpu/rdrand.c +++ b/arch/x86/kernel/cpu/rdrand.c @@ -39,9 +39,9 @@ __setup("nordrand", x86_rdrand_setup); */ #define SANITY_CHECK_LOOPS 8 +#ifdef CONFIG_ARCH_RANDOM void x86_init_rdrand(struct cpuinfo_x86 *c) { -#ifdef CONFIG_ARCH_RANDOM unsigned long tmp; int i; @@ -55,5 +55,5 @@ void x86_init_rdrand(struct cpuinfo_x86 *c) return; } } -#endif } +#endif diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 2bb25c3fe2e8..92e8f0a7159c 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -42,16 +42,14 @@ void printk_address(unsigned long address) static void print_ftrace_graph_addr(unsigned long addr, void *data, const struct stacktrace_ops *ops, - struct thread_info *tinfo, int *graph) + struct task_struct *task, int *graph) { - struct task_struct *task; unsigned long ret_addr; int index; if (addr != (unsigned long)return_to_handler) return; - task = tinfo->task; index = task->curr_ret_stack; if (!task->ret_stack || index < *graph) @@ -68,7 +66,7 @@ print_ftrace_graph_addr(unsigned long addr, void *data, static inline void print_ftrace_graph_addr(unsigned long addr, void *data, const struct stacktrace_ops *ops, - struct thread_info *tinfo, int *graph) + struct task_struct *task, int *graph) { } #endif @@ -79,28 +77,36 @@ print_ftrace_graph_addr(unsigned long addr, void *data, * severe exception (double fault, nmi, stack fault, debug, mce) hardware stack */ -static inline int valid_stack_ptr(struct thread_info *tinfo, +static inline int valid_stack_ptr(struct task_struct *task, void *p, unsigned int size, void *end) { - void *t = tinfo; + void *t = task_stack_page(task); if (end) { if (p < end && p >= (end-THREAD_SIZE)) return 1; else return 0; } - return p > t && p < t + THREAD_SIZE - size; + return p >= t && p < t + THREAD_SIZE - size; } unsigned long -print_context_stack(struct thread_info *tinfo, +print_context_stack(struct task_struct *task, unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data, unsigned long *end, int *graph) { struct stack_frame *frame = (struct stack_frame *)bp; - while (valid_stack_ptr(tinfo, stack, sizeof(*stack), end)) { + /* + * If we overflowed the stack into a guard page, jump back to the + * bottom of the usable stack. + */ + if ((unsigned long)task_stack_page(task) - (unsigned long)stack < + PAGE_SIZE) + stack = (unsigned long *)task_stack_page(task); + + while (valid_stack_ptr(task, stack, sizeof(*stack), end)) { unsigned long addr; addr = *stack; @@ -112,7 +118,7 @@ print_context_stack(struct thread_info *tinfo, } else { ops->address(data, addr, 0); } - print_ftrace_graph_addr(addr, data, ops, tinfo, graph); + print_ftrace_graph_addr(addr, data, ops, task, graph); } stack++; } @@ -121,7 +127,7 @@ print_context_stack(struct thread_info *tinfo, EXPORT_SYMBOL_GPL(print_context_stack); unsigned long -print_context_stack_bp(struct thread_info *tinfo, +print_context_stack_bp(struct task_struct *task, unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data, unsigned long *end, int *graph) @@ -129,7 +135,7 @@ print_context_stack_bp(struct thread_info *tinfo, struct stack_frame *frame = (struct stack_frame *)bp; unsigned long *ret_addr = &frame->return_address; - while (valid_stack_ptr(tinfo, ret_addr, sizeof(*ret_addr), end)) { + while (valid_stack_ptr(task, ret_addr, sizeof(*ret_addr), end)) { unsigned long addr = *ret_addr; if (!__kernel_text_address(addr)) @@ -139,7 +145,7 @@ print_context_stack_bp(struct thread_info *tinfo, break; frame = frame->next_frame; ret_addr = &frame->return_address; - print_ftrace_graph_addr(addr, data, ops, tinfo, graph); + print_ftrace_graph_addr(addr, data, ops, task, graph); } return (unsigned long)frame; @@ -199,6 +205,11 @@ void show_stack(struct task_struct *task, unsigned long *sp) show_stack_log_lvl(task, NULL, sp, bp, ""); } +void show_stack_regs(struct pt_regs *regs) +{ + show_stack_log_lvl(current, regs, (unsigned long *)regs->sp, regs->bp, ""); +} + static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED; static int die_owner = -1; static unsigned int die_nest_count; @@ -228,6 +239,8 @@ unsigned long oops_begin(void) EXPORT_SYMBOL_GPL(oops_begin); NOKPROBE_SYMBOL(oops_begin); +void __noreturn rewind_stack_do_exit(int signr); + void oops_end(unsigned long flags, struct pt_regs *regs, int signr) { if (regs && kexec_should_crash(current)) @@ -249,7 +262,13 @@ void oops_end(unsigned long flags, struct pt_regs *regs, int signr) panic("Fatal exception in interrupt"); if (panic_on_oops) panic("Fatal exception"); - do_exit(signr); + + /* + * We're not going to return, but we might be on an IST stack or + * have very little stack space left. Rewind the stack and kill + * the task. + */ + rewind_stack_do_exit(signr); } NOKPROBE_SYMBOL(oops_end); diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c index 464ffd69b92e..948d77da3881 100644 --- a/arch/x86/kernel/dumpstack_32.c +++ b/arch/x86/kernel/dumpstack_32.c @@ -61,15 +61,13 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs, bp = stack_frame(task, regs); for (;;) { - struct thread_info *context; void *end_stack; end_stack = is_hardirq_stack(stack, cpu); if (!end_stack) end_stack = is_softirq_stack(stack, cpu); - context = task_thread_info(task); - bp = ops->walk_stack(context, stack, bp, ops, data, + bp = ops->walk_stack(task, stack, bp, ops, data, end_stack, &graph); /* Stop if not on irq stack */ @@ -98,7 +96,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs, int i; if (sp == NULL) { - if (task) + if (regs) + sp = (unsigned long *)regs->sp; + else if (task) sp = (unsigned long *)task->thread.sp; else sp = (unsigned long *)&sp; diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 5f1c6266eb30..6dede08dd98b 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -153,7 +153,6 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs, const struct stacktrace_ops *ops, void *data) { const unsigned cpu = get_cpu(); - struct thread_info *tinfo; unsigned long *irq_stack = (unsigned long *)per_cpu(irq_stack_ptr, cpu); unsigned long dummy; unsigned used = 0; @@ -179,7 +178,6 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs, * current stack address. If the stacks consist of nested * exceptions */ - tinfo = task_thread_info(task); while (!done) { unsigned long *stack_end; enum stack_type stype; @@ -202,7 +200,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs, if (ops->stack(data, id) < 0) break; - bp = ops->walk_stack(tinfo, stack, bp, ops, + bp = ops->walk_stack(task, stack, bp, ops, data, stack_end, &graph); ops->stack(data, ""); /* @@ -218,7 +216,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs, if (ops->stack(data, "IRQ") < 0) break; - bp = ops->walk_stack(tinfo, stack, bp, + bp = ops->walk_stack(task, stack, bp, ops, data, stack_end, &graph); /* * We link to the next stack (which would be @@ -240,7 +238,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs, /* * This handles the process stack: */ - bp = ops->walk_stack(tinfo, stack, bp, ops, data, NULL, &graph); + bp = ops->walk_stack(task, stack, bp, ops, data, NULL, &graph); put_cpu(); } EXPORT_SYMBOL(dump_trace); @@ -266,7 +264,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs, * back trace for this cpu: */ if (sp == NULL) { - if (task) + if (regs) + sp = (unsigned long *)regs->sp; + else if (task) sp = (unsigned long *)task->thread.sp; else sp = (unsigned long *)&sp; @@ -274,6 +274,8 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs, stack = sp; for (i = 0; i < kstack_depth_to_print; i++) { + unsigned long word; + if (stack >= irq_stack && stack <= irq_stack_end) { if (stack == irq_stack_end) { stack = (unsigned long *) (irq_stack_end[-1]); @@ -283,12 +285,18 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs, if (kstack_end(stack)) break; } + + if (probe_kernel_address(stack, word)) + break; + if ((i % STACKSLOTS_PER_LINE) == 0) { if (i != 0) pr_cont("\n"); - printk("%s %016lx", log_lvl, *stack++); + printk("%s %016lx", log_lvl, word); } else - pr_cont(" %016lx", *stack++); + pr_cont(" %016lx", word); + + stack++; touch_nmi_watchdog(); } preempt_enable(); diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c index bca14c899137..57b71373bae3 100644 --- a/arch/x86/kernel/early-quirks.c +++ b/arch/x86/kernel/early-quirks.c @@ -11,7 +11,11 @@ #include #include +#include +#include #include +#include +#include #include #include #include @@ -21,6 +25,9 @@ #include #include #include +#include + +#define dev_err(msg) pr_err("pci 0000:%02x:%02x.%d: %s", bus, slot, func, msg) static void __init fix_hypertransport_config(int num, int slot, int func) { @@ -75,6 +82,13 @@ static void __init nvidia_bugs(int num, int slot, int func) { #ifdef CONFIG_ACPI #ifdef CONFIG_X86_IO_APIC + /* + * Only applies to Nvidia root ports (bus 0) and not to + * Nvidia graphics cards with PCI ports on secondary buses. + */ + if (num) + return; + /* * All timer overrides on Nvidia are * wrong unless HPET is enabled. @@ -590,6 +604,61 @@ static void __init force_disable_hpet(int num, int slot, int func) #endif } +#define BCM4331_MMIO_SIZE 16384 +#define BCM4331_PM_CAP 0x40 +#define bcma_aread32(reg) ioread32(mmio + 1 * BCMA_CORE_SIZE + reg) +#define bcma_awrite32(reg, val) iowrite32(val, mmio + 1 * BCMA_CORE_SIZE + reg) + +static void __init apple_airport_reset(int bus, int slot, int func) +{ + void __iomem *mmio; + u16 pmcsr; + u64 addr; + int i; + + if (!dmi_match(DMI_SYS_VENDOR, "Apple Inc.")) + return; + + /* Card may have been put into PCI_D3hot by grub quirk */ + pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL); + + if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) { + pmcsr &= ~PCI_PM_CTRL_STATE_MASK; + write_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL, pmcsr); + mdelay(10); + + pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL); + if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) { + dev_err("Cannot power up Apple AirPort card\n"); + return; + } + } + + addr = read_pci_config(bus, slot, func, PCI_BASE_ADDRESS_0); + addr |= (u64)read_pci_config(bus, slot, func, PCI_BASE_ADDRESS_1) << 32; + addr &= PCI_BASE_ADDRESS_MEM_MASK; + + mmio = early_ioremap(addr, BCM4331_MMIO_SIZE); + if (!mmio) { + dev_err("Cannot iomap Apple AirPort card\n"); + return; + } + + pr_info("Resetting Apple AirPort card (left enabled by EFI)\n"); + + for (i = 0; bcma_aread32(BCMA_RESET_ST) && i < 30; i++) + udelay(10); + + bcma_awrite32(BCMA_RESET_CTL, BCMA_RESET_CTL_RESET); + bcma_aread32(BCMA_RESET_CTL); + udelay(1); + + bcma_awrite32(BCMA_RESET_CTL, 0); + bcma_aread32(BCMA_RESET_CTL); + udelay(10); + + early_iounmap(mmio, BCM4331_MMIO_SIZE); +} #define QFLAG_APPLY_ONCE 0x1 #define QFLAG_APPLIED 0x2 @@ -603,12 +672,6 @@ struct chipset { void (*f)(int num, int slot, int func); }; -/* - * Only works for devices on the root bus. If you add any devices - * not on bus 0 readd another loop level in early_quirks(). But - * be careful because at least the Nvidia quirk here relies on - * only matching on bus 0. - */ static struct chipset early_qrk[] __initdata = { { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, nvidia_bugs }, @@ -638,9 +701,13 @@ static struct chipset early_qrk[] __initdata = { */ { PCI_VENDOR_ID_INTEL, 0x0f00, PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, force_disable_hpet}, + { PCI_VENDOR_ID_BROADCOM, 0x4331, + PCI_CLASS_NETWORK_OTHER, PCI_ANY_ID, 0, apple_airport_reset}, {} }; +static void __init early_pci_scan_bus(int bus); + /** * check_dev_quirk - apply early quirks to a given PCI device * @num: bus number @@ -649,7 +716,7 @@ static struct chipset early_qrk[] __initdata = { * * Check the vendor & device ID against the early quirks table. * - * If the device is single function, let early_quirks() know so we don't + * If the device is single function, let early_pci_scan_bus() know so we don't * poke at this device again. */ static int __init check_dev_quirk(int num, int slot, int func) @@ -658,6 +725,7 @@ static int __init check_dev_quirk(int num, int slot, int func) u16 vendor; u16 device; u8 type; + u8 sec; int i; class = read_pci_config_16(num, slot, func, PCI_CLASS_DEVICE); @@ -685,25 +753,36 @@ static int __init check_dev_quirk(int num, int slot, int func) type = read_pci_config_byte(num, slot, func, PCI_HEADER_TYPE); + + if ((type & 0x7f) == PCI_HEADER_TYPE_BRIDGE) { + sec = read_pci_config_byte(num, slot, func, PCI_SECONDARY_BUS); + if (sec > num) + early_pci_scan_bus(sec); + } + if (!(type & 0x80)) return -1; return 0; } -void __init early_quirks(void) +static void __init early_pci_scan_bus(int bus) { int slot, func; - if (!early_pci_allowed()) - return; - /* Poor man's PCI discovery */ - /* Only scan the root bus */ for (slot = 0; slot < 32; slot++) for (func = 0; func < 8; func++) { /* Only probe function 0 on single fn devices */ - if (check_dev_quirk(0, slot, func)) + if (check_dev_quirk(bus, slot, func)) break; } } + +void __init early_quirks(void) +{ + if (!early_pci_allowed()) + return; + + early_pci_scan_bus(0); +} diff --git a/arch/x86/kernel/ebda.c b/arch/x86/kernel/ebda.c index afe65dffee80..4312f8ae71b7 100644 --- a/arch/x86/kernel/ebda.c +++ b/arch/x86/kernel/ebda.c @@ -6,66 +6,92 @@ #include /* + * This function reserves all conventional PC system BIOS related + * firmware memory areas (some of which are data, some of which + * are code), that must not be used by the kernel as available + * RAM. + * * The BIOS places the EBDA/XBDA at the top of conventional * memory, and usually decreases the reported amount of - * conventional memory (int 0x12) too. This also contains a - * workaround for Dell systems that neglect to reserve EBDA. - * The same workaround also avoids a problem with the AMD768MPX - * chipset: reserve a page before VGA to prevent PCI prefetch - * into it (errata #56). Usually the page is reserved anyways, - * unless you have no PS/2 mouse plugged in. + * conventional memory (int 0x12) too. + * + * This means that as a first approximation on most systems we can + * guess the reserved BIOS area by looking at the low BIOS RAM size + * value and assume that everything above that value (up to 1MB) is + * reserved. + * + * But life in firmware country is not that simple: + * + * - This code also contains a quirk for Dell systems that neglect + * to reserve the EBDA area in the 'RAM size' value ... + * + * - The same quirk also avoids a problem with the AMD768MPX + * chipset: reserve a page before VGA to prevent PCI prefetch + * into it (errata #56). (Usually the page is reserved anyways, + * unless you have no PS/2 mouse plugged in.) + * + * - Plus paravirt systems don't have a reliable value in the + * 'BIOS RAM size' pointer we can rely on, so we must quirk + * them too. + * + * Due to those various problems this function is deliberately + * very conservative and tries to err on the side of reserving + * too much, to not risk reserving too little. + * + * Losing a small amount of memory in the bottom megabyte is + * rarely a problem, as long as we have enough memory to install + * the SMP bootup trampoline which *must* be in this area. * - * This functions is deliberately very conservative. Losing - * memory in the bottom megabyte is rarely a problem, as long - * as we have enough memory to install the trampoline. Using - * memory that is in use by the BIOS or by some DMA device - * the BIOS didn't shut down *is* a big problem. + * Using memory that is in use by the BIOS or by some DMA device + * the BIOS didn't shut down *is* a big problem to the kernel, + * obviously. */ -#define BIOS_LOWMEM_KILOBYTES 0x413 -#define LOWMEM_CAP 0x9f000U /* Absolute maximum */ -#define INSANE_CUTOFF 0x20000U /* Less than this = insane */ +#define BIOS_RAM_SIZE_KB_PTR 0x413 -void __init reserve_ebda_region(void) +#define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */ +#define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */ + +void __init reserve_bios_regions(void) { - unsigned int lowmem, ebda_addr; + unsigned int bios_start, ebda_start; /* - * To determine the position of the EBDA and the - * end of conventional memory, we need to look at - * the BIOS data area. In a paravirtual environment - * that area is absent. We'll just have to assume - * that the paravirt case can handle memory setup - * correctly, without our help. + * NOTE: In a paravirtual environment the BIOS reserved + * area is absent. We'll just have to assume that the + * paravirt case can handle memory setup correctly, + * without our help. */ - if (!x86_platform.legacy.ebda_search) + if (!x86_platform.legacy.reserve_bios_regions) return; - /* end of low (conventional) memory */ - lowmem = *(unsigned short *)__va(BIOS_LOWMEM_KILOBYTES); - lowmem <<= 10; - - /* start of EBDA area */ - ebda_addr = get_bios_ebda(); - /* - * Note: some old Dells seem to need 4k EBDA without - * reporting so, so just consider the memory above 0x9f000 - * to be off limits (bugzilla 2990). + * BIOS RAM size is encoded in kilobytes, convert it + * to bytes to get a first guess at where the BIOS + * firmware area starts: */ + bios_start = *(unsigned short *)__va(BIOS_RAM_SIZE_KB_PTR); + bios_start <<= 10; - /* If the EBDA address is below 128K, assume it is bogus */ - if (ebda_addr < INSANE_CUTOFF) - ebda_addr = LOWMEM_CAP; + /* + * If bios_start is less than 128K, assume it is bogus + * and bump it up to 640K. Similarly, if bios_start is above 640K, + * don't trust it. + */ + if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX) + bios_start = BIOS_START_MAX; - /* If lowmem is less than 128K, assume it is bogus */ - if (lowmem < INSANE_CUTOFF) - lowmem = LOWMEM_CAP; + /* Get the start address of the EBDA page: */ + ebda_start = get_bios_ebda(); - /* Use the lower of the lowmem and EBDA markers as the cutoff */ - lowmem = min(lowmem, ebda_addr); - lowmem = min(lowmem, LOWMEM_CAP); /* Absolute cap */ + /* + * If the EBDA start address is sane and is below the BIOS region, + * then also reserve everything from the EBDA start address up to + * the BIOS region. + */ + if (ebda_start >= BIOS_START_MIN && ebda_start < bios_start) + bios_start = ebda_start; - /* reserve all memory between lowmem and the 1MB mark */ - memblock_reserve(lowmem, 0x100000 - lowmem); + /* Reserve all memory between bios_start and the 1MB mark: */ + memblock_reserve(bios_start, 0x100000 - bios_start); } diff --git a/arch/x86/kernel/espfix_64.c b/arch/x86/kernel/espfix_64.c index 4d38416e2a7f..04f89caef9c4 100644 --- a/arch/x86/kernel/espfix_64.c +++ b/arch/x86/kernel/espfix_64.c @@ -57,7 +57,7 @@ # error "Need more than one PGD for the ESPFIX hack" #endif -#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) +#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO) /* This contains the *bottom* address of the espfix stack */ DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack); diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 97027545a72d..3fc03a09a93b 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -8,10 +8,14 @@ #include #include #include +#include #include #include +#define CREATE_TRACE_POINTS +#include + /* * Represents the initial FPU state. It's mostly (but not completely) zeroes, * depending on the FPU hardware format: @@ -192,6 +196,7 @@ void fpu__save(struct fpu *fpu) WARN_ON_FPU(fpu != ¤t->thread.fpu); preempt_disable(); + trace_x86_fpu_before_save(fpu); if (fpu->fpregs_active) { if (!copy_fpregs_to_fpstate(fpu)) { if (use_eager_fpu()) @@ -200,6 +205,7 @@ void fpu__save(struct fpu *fpu) fpregs_deactivate(fpu); } } + trace_x86_fpu_after_save(fpu); preempt_enable(); } EXPORT_SYMBOL_GPL(fpu__save); @@ -222,7 +228,14 @@ void fpstate_init(union fpregs_state *state) return; } - memset(state, 0, xstate_size); + memset(state, 0, fpu_kernel_xstate_size); + + /* + * XRSTORS requires that this bit is set in xcomp_bv, or + * it will #GP. Make sure it is replaced after the memset(). + */ + if (static_cpu_has(X86_FEATURE_XSAVES)) + state->xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT; if (static_cpu_has(X86_FEATURE_FXSR)) fpstate_init_fxstate(&state->fxsave); @@ -247,7 +260,7 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) * leak into the child task: */ if (use_eager_fpu()) - memset(&dst_fpu->state.xsave, 0, xstate_size); + memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size); /* * Save current FPU registers directly into the child @@ -266,7 +279,8 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) */ preempt_disable(); if (!copy_fpregs_to_fpstate(dst_fpu)) { - memcpy(&src_fpu->state, &dst_fpu->state, xstate_size); + memcpy(&src_fpu->state, &dst_fpu->state, + fpu_kernel_xstate_size); if (use_eager_fpu()) copy_kernel_to_fpregs(&src_fpu->state); @@ -275,6 +289,9 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) } preempt_enable(); + trace_x86_fpu_copy_src(src_fpu); + trace_x86_fpu_copy_dst(dst_fpu); + return 0; } @@ -288,7 +305,9 @@ void fpu__activate_curr(struct fpu *fpu) if (!fpu->fpstate_active) { fpstate_init(&fpu->state); + trace_x86_fpu_init_state(fpu); + trace_x86_fpu_activate_state(fpu); /* Safe to do for the current task: */ fpu->fpstate_active = 1; } @@ -314,7 +333,9 @@ void fpu__activate_fpstate_read(struct fpu *fpu) } else { if (!fpu->fpstate_active) { fpstate_init(&fpu->state); + trace_x86_fpu_init_state(fpu); + trace_x86_fpu_activate_state(fpu); /* Safe to do for current and for stopped child tasks: */ fpu->fpstate_active = 1; } @@ -347,7 +368,9 @@ void fpu__activate_fpstate_write(struct fpu *fpu) fpu->last_cpu = -1; } else { fpstate_init(&fpu->state); + trace_x86_fpu_init_state(fpu); + trace_x86_fpu_activate_state(fpu); /* Safe to do for stopped child tasks: */ fpu->fpstate_active = 1; } @@ -432,9 +455,11 @@ void fpu__restore(struct fpu *fpu) /* Avoid __kernel_fpu_begin() right after fpregs_activate() */ kernel_fpu_disable(); + trace_x86_fpu_before_restore(fpu); fpregs_activate(fpu); copy_kernel_to_fpregs(&fpu->state); fpu->counter++; + trace_x86_fpu_after_restore(fpu); kernel_fpu_enable(); } EXPORT_SYMBOL_GPL(fpu__restore); @@ -463,6 +488,8 @@ void fpu__drop(struct fpu *fpu) fpu->fpstate_active = 0; + trace_x86_fpu_dropped(fpu); + preempt_enable(); } diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index aacfd7a82cec..93982aebb398 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -145,8 +145,8 @@ static void __init fpu__init_system_generic(void) * This is inherent to the XSAVE architecture which puts all state * components into a single, continuous memory block: */ -unsigned int xstate_size; -EXPORT_SYMBOL_GPL(xstate_size); +unsigned int fpu_kernel_xstate_size; +EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size); /* Get alignment of the TYPE. */ #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test) @@ -178,7 +178,7 @@ static void __init fpu__init_task_struct_size(void) * Add back the dynamically-calculated register state * size. */ - task_size += xstate_size; + task_size += fpu_kernel_xstate_size; /* * We dynamically size 'struct fpu', so we require that @@ -195,7 +195,7 @@ static void __init fpu__init_task_struct_size(void) } /* - * Set up the xstate_size based on the legacy FPU context size. + * Set up the user and kernel xstate sizes based on the legacy FPU context size. * * We set this up first, and later it will be overwritten by * fpu__init_system_xstate() if the CPU knows about xstates. @@ -208,7 +208,7 @@ static void __init fpu__init_system_xstate_size_legacy(void) on_boot_cpu = 0; /* - * Note that xstate_size might be overwriten later during + * Note that xstate sizes might be overwritten later during * fpu__init_system_xstate(). */ @@ -219,27 +219,17 @@ static void __init fpu__init_system_xstate_size_legacy(void) */ setup_clear_cpu_cap(X86_FEATURE_XSAVE); setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT); - xstate_size = sizeof(struct swregs_state); + fpu_kernel_xstate_size = sizeof(struct swregs_state); } else { if (boot_cpu_has(X86_FEATURE_FXSR)) - xstate_size = sizeof(struct fxregs_state); + fpu_kernel_xstate_size = + sizeof(struct fxregs_state); else - xstate_size = sizeof(struct fregs_state); + fpu_kernel_xstate_size = + sizeof(struct fregs_state); } - /* - * Quirk: we don't yet handle the XSAVES* instructions - * correctly, as we don't correctly convert between - * standard and compacted format when interfacing - * with user-space - so disable it for now. - * - * The difference is small: with recent CPUs the - * compacted format is only marginally smaller than - * the standard FPU state format. - * - * ( This is easy to backport while we are fixing - * XSAVES* support. ) - */ - setup_clear_cpu_cap(X86_FEATURE_XSAVES); + + fpu_user_xstate_size = fpu_kernel_xstate_size; } /* diff --git a/arch/x86/kernel/fpu/regset.c b/arch/x86/kernel/fpu/regset.c index 81422dfb152b..c114b132d121 100644 --- a/arch/x86/kernel/fpu/regset.c +++ b/arch/x86/kernel/fpu/regset.c @@ -4,6 +4,7 @@ #include #include #include +#include /* * The xstateregs_active() routine is the same as the regset_fpregs_active() routine, @@ -85,21 +86,26 @@ int xstateregs_get(struct task_struct *target, const struct user_regset *regset, if (!boot_cpu_has(X86_FEATURE_XSAVE)) return -ENODEV; - fpu__activate_fpstate_read(fpu); - xsave = &fpu->state.xsave; - /* - * Copy the 48bytes defined by the software first into the xstate - * memory layout in the thread struct, so that we can copy the entire - * xstateregs to the user using one user_regset_copyout(). - */ - memcpy(&xsave->i387.sw_reserved, - xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes)); - /* - * Copy the xstate memory layout. - */ - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, xsave, 0, -1); + fpu__activate_fpstate_read(fpu); + + if (using_compacted_format()) { + ret = copyout_from_xsaves(pos, count, kbuf, ubuf, xsave); + } else { + fpstate_sanitize_xstate(fpu); + /* + * Copy the 48 bytes defined by the software into the xsave + * area in the thread struct, so that we can copy the whole + * area to user using one user_regset_copyout(). + */ + memcpy(&xsave->i387.sw_reserved, xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes)); + + /* + * Copy the xstate memory layout. + */ + ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, xsave, 0, -1); + } return ret; } @@ -114,11 +120,27 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset, if (!boot_cpu_has(X86_FEATURE_XSAVE)) return -ENODEV; - fpu__activate_fpstate_write(fpu); + /* + * A whole standard-format XSAVE buffer is needed: + */ + if ((pos != 0) || (count < fpu_user_xstate_size)) + return -EFAULT; xsave = &fpu->state.xsave; - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, xsave, 0, -1); + fpu__activate_fpstate_write(fpu); + + if (boot_cpu_has(X86_FEATURE_XSAVES)) + ret = copyin_to_xsaves(kbuf, ubuf, xsave); + else + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, xsave, 0, -1); + + /* + * In case of failure, mark all states as init: + */ + if (ret) + fpstate_init(&fpu->state); + /* * mxcsr reserved bits must be masked to zero for security reasons. */ diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c index 31c6a60505e6..9e231d88bb33 100644 --- a/arch/x86/kernel/fpu/signal.c +++ b/arch/x86/kernel/fpu/signal.c @@ -8,8 +8,10 @@ #include #include #include +#include #include +#include static struct _fpx_sw_bytes fx_sw_reserved, fx_sw_reserved_ia32; @@ -31,7 +33,7 @@ static inline int check_for_xstate(struct fxregs_state __user *buf, /* Check for the first magic field and other error scenarios. */ if (fx_sw->magic1 != FP_XSTATE_MAGIC1 || fx_sw->xstate_size < min_xstate_size || - fx_sw->xstate_size > xstate_size || + fx_sw->xstate_size > fpu_user_xstate_size || fx_sw->xstate_size > fx_sw->extended_size) return -1; @@ -88,7 +90,8 @@ static inline int save_xstate_epilog(void __user *buf, int ia32_frame) if (!use_xsave()) return err; - err |= __put_user(FP_XSTATE_MAGIC2, (__u32 *)(buf + xstate_size)); + err |= __put_user(FP_XSTATE_MAGIC2, + (__u32 *)(buf + fpu_user_xstate_size)); /* * Read the xfeatures which we copied (directly from the cpu or @@ -125,7 +128,7 @@ static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf) else err = copy_fregs_to_user((struct fregs_state __user *) buf); - if (unlikely(err) && __clear_user(buf, xstate_size)) + if (unlikely(err) && __clear_user(buf, fpu_user_xstate_size)) err = -EFAULT; return err; } @@ -167,7 +170,7 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) sizeof(struct user_i387_ia32_struct), NULL, (struct _fpstate_32 __user *) buf) ? -1 : 1; - if (fpregs_active()) { + if (fpregs_active() || using_compacted_format()) { /* Save the live register state to the user directly. */ if (copy_fpregs_to_sigframe(buf_fx)) return -1; @@ -175,8 +178,19 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) if (ia32_fxstate) copy_fxregs_to_kernel(&tsk->thread.fpu); } else { + /* + * It is a *bug* if kernel uses compacted-format for xsave + * area and we copy it out directly to a signal frame. It + * should have been handled above by saving the registers + * directly. + */ + if (boot_cpu_has(X86_FEATURE_XSAVES)) { + WARN_ONCE(1, "x86/fpu: saving compacted-format xsave area to a signal frame!\n"); + return -1; + } + fpstate_sanitize_xstate(&tsk->thread.fpu); - if (__copy_to_user(buf_fx, xsave, xstate_size)) + if (__copy_to_user(buf_fx, xsave, fpu_user_xstate_size)) return -1; } @@ -250,7 +264,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) int ia32_fxstate = (buf != buf_fx); struct task_struct *tsk = current; struct fpu *fpu = &tsk->thread.fpu; - int state_size = xstate_size; + int state_size = fpu_kernel_xstate_size; u64 xfeatures = 0; int fx_only = 0; @@ -282,6 +296,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) */ state_size = sizeof(struct fxregs_state); fx_only = 1; + trace_x86_fpu_xstate_check_failed(fpu); } else { state_size = fx_sw_user.xstate_size; xfeatures = fx_sw_user.xfeatures; @@ -308,9 +323,17 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) */ fpu__drop(fpu); - if (__copy_from_user(&fpu->state.xsave, buf_fx, state_size) || - __copy_from_user(&env, buf, sizeof(env))) { + if (using_compacted_format()) { + err = copyin_to_xsaves(NULL, buf_fx, + &fpu->state.xsave); + } else { + err = __copy_from_user(&fpu->state.xsave, + buf_fx, state_size); + } + + if (err || __copy_from_user(&env, buf, sizeof(env))) { fpstate_init(&fpu->state); + trace_x86_fpu_init_state(fpu); err = -1; } else { sanitize_restored_xstate(tsk, &env, xfeatures, fx_only); @@ -341,7 +364,8 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) static inline int xstate_sigframe_size(void) { - return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size; + return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE : + fpu_user_xstate_size; } /* @@ -385,12 +409,12 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame, */ void fpu__init_prepare_fx_sw_frame(void) { - int size = xstate_size + FP_XSTATE_MAGIC2_SIZE; + int size = fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE; fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1; fx_sw_reserved.extended_size = size; fx_sw_reserved.xfeatures = xfeatures_mask; - fx_sw_reserved.xstate_size = xstate_size; + fx_sw_reserved.xstate_size = fpu_user_xstate_size; if (config_enabled(CONFIG_IA32_EMULATION) || config_enabled(CONFIG_X86_32)) { diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 4ea2a59483c7..680049aa4593 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -43,6 +44,13 @@ static unsigned int xstate_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = static unsigned int xstate_sizes[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1}; static unsigned int xstate_comp_offsets[sizeof(xfeatures_mask)*8]; +/* + * The XSAVE area of kernel can be in standard or compacted format; + * it is always in standard format for user mode. This is the user + * mode standard format size used for signal and ptrace frames. + */ +unsigned int fpu_user_xstate_size; + /* * Clear all of the X86_FEATURE_* bits that are unavailable * when the CPU has no XSAVE support. @@ -105,6 +113,27 @@ int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) } EXPORT_SYMBOL_GPL(cpu_has_xfeatures); +static int xfeature_is_supervisor(int xfeature_nr) +{ + /* + * We currently do not support supervisor states, but if + * we did, we could find out like this. + * + * SDM says: If state component 'i' is a user state component, + * ECX[0] return 0; if state component i is a supervisor + * state component, ECX[0] returns 1. + */ + u32 eax, ebx, ecx, edx; + + cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); + return !!(ecx & 1); +} + +static int xfeature_is_user(int xfeature_nr) +{ + return !xfeature_is_supervisor(xfeature_nr); +} + /* * When executing XSAVEOPT (or other optimized XSAVE instructions), if * a processor implementation detects that an FPU state component is still @@ -171,7 +200,7 @@ void fpstate_sanitize_xstate(struct fpu *fpu) */ while (xfeatures) { if (xfeatures & 0x1) { - int offset = xstate_offsets[feature_bit]; + int offset = xstate_comp_offsets[feature_bit]; int size = xstate_sizes[feature_bit]; memcpy((void *)fx + offset, @@ -192,6 +221,15 @@ void fpu__init_cpu_xstate(void) { if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask) return; + /* + * Make it clear that XSAVES supervisor states are not yet + * implemented should anyone expect it to work by changing + * bits in XFEATURE_MASK_* macros and XCR0. + */ + WARN_ONCE((xfeatures_mask & XFEATURE_MASK_SUPERVISOR), + "x86/fpu: XSAVES supervisor states are not yet implemented.\n"); + + xfeatures_mask &= ~XFEATURE_MASK_SUPERVISOR; cr4_set_bits(X86_CR4_OSXSAVE); xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask); @@ -217,13 +255,29 @@ static void __init setup_xstate_features(void) /* start at the beginnning of the "extended state" */ unsigned int last_good_offset = offsetof(struct xregs_state, extended_state_area); + /* + * The FP xstates and SSE xstates are legacy states. They are always + * in the fixed offsets in the xsave area in either compacted form + * or standard form. + */ + xstate_offsets[0] = 0; + xstate_sizes[0] = offsetof(struct fxregs_state, xmm_space); + xstate_offsets[1] = xstate_sizes[0]; + xstate_sizes[1] = FIELD_SIZEOF(struct fxregs_state, xmm_space); for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { if (!xfeature_enabled(i)) continue; cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); - xstate_offsets[i] = ebx; + + /* + * If an xfeature is supervisor state, the offset + * in EBX is invalid. We leave it to -1. + */ + if (xfeature_is_user(i)) + xstate_offsets[i] = ebx; + xstate_sizes[i] = eax; /* * In our xstate size checks, we assume that the @@ -233,8 +287,6 @@ static void __init setup_xstate_features(void) WARN_ONCE(last_good_offset > xstate_offsets[i], "x86/fpu: misordered xstate at %d\n", last_good_offset); last_good_offset = xstate_offsets[i]; - - printk(KERN_INFO "x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", i, ebx, i, eax); } } @@ -262,6 +314,33 @@ static void __init print_xstate_features(void) print_xstate_feature(XFEATURE_MASK_PKRU); } +/* + * This check is important because it is easy to get XSTATE_* + * confused with XSTATE_BIT_*. + */ +#define CHECK_XFEATURE(nr) do { \ + WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \ + WARN_ON(nr >= XFEATURE_MAX); \ +} while (0) + +/* + * We could cache this like xstate_size[], but we only use + * it here, so it would be a waste of space. + */ +static int xfeature_is_aligned(int xfeature_nr) +{ + u32 eax, ebx, ecx, edx; + + CHECK_XFEATURE(xfeature_nr); + cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); + /* + * The value returned by ECX[1] indicates the alignment + * of state component 'i' when the compacted format + * of the extended region of an XSAVE area is used: + */ + return !!(ecx & 2); +} + /* * This function sets up offsets and sizes of all extended states in * xsave area. This supports both standard format and compacted format @@ -299,10 +378,29 @@ static void __init setup_xstate_comp(void) else xstate_comp_sizes[i] = 0; - if (i > FIRST_EXTENDED_XFEATURE) + if (i > FIRST_EXTENDED_XFEATURE) { xstate_comp_offsets[i] = xstate_comp_offsets[i-1] + xstate_comp_sizes[i-1]; + if (xfeature_is_aligned(i)) + xstate_comp_offsets[i] = + ALIGN(xstate_comp_offsets[i], 64); + } + } +} + +/* + * Print out xstate component offsets and sizes + */ +static void __init print_xstate_offset_size(void) +{ + int i; + + for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { + if (!xfeature_enabled(i)) + continue; + pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", + i, xstate_comp_offsets[i], i, xstate_sizes[i]); } } @@ -322,13 +420,11 @@ static void __init setup_init_fpu_buf(void) setup_xstate_features(); print_xstate_features(); - if (boot_cpu_has(X86_FEATURE_XSAVES)) { + if (boot_cpu_has(X86_FEATURE_XSAVES)) init_fpstate.xsave.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask; - init_fpstate.xsave.header.xfeatures = xfeatures_mask; - } /* - * Init all the features state with header_bv being 0x0 + * Init all the features state with header.xfeatures being 0x0 */ copy_kernel_to_xregs_booting(&init_fpstate.xsave); @@ -339,58 +435,19 @@ static void __init setup_init_fpu_buf(void) copy_xregs_to_kernel_booting(&init_fpstate.xsave); } -static int xfeature_is_supervisor(int xfeature_nr) -{ - /* - * We currently do not support supervisor states, but if - * we did, we could find out like this. - * - * SDM says: If state component i is a user state component, - * ECX[0] return 0; if state component i is a supervisor - * state component, ECX[0] returns 1. - u32 eax, ebx, ecx, edx; - cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx; - return !!(ecx & 1); - */ - return 0; -} -/* -static int xfeature_is_user(int xfeature_nr) -{ - return !xfeature_is_supervisor(xfeature_nr); -} -*/ - -/* - * This check is important because it is easy to get XSTATE_* - * confused with XSTATE_BIT_*. - */ -#define CHECK_XFEATURE(nr) do { \ - WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \ - WARN_ON(nr >= XFEATURE_MAX); \ -} while (0) - -/* - * We could cache this like xstate_size[], but we only use - * it here, so it would be a waste of space. - */ -static int xfeature_is_aligned(int xfeature_nr) +static int xfeature_uncompacted_offset(int xfeature_nr) { u32 eax, ebx, ecx, edx; - CHECK_XFEATURE(xfeature_nr); - cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); /* - * The value returned by ECX[1] indicates the alignment - * of state component i when the compacted format - * of the extended region of an XSAVE area is used + * Only XSAVES supports supervisor states and it uses compacted + * format. Checking a supervisor state's uncompacted offset is + * an error. */ - return !!(ecx & 2); -} - -static int xfeature_uncompacted_offset(int xfeature_nr) -{ - u32 eax, ebx, ecx, edx; + if (XFEATURE_MASK_SUPERVISOR & (1 << xfeature_nr)) { + WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr); + return -1; + } CHECK_XFEATURE(xfeature_nr); cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); @@ -415,7 +472,7 @@ static int xfeature_size(int xfeature_nr) * that it is obvious which aspect of 'XSAVES' is being handled * by the calling code. */ -static int using_compacted_format(void) +int using_compacted_format(void) { return boot_cpu_has(X86_FEATURE_XSAVES); } @@ -530,11 +587,12 @@ static void do_extra_xstate_size_checks(void) */ paranoid_xstate_size += xfeature_size(i); } - XSTATE_WARN_ON(paranoid_xstate_size != xstate_size); + XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size); } + /* - * Calculate total size of enabled xstates in XCR0/xfeatures_mask. + * Get total size of enabled xstates in XCR0/xfeatures_mask. * * Note the SDM's wording here. "sub-function 0" only enumerates * the size of the *user* states. If we use it to size a buffer @@ -544,34 +602,33 @@ static void do_extra_xstate_size_checks(void) * Note that we do not currently set any bits on IA32_XSS so * 'XCR0 | IA32_XSS == XCR0' for now. */ -static unsigned int __init calculate_xstate_size(void) +static unsigned int __init get_xsaves_size(void) { unsigned int eax, ebx, ecx, edx; - unsigned int calculated_xstate_size; + /* + * - CPUID function 0DH, sub-function 1: + * EBX enumerates the size (in bytes) required by + * the XSAVES instruction for an XSAVE area + * containing all the state components + * corresponding to bits currently set in + * XCR0 | IA32_XSS. + */ + cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); + return ebx; +} - if (!boot_cpu_has(X86_FEATURE_XSAVES)) { - /* - * - CPUID function 0DH, sub-function 0: - * EBX enumerates the size (in bytes) required by - * the XSAVE instruction for an XSAVE area - * containing all the *user* state components - * corresponding to bits currently set in XCR0. - */ - cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); - calculated_xstate_size = ebx; - } else { - /* - * - CPUID function 0DH, sub-function 1: - * EBX enumerates the size (in bytes) required by - * the XSAVES instruction for an XSAVE area - * containing all the state components - * corresponding to bits currently set in - * XCR0 | IA32_XSS. - */ - cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); - calculated_xstate_size = ebx; - } - return calculated_xstate_size; +static unsigned int __init get_xsave_size(void) +{ + unsigned int eax, ebx, ecx, edx; + /* + * - CPUID function 0DH, sub-function 0: + * EBX enumerates the size (in bytes) required by + * the XSAVE instruction for an XSAVE area + * containing all the *user* state components + * corresponding to bits currently set in XCR0. + */ + cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); + return ebx; } /* @@ -591,7 +648,15 @@ static bool is_supported_xstate_size(unsigned int test_xstate_size) static int init_xstate_size(void) { /* Recompute the context size for enabled features: */ - unsigned int possible_xstate_size = calculate_xstate_size(); + unsigned int possible_xstate_size; + unsigned int xsave_size; + + xsave_size = get_xsave_size(); + + if (boot_cpu_has(X86_FEATURE_XSAVES)) + possible_xstate_size = get_xsaves_size(); + else + possible_xstate_size = xsave_size; /* Ensure we have the space to store all enabled: */ if (!is_supported_xstate_size(possible_xstate_size)) @@ -601,8 +666,13 @@ static int init_xstate_size(void) * The size is OK, we are definitely going to use xsave, * make it known to the world that we need more space. */ - xstate_size = possible_xstate_size; + fpu_kernel_xstate_size = possible_xstate_size; do_extra_xstate_size_checks(); + + /* + * User space is always in standard format. + */ + fpu_user_xstate_size = xsave_size; return 0; } @@ -644,8 +714,13 @@ void __init fpu__init_system_xstate(void) xfeatures_mask = eax + ((u64)edx << 32); if ((xfeatures_mask & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { + /* + * This indicates that something really unexpected happened + * with the enumeration. Disable XSAVE and try to continue + * booting without it. This is too early to BUG(). + */ pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask); - BUG(); + goto out_disable; } xfeatures_mask &= fpu__get_supported_xfeatures_mask(); @@ -653,21 +728,29 @@ void __init fpu__init_system_xstate(void) /* Enable xstate instructions to be able to continue with initialization: */ fpu__init_cpu_xstate(); err = init_xstate_size(); - if (err) { - /* something went wrong, boot without any XSAVE support */ - fpu__init_disable_system_xstate(); - return; - } + if (err) + goto out_disable; + + /* + * Update info used for ptrace frames; use standard-format size and no + * supervisor xstates: + */ + update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask & ~XFEATURE_MASK_SUPERVISOR); - update_regset_xstate_info(xstate_size, xfeatures_mask); fpu__init_prepare_fx_sw_frame(); setup_init_fpu_buf(); setup_xstate_comp(); + print_xstate_offset_size(); pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", xfeatures_mask, - xstate_size, + fpu_kernel_xstate_size, boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard"); + return; + +out_disable: + /* something went wrong, try to boot without any XSAVE support */ + fpu__init_disable_system_xstate(); } /* @@ -693,6 +776,11 @@ void *__raw_xsave_addr(struct xregs_state *xsave, int xstate_feature_mask) { int feature_nr = fls64(xstate_feature_mask) - 1; + if (!xfeature_enabled(feature_nr)) { + WARN_ON_FPU(1); + return NULL; + } + return (void *)xsave + xstate_comp_offsets[feature_nr]; } /* @@ -887,16 +975,16 @@ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, if (!boot_cpu_has(X86_FEATURE_OSPKE)) return -EINVAL; - /* Set the bits we need in PKRU */ + /* Set the bits we need in PKRU: */ if (init_val & PKEY_DISABLE_ACCESS) new_pkru_bits |= PKRU_AD_BIT; if (init_val & PKEY_DISABLE_WRITE) new_pkru_bits |= PKRU_WD_BIT; - /* Shift the bits in to the correct place in PKRU for pkey. */ + /* Shift the bits in to the correct place in PKRU for pkey: */ new_pkru_bits <<= pkey_shift; - /* Locate old copy of the state in the xsave buffer */ + /* Locate old copy of the state in the xsave buffer: */ old_pkru_state = get_xsave_addr(xsave, XFEATURE_MASK_PKRU); /* @@ -909,9 +997,10 @@ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, else new_pkru_state.pkru = old_pkru_state->pkru; - /* mask off any old bits in place */ + /* Mask off any old bits in place: */ new_pkru_state.pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift); - /* Set the newly-requested bits */ + + /* Set the newly-requested bits: */ new_pkru_state.pkru |= new_pkru_bits; /* @@ -925,8 +1014,168 @@ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, */ new_pkru_state.pad = 0; - fpu__xfeature_set_state(XFEATURE_MASK_PKRU, &new_pkru_state, - sizeof(new_pkru_state)); + fpu__xfeature_set_state(XFEATURE_MASK_PKRU, &new_pkru_state, sizeof(new_pkru_state)); + + return 0; +} + +/* + * This is similar to user_regset_copyout(), but will not add offset to + * the source data pointer or increment pos, count, kbuf, and ubuf. + */ +static inline int xstate_copyout(unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf, + const void *data, const int start_pos, + const int end_pos) +{ + if ((count == 0) || (pos < start_pos)) + return 0; + + if (end_pos < 0 || pos < end_pos) { + unsigned int copy = (end_pos < 0 ? count : min(count, end_pos - pos)); + + if (kbuf) { + memcpy(kbuf + pos, data, copy); + } else { + if (__copy_to_user(ubuf + pos, data, copy)) + return -EFAULT; + } + } + return 0; +} + +/* + * Convert from kernel XSAVES compacted format to standard format and copy + * to a ptrace buffer. It supports partial copy but pos always starts from + * zero. This is called from xstateregs_get() and there we check the CPU + * has XSAVES. + */ +int copyout_from_xsaves(unsigned int pos, unsigned int count, void *kbuf, + void __user *ubuf, struct xregs_state *xsave) +{ + unsigned int offset, size; + int ret, i; + struct xstate_header header; + + /* + * Currently copy_regset_to_user() starts from pos 0: + */ + if (unlikely(pos != 0)) + return -EFAULT; + + /* + * The destination is a ptrace buffer; we put in only user xstates: + */ + memset(&header, 0, sizeof(header)); + header.xfeatures = xsave->header.xfeatures; + header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR; + + /* + * Copy xregs_state->header: + */ + offset = offsetof(struct xregs_state, header); + size = sizeof(header); + + ret = xstate_copyout(offset, size, kbuf, ubuf, &header, 0, count); + + if (ret) + return ret; + + for (i = 0; i < XFEATURE_MAX; i++) { + /* + * Copy only in-use xstates: + */ + if ((header.xfeatures >> i) & 1) { + void *src = __raw_xsave_addr(xsave, 1 << i); + + offset = xstate_offsets[i]; + size = xstate_sizes[i]; + + ret = xstate_copyout(offset, size, kbuf, ubuf, src, 0, count); + + if (ret) + return ret; + + if (offset + size >= count) + break; + } + + } + + /* + * Fill xsave->i387.sw_reserved value for ptrace frame: + */ + offset = offsetof(struct fxregs_state, sw_reserved); + size = sizeof(xstate_fx_sw_bytes); + + ret = xstate_copyout(offset, size, kbuf, ubuf, xstate_fx_sw_bytes, 0, count); + + if (ret) + return ret; + + return 0; +} + +/* + * Convert from a ptrace standard-format buffer to kernel XSAVES format + * and copy to the target thread. This is called from xstateregs_set() and + * there we check the CPU has XSAVES and a whole standard-sized buffer + * exists. + */ +int copyin_to_xsaves(const void *kbuf, const void __user *ubuf, + struct xregs_state *xsave) +{ + unsigned int offset, size; + int i; + u64 xfeatures; + u64 allowed_features; + + offset = offsetof(struct xregs_state, header); + size = sizeof(xfeatures); + + if (kbuf) { + memcpy(&xfeatures, kbuf + offset, size); + } else { + if (__copy_from_user(&xfeatures, ubuf + offset, size)) + return -EFAULT; + } + + /* + * Reject if the user sets any disabled or supervisor features: + */ + allowed_features = xfeatures_mask & ~XFEATURE_MASK_SUPERVISOR; + + if (xfeatures & ~allowed_features) + return -EINVAL; + + for (i = 0; i < XFEATURE_MAX; i++) { + u64 mask = ((u64)1 << i); + + if (xfeatures & mask) { + void *dst = __raw_xsave_addr(xsave, 1 << i); + + offset = xstate_offsets[i]; + size = xstate_sizes[i]; + + if (kbuf) { + memcpy(dst, kbuf + offset, size); + } else { + if (__copy_from_user(dst, ubuf + offset, size)) + return -EFAULT; + } + } + } + + /* + * The state that came in from userspace was user-state only. + * Mask all the user states out of 'xfeatures': + */ + xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR; + + /* + * Add back in the features that came in from userspace: + */ + xsave->header.xfeatures |= xfeatures; return 0; } diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c index d784bb547a9d..2dda0bc4576e 100644 --- a/arch/x86/kernel/head32.c +++ b/arch/x86/kernel/head32.c @@ -26,7 +26,7 @@ static void __init i386_default_early_setup(void) x86_init.resources.reserve_resources = i386_reserve_resources; x86_init.mpparse.setup_ioapic_ids = setup_ioapic_ids_from_mpc; - reserve_ebda_region(); + reserve_bios_regions(); } asmlinkage __visible void __init i386_start_kernel(void) diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index b72fb0b71dd1..99d48e7d2974 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -183,7 +183,7 @@ void __init x86_64_start_reservations(char *real_mode_data) copy_bootdata(__va(real_mode_data)); x86_early_init_platform_quirks(); - reserve_ebda_region(); + reserve_bios_regions(); switch (boot_params.hdr.hardware_subarch) { case X86_SUBARCH_INTEL_MID: diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 5df831ef1442..9f8efc9f0075 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -38,7 +38,7 @@ #define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) -L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET) +L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET_BASE) L4_START_KERNEL = pgd_index(__START_KERNEL_map) L3_START_KERNEL = pud_index(__START_KERNEL_map) @@ -299,6 +299,7 @@ ENTRY(secondary_startup_64) pushq $__KERNEL_CS # set correct cs pushq %rax # target address in negative space lretq +ENDPROC(secondary_startup_64) #include "verify_cpu.S" diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c index 64341aa485ae..d40ee8a38fed 100644 --- a/arch/x86/kernel/i386_ksyms_32.c +++ b/arch/x86/kernel/i386_ksyms_32.c @@ -42,3 +42,5 @@ EXPORT_SYMBOL(empty_zero_page); EXPORT_SYMBOL(___preempt_schedule); EXPORT_SYMBOL(___preempt_schedule_notrace); #endif + +EXPORT_SYMBOL(__sw_hweight32); diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index 38da8f29a9c8..c627bf8d98ad 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -130,11 +130,9 @@ void irq_ctx_init(int cpu) void do_softirq_own_stack(void) { - struct thread_info *curstk; struct irq_stack *irqstk; u32 *isp, *prev_esp; - curstk = current_stack(); irqstk = __this_cpu_read(softirq_stack); /* build the stack frame on the softirq stack */ diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index 38cf7a741250..7847e5c0e0b5 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -961,7 +961,19 @@ int kprobe_fault_handler(struct pt_regs *regs, int trapnr) * normal page fault. */ regs->ip = (unsigned long)cur->addr; + /* + * Trap flag (TF) has been set here because this fault + * happened where the single stepping will be done. + * So clear it by resetting the current kprobe: + */ + regs->flags &= ~X86_EFLAGS_TF; + + /* + * If the TF flag was set before the kprobe hit, + * don't touch it: + */ regs->flags |= kcb->kprobe_old_flags; + if (kcb->kprobe_status == KPROBE_REENTER) restore_previous_kprobe(kcb); else diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index eea2a6f72b31..1ef5e48b3a36 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -301,8 +301,6 @@ static void kvm_register_steal_time(void) if (!has_steal_clock) return; - memset(st, 0, sizeof(*st)); - wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED)); pr_info("kvm-stealtime: cpu %d, msr %llx\n", cpu, (unsigned long long) slow_virt_to_phys(st)); diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index b2f8a33b36ff..24a50301f150 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -7,12 +7,12 @@ void __init x86_early_init_platform_quirks(void) { x86_platform.legacy.rtc = 1; - x86_platform.legacy.ebda_search = 0; + x86_platform.legacy.reserve_bios_regions = 0; x86_platform.legacy.devices.pnpbios = 1; switch (boot_params.hdr.hardware_subarch) { case X86_SUBARCH_PC: - x86_platform.legacy.ebda_search = 1; + x86_platform.legacy.reserve_bios_regions = 1; break; case X86_SUBARCH_XEN: case X86_SUBARCH_LGUEST: diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c index 99bfc025111d..06c58ce46762 100644 --- a/arch/x86/kernel/pvclock.c +++ b/arch/x86/kernel/pvclock.c @@ -61,11 +61,16 @@ void pvclock_resume(void) u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src) { unsigned version; - cycle_t ret; u8 flags; do { - version = __pvclock_read_cycles(src, &ret, &flags); + version = src->version; + /* Make the latest version visible */ + smp_rmb(); + + flags = src->flags; + /* Make sure that the version double-check is last. */ + smp_rmb(); } while ((src->version & 1) || version != src->version); return flags & valid_flags; @@ -80,6 +85,8 @@ cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src) do { version = __pvclock_read_cycles(src, &ret, &flags); + /* Make sure that the version double-check is last. */ + smp_rmb(); } while ((src->version & 1) || version != src->version); if (unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) { diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index a9b31eb815f2..15ed70f8278b 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -54,6 +54,19 @@ bool port_cf9_safe = false; * Dell Inc. so their systems "just work". :-) */ +/* + * Some machines require the "reboot=a" commandline options + */ +static int __init set_acpi_reboot(const struct dmi_system_id *d) +{ + if (reboot_type != BOOT_ACPI) { + reboot_type = BOOT_ACPI; + pr_info("%s series board detected. Selecting %s-method for reboots.\n", + d->ident, "ACPI"); + } + return 0; +} + /* * Some machines require the "reboot=b" or "reboot=k" commandline options, * this quirk makes that automatic. @@ -395,6 +408,14 @@ static struct dmi_system_id __initdata reboot_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Dell XPS710"), }, }, + { /* Handle problems with rebooting on Dell Optiplex 7450 AIO */ + .callback = set_acpi_reboot, + .ident = "Dell OptiPlex 7450 AIO", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 7450 AIO"), + }, + }, /* Hewlett-Packard */ { /* Handle problems with rebooting on HP laptops */ diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index c4e7b3991b60..6cb2b02fcc87 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -113,6 +113,7 @@ #include #include #include +#include /* * max_low_pfn_mapped: highest direct mapped pfn under 4GB @@ -399,10 +400,6 @@ static void __init reserve_initrd(void) memblock_free(ramdisk_image, ramdisk_end - ramdisk_image); } -static void __init early_initrd_acpi_init(void) -{ - early_acpi_table_init((void *)initrd_start, initrd_end - initrd_start); -} #else static void __init early_reserve_initrd(void) { @@ -410,9 +407,6 @@ static void __init early_reserve_initrd(void) static void __init reserve_initrd(void) { } -static void __init early_initrd_acpi_init(void) -{ -} #endif /* CONFIG_BLK_DEV_INITRD */ static void __init parse_setup_data(void) @@ -942,6 +936,8 @@ void __init setup_arch(char **cmdline_p) x86_init.oem.arch_setup(); + kernel_randomize_memory(); + iomem_resource.end = (1ULL << boot_cpu_data.x86_phys_bits) - 1; setup_memory_map(); parse_setup_data(); @@ -1146,7 +1142,7 @@ void __init setup_arch(char **cmdline_p) reserve_initrd(); - early_initrd_acpi_init(); + acpi_table_upgrade(); vsmp_init(); diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index e4fcb87ba7a6..7a40e068302d 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -236,6 +236,8 @@ void __init setup_per_cpu_areas(void) early_per_cpu_map(x86_cpu_to_apicid, cpu); per_cpu(x86_bios_cpu_apicid, cpu) = early_per_cpu_map(x86_bios_cpu_apicid, cpu); + per_cpu(x86_cpu_to_acpiid, cpu) = + early_per_cpu_map(x86_cpu_to_acpiid, cpu); #endif #ifdef CONFIG_X86_32 per_cpu(x86_cpu_to_logical_apicid, cpu) = @@ -271,6 +273,7 @@ void __init setup_per_cpu_areas(void) #ifdef CONFIG_X86_LOCAL_APIC early_per_cpu_ptr(x86_cpu_to_apicid) = NULL; early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL; + early_per_cpu_ptr(x86_cpu_to_acpiid) = NULL; #endif #ifdef CONFIG_X86_32 early_per_cpu_ptr(x86_cpu_to_logical_apicid) = NULL; diff --git a/arch/x86/kernel/signal_compat.c b/arch/x86/kernel/signal_compat.c index dc3c0b1c816f..b44564bf86a8 100644 --- a/arch/x86/kernel/signal_compat.c +++ b/arch/x86/kernel/signal_compat.c @@ -1,11 +1,104 @@ #include #include +/* + * The compat_siginfo_t structure and handing code is very easy + * to break in several ways. It must always be updated when new + * updates are made to the main siginfo_t, and + * copy_siginfo_to_user32() must be updated when the + * (arch-independent) copy_siginfo_to_user() is updated. + * + * It is also easy to put a new member in the compat_siginfo_t + * which has implicit alignment which can move internal structure + * alignment around breaking the ABI. This can happen if you, + * for instance, put a plain 64-bit value in there. + */ +static inline void signal_compat_build_tests(void) +{ + int _sifields_offset = offsetof(compat_siginfo_t, _sifields); + + /* + * If adding a new si_code, there is probably new data in + * the siginfo. Make sure folks bumping the si_code + * limits also have to look at this code. Make sure any + * new fields are handled in copy_siginfo_to_user32()! + */ + BUILD_BUG_ON(NSIGILL != 8); + BUILD_BUG_ON(NSIGFPE != 8); + BUILD_BUG_ON(NSIGSEGV != 4); + BUILD_BUG_ON(NSIGBUS != 5); + BUILD_BUG_ON(NSIGTRAP != 4); + BUILD_BUG_ON(NSIGCHLD != 6); + BUILD_BUG_ON(NSIGSYS != 1); + + /* This is part of the ABI and can never change in size: */ + BUILD_BUG_ON(sizeof(compat_siginfo_t) != 128); + /* + * The offsets of all the (unioned) si_fields are fixed + * in the ABI, of course. Make sure none of them ever + * move and are always at the beginning: + */ + BUILD_BUG_ON(offsetof(compat_siginfo_t, _sifields) != 3 * sizeof(int)); +#define CHECK_CSI_OFFSET(name) BUILD_BUG_ON(_sifields_offset != offsetof(compat_siginfo_t, _sifields.name)) + + /* + * Ensure that the size of each si_field never changes. + * If it does, it is a sign that the + * copy_siginfo_to_user32() code below needs to updated + * along with the size in the CHECK_SI_SIZE(). + * + * We repeat this check for both the generic and compat + * siginfos. + * + * Note: it is OK for these to grow as long as the whole + * structure stays within the padding size (checked + * above). + */ +#define CHECK_CSI_SIZE(name, size) BUILD_BUG_ON(size != sizeof(((compat_siginfo_t *)0)->_sifields.name)) +#define CHECK_SI_SIZE(name, size) BUILD_BUG_ON(size != sizeof(((siginfo_t *)0)->_sifields.name)) + + CHECK_CSI_OFFSET(_kill); + CHECK_CSI_SIZE (_kill, 2*sizeof(int)); + CHECK_SI_SIZE (_kill, 2*sizeof(int)); + + CHECK_CSI_OFFSET(_timer); + CHECK_CSI_SIZE (_timer, 5*sizeof(int)); + CHECK_SI_SIZE (_timer, 6*sizeof(int)); + + CHECK_CSI_OFFSET(_rt); + CHECK_CSI_SIZE (_rt, 3*sizeof(int)); + CHECK_SI_SIZE (_rt, 4*sizeof(int)); + + CHECK_CSI_OFFSET(_sigchld); + CHECK_CSI_SIZE (_sigchld, 5*sizeof(int)); + CHECK_SI_SIZE (_sigchld, 8*sizeof(int)); + + CHECK_CSI_OFFSET(_sigchld_x32); + CHECK_CSI_SIZE (_sigchld_x32, 7*sizeof(int)); + /* no _sigchld_x32 in the generic siginfo_t */ + + CHECK_CSI_OFFSET(_sigfault); + CHECK_CSI_SIZE (_sigfault, 4*sizeof(int)); + CHECK_SI_SIZE (_sigfault, 8*sizeof(int)); + + CHECK_CSI_OFFSET(_sigpoll); + CHECK_CSI_SIZE (_sigpoll, 2*sizeof(int)); + CHECK_SI_SIZE (_sigpoll, 4*sizeof(int)); + + CHECK_CSI_OFFSET(_sigsys); + CHECK_CSI_SIZE (_sigsys, 3*sizeof(int)); + CHECK_SI_SIZE (_sigsys, 4*sizeof(int)); + + /* any new si_fields should be added here */ +} + int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from) { int err = 0; bool ia32 = test_thread_flag(TIF_IA32); + signal_compat_build_tests(); + if (!access_ok(VERIFY_WRITE, to, sizeof(compat_siginfo_t))) return -EFAULT; @@ -32,6 +125,21 @@ int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from) &to->_sifields._pad[0]); switch (from->si_code >> 16) { case __SI_FAULT >> 16: + if (from->si_signo == SIGBUS && + (from->si_code == BUS_MCEERR_AR || + from->si_code == BUS_MCEERR_AO)) + put_user_ex(from->si_addr_lsb, &to->si_addr_lsb); + + if (from->si_signo == SIGSEGV) { + if (from->si_code == SEGV_BNDERR) { + compat_uptr_t lower = (unsigned long)&to->si_lower; + compat_uptr_t upper = (unsigned long)&to->si_upper; + put_user_ex(lower, &to->si_lower); + put_user_ex(upper, &to->si_upper); + } + if (from->si_code == SEGV_PKUERR) + put_user_ex(from->si_pkey, &to->si_pkey); + } break; case __SI_SYS >> 16: put_user_ex(from->si_syscall, &to->si_syscall); diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index fafe8b923cac..c93609c97406 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -105,6 +105,9 @@ static unsigned int max_physical_pkg_id __read_mostly; unsigned int __max_logical_packages __read_mostly; EXPORT_SYMBOL(__max_logical_packages); +/* Maximum number of SMT threads on any online core */ +int __max_smt_threads __read_mostly; + static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip) { unsigned long flags; @@ -493,7 +496,7 @@ void set_cpu_sibling_map(int cpu) bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1; struct cpuinfo_x86 *c = &cpu_data(cpu); struct cpuinfo_x86 *o; - int i; + int i, threads; cpumask_set_cpu(cpu, cpu_sibling_setup_mask); @@ -550,6 +553,10 @@ void set_cpu_sibling_map(int cpu) if (match_die(c, o) && !topology_same_node(c, o)) primarily_use_numa_for_topology(); } + + threads = cpumask_weight(topology_sibling_cpumask(cpu)); + if (threads > __max_smt_threads) + __max_smt_threads = threads; } /* maps the cpu to the sched domain representing multi-core */ @@ -1285,7 +1292,6 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) cpumask_copy(cpu_callin_mask, cpumask_of(0)); mb(); - current_thread_info()->cpu = 0; /* needed? */ for_each_possible_cpu(i) { zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL); zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL); @@ -1441,6 +1447,21 @@ __init void prefill_possible_map(void) #ifdef CONFIG_HOTPLUG_CPU +/* Recompute SMT state for all CPUs on offline */ +static void recompute_smt_state(void) +{ + int max_threads, cpu; + + max_threads = 0; + for_each_online_cpu (cpu) { + int threads = cpumask_weight(topology_sibling_cpumask(cpu)); + + if (threads > max_threads) + max_threads = threads; + } + __max_smt_threads = max_threads; +} + static void remove_siblinginfo(int cpu) { int sibling; @@ -1465,6 +1486,7 @@ static void remove_siblinginfo(int cpu) c->phys_proc_id = 0; c->cpu_core_id = 0; cpumask_clear_cpu(cpu, cpu_sibling_setup_mask); + recompute_smt_state(); } static void remove_cpu_from_maps(int cpu) @@ -1622,7 +1644,7 @@ static inline void mwait_play_dead(void) } } -static inline void hlt_play_dead(void) +void hlt_play_dead(void) { if (__this_cpu_read(cpu_info.x86) >= 4) wbinvd(); diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 38ba6de56ede..a804b5ab32d0 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -239,7 +239,7 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc) return ns; } -static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) +static void set_cyc2ns_scale(unsigned long khz, int cpu) { unsigned long long tsc_now, ns_now; struct cyc2ns_data *data; @@ -248,7 +248,7 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) local_irq_save(flags); sched_clock_idle_sleep_event(); - if (!cpu_khz) + if (!khz) goto done; data = cyc2ns_write_begin(cpu); @@ -261,7 +261,7 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) * time function is continuous; see the comment near struct * cyc2ns_data. */ - clocks_calc_mult_shift(&data->cyc2ns_mul, &data->cyc2ns_shift, cpu_khz, + clocks_calc_mult_shift(&data->cyc2ns_mul, &data->cyc2ns_shift, khz, NSEC_PER_MSEC, 0); /* @@ -335,12 +335,6 @@ int check_tsc_unstable(void) } EXPORT_SYMBOL_GPL(check_tsc_unstable); -int check_tsc_disabled(void) -{ - return tsc_disabled; -} -EXPORT_SYMBOL_GPL(check_tsc_disabled); - #ifdef CONFIG_X86_TSC int __init notsc_setup(char *str) { @@ -665,19 +659,77 @@ success: } /** - * native_calibrate_tsc - calibrate the tsc on boot + * native_calibrate_tsc + * Determine TSC frequency via CPUID, else return 0. */ unsigned long native_calibrate_tsc(void) +{ + unsigned int eax_denominator, ebx_numerator, ecx_hz, edx; + unsigned int crystal_khz; + + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + return 0; + + if (boot_cpu_data.cpuid_level < 0x15) + return 0; + + eax_denominator = ebx_numerator = ecx_hz = edx = 0; + + /* CPUID 15H TSC/Crystal ratio, plus optionally Crystal Hz */ + cpuid(0x15, &eax_denominator, &ebx_numerator, &ecx_hz, &edx); + + if (ebx_numerator == 0 || eax_denominator == 0) + return 0; + + crystal_khz = ecx_hz / 1000; + + if (crystal_khz == 0) { + switch (boot_cpu_data.x86_model) { + case 0x4E: /* SKL */ + case 0x5E: /* SKL */ + crystal_khz = 24000; /* 24.0 MHz */ + break; + case 0x5C: /* BXT */ + crystal_khz = 19200; /* 19.2 MHz */ + break; + } + } + + return crystal_khz * ebx_numerator / eax_denominator; +} + +static unsigned long cpu_khz_from_cpuid(void) +{ + unsigned int eax_base_mhz, ebx_max_mhz, ecx_bus_mhz, edx; + + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + return 0; + + if (boot_cpu_data.cpuid_level < 0x16) + return 0; + + eax_base_mhz = ebx_max_mhz = ecx_bus_mhz = edx = 0; + + cpuid(0x16, &eax_base_mhz, &ebx_max_mhz, &ecx_bus_mhz, &edx); + + return eax_base_mhz * 1000; +} + +/** + * native_calibrate_cpu - calibrate the cpu on boot + */ +unsigned long native_calibrate_cpu(void) { u64 tsc1, tsc2, delta, ref1, ref2; unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX; unsigned long flags, latch, ms, fast_calibrate; int hpet = is_hpet_enabled(), i, loopmin; - /* Calibrate TSC using MSR for Intel Atom SoCs */ - local_irq_save(flags); - fast_calibrate = try_msr_calibrate_tsc(); - local_irq_restore(flags); + fast_calibrate = cpu_khz_from_cpuid(); + if (fast_calibrate) + return fast_calibrate; + + fast_calibrate = cpu_khz_from_msr(); if (fast_calibrate) return fast_calibrate; @@ -837,8 +889,12 @@ int recalibrate_cpu_khz(void) if (!boot_cpu_has(X86_FEATURE_TSC)) return -ENODEV; + cpu_khz = x86_platform.calibrate_cpu(); tsc_khz = x86_platform.calibrate_tsc(); - cpu_khz = tsc_khz; + if (tsc_khz == 0) + tsc_khz = cpu_khz; + else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz) + cpu_khz = tsc_khz; cpu_data(0).loops_per_jiffy = cpufreq_scale(cpu_data(0).loops_per_jiffy, cpu_khz_old, cpu_khz); @@ -1244,8 +1300,18 @@ void __init tsc_init(void) return; } + cpu_khz = x86_platform.calibrate_cpu(); tsc_khz = x86_platform.calibrate_tsc(); - cpu_khz = tsc_khz; + + /* + * Trust non-zero tsc_khz as authorative, + * and use it to sanity check cpu_khz, + * which will be off if system timer is off. + */ + if (tsc_khz == 0) + tsc_khz = cpu_khz; + else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz) + cpu_khz = tsc_khz; if (!tsc_khz) { mark_tsc_unstable("could not calculate TSC khz"); @@ -1265,7 +1331,7 @@ void __init tsc_init(void) */ for_each_possible_cpu(cpu) { cyc2ns_init(cpu); - set_cyc2ns_scale(cpu_khz, cpu); + set_cyc2ns_scale(tsc_khz, cpu); } if (tsc_disabled > 0) diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c index 9911a0620f9a..0fe720d64fef 100644 --- a/arch/x86/kernel/tsc_msr.c +++ b/arch/x86/kernel/tsc_msr.c @@ -1,14 +1,5 @@ /* - * tsc_msr.c - MSR based TSC calibration on Intel Atom SoC platforms. - * - * TSC in Intel Atom SoC runs at a constant rate which can be figured - * by this formula: - * * - * See Intel 64 and IA-32 System Programming Guid section 16.12 and 30.11.5 - * for details. - * Especially some Intel Atom SoCs don't have PIT(i8254) or HPET, so MSR - * based calibration is the only option. - * + * tsc_msr.c - TSC frequency enumeration via MSR * * Copyright (C) 2013 Intel Corporation * Author: Bin Gao @@ -22,18 +13,10 @@ #include #include -/* CPU reference clock frequency: in KHz */ -#define FREQ_80 80000 -#define FREQ_83 83200 -#define FREQ_100 99840 -#define FREQ_133 133200 -#define FREQ_166 166400 - -#define MAX_NUM_FREQS 8 +#define MAX_NUM_FREQS 9 /* - * According to Intel 64 and IA-32 System Programming Guide, - * if MSR_PERF_STAT[31] is set, the maximum resolved bus ratio can be + * If MSR_PERF_STAT[31] is set, the maximum resolved bus ratio can be * read in MSR_PLATFORM_ID[12:8], otherwise in MSR_PERF_STAT[44:40]. * Unfortunately some Intel Atom SoCs aren't quite compliant to this, * so we need manually differentiate SoC families. This is what the @@ -48,17 +31,18 @@ struct freq_desc { static struct freq_desc freq_desc_tables[] = { /* PNW */ - { 6, 0x27, 0, { 0, 0, 0, 0, 0, FREQ_100, 0, FREQ_83 } }, + { 6, 0x27, 0, { 0, 0, 0, 0, 0, 99840, 0, 83200 } }, /* CLV+ */ - { 6, 0x35, 0, { 0, FREQ_133, 0, 0, 0, FREQ_100, 0, FREQ_83 } }, - /* TNG */ - { 6, 0x4a, 1, { 0, FREQ_100, FREQ_133, 0, 0, 0, 0, 0 } }, - /* VLV2 */ - { 6, 0x37, 1, { FREQ_83, FREQ_100, FREQ_133, FREQ_166, 0, 0, 0, 0 } }, - /* ANN */ - { 6, 0x5a, 1, { FREQ_83, FREQ_100, FREQ_133, FREQ_100, 0, 0, 0, 0 } }, - /* AIRMONT */ - { 6, 0x4c, 1, { FREQ_83, FREQ_100, FREQ_133, FREQ_166, FREQ_80, 0, 0, 0 } }, + { 6, 0x35, 0, { 0, 133200, 0, 0, 0, 99840, 0, 83200 } }, + /* TNG - Intel Atom processor Z3400 series */ + { 6, 0x4a, 1, { 0, 100000, 133300, 0, 0, 0, 0, 0 } }, + /* VLV2 - Intel Atom processor E3000, Z3600, Z3700 series */ + { 6, 0x37, 1, { 83300, 100000, 133300, 116700, 80000, 0, 0, 0 } }, + /* ANN - Intel Atom processor Z3500 series */ + { 6, 0x5a, 1, { 83300, 100000, 133300, 100000, 0, 0, 0, 0 } }, + /* AMT - Intel Atom processor X7-Z8000 and X5-Z8000 series */ + { 6, 0x4c, 1, { 83300, 100000, 133300, 116700, + 80000, 93300, 90000, 88900, 87500 } }, }; static int match_cpu(u8 family, u8 model) @@ -79,16 +63,20 @@ static int match_cpu(u8 family, u8 model) (freq_desc_tables[cpu_index].freqs[freq_id]) /* - * Do MSR calibration only for known/supported CPUs. + * MSR-based CPU/TSC frequency discovery for certain CPUs. * - * Returns the calibration value or 0 if MSR calibration failed. + * Set global "lapic_timer_frequency" to bus_clock_cycles/jiffy + * Return processor base frequency in KHz, or 0 on failure. */ -unsigned long try_msr_calibrate_tsc(void) +unsigned long cpu_khz_from_msr(void) { u32 lo, hi, ratio, freq_id, freq; unsigned long res; int cpu_index; + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + return 0; + cpu_index = match_cpu(boot_cpu_data.x86, boot_cpu_data.x86_model); if (cpu_index < 0) return 0; @@ -100,31 +88,17 @@ unsigned long try_msr_calibrate_tsc(void) rdmsr(MSR_IA32_PERF_STATUS, lo, hi); ratio = (hi >> 8) & 0x1f; } - pr_info("Maximum core-clock to bus-clock ratio: 0x%x\n", ratio); - - if (!ratio) - goto fail; /* Get FSB FREQ ID */ rdmsr(MSR_FSB_FREQ, lo, hi); freq_id = lo & 0x7; freq = id_to_freq(cpu_index, freq_id); - pr_info("Resolved frequency ID: %u, frequency: %u KHz\n", - freq_id, freq); - if (!freq) - goto fail; /* TSC frequency = maximum resolved freq * maximum resolved bus ratio */ res = freq * ratio; - pr_info("TSC runs at %lu KHz\n", res); #ifdef CONFIG_X86_LOCAL_APIC lapic_timer_frequency = (freq * 1000) / HZ; - pr_info("lapic_timer_frequency = %d\n", lapic_timer_frequency); #endif return res; - -fail: - pr_warn("Fast TSC calibration using MSR failed\n"); - return 0; } diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c index 3dce1ca0a653..01f30e56f99e 100644 --- a/arch/x86/kernel/vm86_32.c +++ b/arch/x86/kernel/vm86_32.c @@ -440,10 +440,7 @@ static inline unsigned long get_vflags(struct kernel_vm86_regs *regs) static inline int is_revectored(int nr, struct revectored_struct *bitmap) { - __asm__ __volatile__("btl %2,%1\n\tsbbl %0,%0" - :"=r" (nr) - :"m" (*bitmap), "r" (nr)); - return nr; + return test_bit(nr, bitmap->__map); } #define val_byte(val, n) (((__u8 *)&val)[n]) diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c index cd05942bc918..f1aebfb49c36 100644 --- a/arch/x86/kernel/x8664_ksyms_64.c +++ b/arch/x86/kernel/x8664_ksyms_64.c @@ -44,6 +44,9 @@ EXPORT_SYMBOL(clear_page); EXPORT_SYMBOL(csum_partial); +EXPORT_SYMBOL(__sw_hweight32); +EXPORT_SYMBOL(__sw_hweight64); + /* * Export string functions. We normally rely on gcc builtin for most of these, * but gcc sometimes decides not to inline them. diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index dad5fe9633a3..58b459296e13 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -92,6 +92,7 @@ static void default_nmi_init(void) { }; static int default_i8042_detect(void) { return 1; }; struct x86_platform_ops x86_platform = { + .calibrate_cpu = native_calibrate_cpu, .calibrate_tsc = native_calibrate_tsc, .get_wallclock = mach_get_cmos_time, .set_wallclock = mach_set_rtc_mmss, diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index bbb5b283ff63..a397200281c1 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1310,7 +1310,8 @@ void wait_lapic_expire(struct kvm_vcpu *vcpu) /* __delay is delay_tsc whenever the hardware has TSC, thus always. */ if (guest_tsc < tsc_deadline) - __delay(tsc_deadline - guest_tsc); + __delay(min(tsc_deadline - guest_tsc, + nsec_to_cycles(vcpu, lapic_timer_advance_ns))); } static void start_apic_timer(struct kvm_lapic *apic) diff --git a/arch/x86/kvm/mtrr.c b/arch/x86/kvm/mtrr.c index c146f3c262c3..0149ac59c273 100644 --- a/arch/x86/kvm/mtrr.c +++ b/arch/x86/kvm/mtrr.c @@ -539,6 +539,7 @@ static void mtrr_lookup_var_start(struct mtrr_iter *iter) iter->fixed = false; iter->start_max = iter->start; + iter->range = NULL; iter->range = list_prepare_entry(iter->range, &mtrr_state->head, node); __mtrr_lookup_var_next(iter); diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 003618e324ce..7758680db20b 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -4979,6 +4979,12 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) if (vmx_xsaves_supported()) vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); + if (enable_pml) { + ASSERT(vmx->pml_pg); + vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); + vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); + } + return 0; } @@ -6671,7 +6677,13 @@ static int get_vmx_mem_address(struct kvm_vcpu *vcpu, /* Checks for #GP/#SS exceptions. */ exn = false; - if (is_protmode(vcpu)) { + if (is_long_mode(vcpu)) { + /* Long mode: #GP(0)/#SS(0) if the memory address is in a + * non-canonical form. This is the only check on the memory + * destination for long mode! + */ + exn = is_noncanonical_address(*ret); + } else if (is_protmode(vcpu)) { /* Protected mode: apply checks for segment validity in the * following order: * - segment type check (#GP(0) may be thrown) @@ -6688,17 +6700,10 @@ static int get_vmx_mem_address(struct kvm_vcpu *vcpu, * execute-only code segment */ exn = ((s.type & 0xa) == 8); - } - if (exn) { - kvm_queue_exception_e(vcpu, GP_VECTOR, 0); - return 1; - } - if (is_long_mode(vcpu)) { - /* Long mode: #GP(0)/#SS(0) if the memory address is in a - * non-canonical form. This is an only check for long mode. - */ - exn = is_noncanonical_address(*ret); - } else if (is_protmode(vcpu)) { + if (exn) { + kvm_queue_exception_e(vcpu, GP_VECTOR, 0); + return 1; + } /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. */ exn = (s.unusable != 0); @@ -7938,22 +7943,6 @@ static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) *info2 = vmcs_read32(VM_EXIT_INTR_INFO); } -static int vmx_create_pml_buffer(struct vcpu_vmx *vmx) -{ - struct page *pml_pg; - - pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!pml_pg) - return -ENOMEM; - - vmx->pml_pg = pml_pg; - - vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); - vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); - - return 0; -} - static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) { if (vmx->pml_pg) { @@ -8225,6 +8214,7 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu) if ((vectoring_info & VECTORING_INFO_VALID_MASK) && (exit_reason != EXIT_REASON_EXCEPTION_NMI && exit_reason != EXIT_REASON_EPT_VIOLATION && + exit_reason != EXIT_REASON_PML_FULL && exit_reason != EXIT_REASON_TASK_SWITCH)) { vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; @@ -8855,6 +8845,22 @@ static void vmx_load_vmcs01(struct kvm_vcpu *vcpu) put_cpu(); } +/* + * Ensure that the current vmcs of the logical processor is the + * vmcs01 of the vcpu before calling free_nested(). + */ +static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + int r; + + r = vcpu_load(vcpu); + BUG_ON(r); + vmx_load_vmcs01(vcpu); + free_nested(vmx); + vcpu_put(vcpu); +} + static void vmx_free_vcpu(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -8863,8 +8869,7 @@ static void vmx_free_vcpu(struct kvm_vcpu *vcpu) vmx_destroy_pml_buffer(vmx); free_vpid(vmx->vpid); leave_guest_mode(vcpu); - vmx_load_vmcs01(vcpu); - free_nested(vmx); + vmx_free_vcpu_nested(vcpu); free_loaded_vmcs(vmx->loaded_vmcs); kfree(vmx->guest_msrs); kvm_vcpu_uninit(vcpu); @@ -8886,14 +8891,26 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) if (err) goto free_vcpu; + err = -ENOMEM; + + /* + * If PML is turned on, failure on enabling PML just results in failure + * of creating the vcpu, therefore we can simplify PML logic (by + * avoiding dealing with cases, such as enabling PML partially on vcpus + * for the guest, etc. + */ + if (enable_pml) { + vmx->pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!vmx->pml_pg) + goto uninit_vcpu; + } + vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0]) > PAGE_SIZE); - err = -ENOMEM; - if (!vmx->guest_msrs) { - goto uninit_vcpu; - } + if (!vmx->guest_msrs) + goto free_pml; vmx->loaded_vmcs = &vmx->vmcs01; vmx->loaded_vmcs->vmcs = alloc_vmcs(); @@ -8937,18 +8954,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) vmx->nested.current_vmptr = -1ull; vmx->nested.current_vmcs12 = NULL; - /* - * If PML is turned on, failure on enabling PML just results in failure - * of creating the vcpu, therefore we can simplify PML logic (by - * avoiding dealing with cases, such as enabling PML partially on vcpus - * for the guest, etc. - */ - if (enable_pml) { - err = vmx_create_pml_buffer(vmx); - if (err) - goto free_vmcs; - } - return &vmx->vcpu; free_vmcs: @@ -8956,6 +8961,8 @@ free_vmcs: free_loaded_vmcs(vmx->loaded_vmcs); free_msrs: kfree(vmx->guest_msrs); +free_pml: + vmx_destroy_pml_buffer(vmx); uninit_vcpu: kvm_vcpu_uninit(&vmx->vcpu); free_vcpu: diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 902d9da12392..b2766723c951 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -55,9 +55,6 @@ #include #include -#define CREATE_TRACE_POINTS -#include "trace.h" - #include #include #include @@ -68,6 +65,9 @@ #include #include +#define CREATE_TRACE_POINTS +#include "trace.h" + #define MAX_IO_MSRS 256 #define KVM_MAX_MCE_BANKS 32 #define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P) @@ -1244,12 +1244,6 @@ static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0); static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz); static unsigned long max_tsc_khz; -static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec) -{ - return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult, - vcpu->arch.virtual_tsc_shift); -} - static u32 adjust_tsc_khz(u32 khz, s32 ppm) { u64 v = (u64)khz * (1000000 + ppm); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 7ce3634ab5fe..a82ca466b62e 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -2,6 +2,7 @@ #define ARCH_X86_KVM_X86_H #include +#include #include "kvm_cache_regs.h" #define MSR_IA32_CR_PAT_DEFAULT 0x0007040600070406ULL @@ -195,6 +196,12 @@ extern unsigned int lapic_timer_advance_ns; extern struct static_key kvm_no_apic_vcpu; +static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec) +{ + return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult, + vcpu->arch.virtual_tsc_shift); +} + /* Same "calling convention" as do_div: * - divide (n << 32) by base * - put result in n diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c index 3847e736702e..25da5bc8d83d 100644 --- a/arch/x86/lguest/boot.c +++ b/arch/x86/lguest/boot.c @@ -1233,8 +1233,6 @@ static void write_bar_via_cfg(u32 cfg_offset, u32 off, u32 val) static void probe_pci_console(void) { u8 cap, common_cap = 0, device_cap = 0; - /* Offset within BAR0 */ - u32 device_offset; u32 device_len; /* Avoid recursive printk into here. */ @@ -1258,24 +1256,16 @@ static void probe_pci_console(void) u8 vndr = read_pci_config_byte(0, 1, 0, cap); if (vndr == PCI_CAP_ID_VNDR) { u8 type, bar; - u32 offset, length; type = read_pci_config_byte(0, 1, 0, cap + offsetof(struct virtio_pci_cap, cfg_type)); bar = read_pci_config_byte(0, 1, 0, cap + offsetof(struct virtio_pci_cap, bar)); - offset = read_pci_config(0, 1, 0, - cap + offsetof(struct virtio_pci_cap, offset)); - length = read_pci_config(0, 1, 0, - cap + offsetof(struct virtio_pci_cap, length)); switch (type) { case VIRTIO_PCI_CAP_DEVICE_CFG: - if (bar == 0) { + if (bar == 0) device_cap = cap; - device_offset = offset; - device_len = length; - } break; case VIRTIO_PCI_CAP_PCI_CFG: console_access_cap = cap; @@ -1297,13 +1287,16 @@ static void probe_pci_console(void) * emerg_wr. If it doesn't support VIRTIO_CONSOLE_F_EMERG_WRITE * it should ignore the access. */ + device_len = read_pci_config(0, 1, 0, + device_cap + offsetof(struct virtio_pci_cap, length)); if (device_len < (offsetof(struct virtio_console_config, emerg_wr) + sizeof(u32))) { printk(KERN_ERR "lguest: console missing emerg_wr field\n"); return; } - console_cfg_offset = device_offset; + console_cfg_offset = read_pci_config(0, 1, 0, + device_cap + offsetof(struct virtio_pci_cap, offset)); printk(KERN_INFO "lguest: Console via virtio-pci emerg_wr\n"); } diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 72a576752a7e..34a74131a12c 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -24,8 +24,9 @@ lib-y += usercopy_$(BITS).o usercopy.o getuser.o putuser.o lib-y += memcpy_$(BITS).o lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o +lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o -obj-y += msr.o msr-reg.o msr-reg-export.o +obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o ifeq ($(CONFIG_X86_32),y) obj-y += atomic64_32.o diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S index 2b0ef26da0bd..bf603ebbfd8e 100644 --- a/arch/x86/lib/copy_user_64.S +++ b/arch/x86/lib/copy_user_64.S @@ -17,11 +17,11 @@ /* Standard copy_to_user with segment limit checking */ ENTRY(_copy_to_user) - GET_THREAD_INFO(%rax) + mov PER_CPU_VAR(current_task), %rax movq %rdi,%rcx addq %rdx,%rcx jc bad_to_user - cmpq TI_addr_limit(%rax),%rcx + cmpq TASK_addr_limit(%rax),%rcx ja bad_to_user ALTERNATIVE_2 "jmp copy_user_generic_unrolled", \ "jmp copy_user_generic_string", \ @@ -32,11 +32,11 @@ ENDPROC(_copy_to_user) /* Standard copy_from_user with segment limit checking */ ENTRY(_copy_from_user) - GET_THREAD_INFO(%rax) + mov PER_CPU_VAR(current_task), %rax movq %rsi,%rcx addq %rdx,%rcx jc bad_from_user - cmpq TI_addr_limit(%rax),%rcx + cmpq TASK_addr_limit(%rax),%rcx ja bad_from_user ALTERNATIVE_2 "jmp copy_user_generic_unrolled", \ "jmp copy_user_generic_string", \ diff --git a/arch/x86/lib/csum-wrappers_64.c b/arch/x86/lib/csum-wrappers_64.c index 28a6654f0d08..b6fcb9a9ddbc 100644 --- a/arch/x86/lib/csum-wrappers_64.c +++ b/arch/x86/lib/csum-wrappers_64.c @@ -6,6 +6,7 @@ */ #include #include +#include #include /** diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S index 46668cda4ffd..0ef5128c2de8 100644 --- a/arch/x86/lib/getuser.S +++ b/arch/x86/lib/getuser.S @@ -35,8 +35,8 @@ .text ENTRY(__get_user_1) - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + mov PER_CPU_VAR(current_task), %_ASM_DX + cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user ASM_STAC 1: movzbl (%_ASM_AX),%edx @@ -48,8 +48,8 @@ ENDPROC(__get_user_1) ENTRY(__get_user_2) add $1,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + mov PER_CPU_VAR(current_task), %_ASM_DX + cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user ASM_STAC 2: movzwl -1(%_ASM_AX),%edx @@ -61,8 +61,8 @@ ENDPROC(__get_user_2) ENTRY(__get_user_4) add $3,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + mov PER_CPU_VAR(current_task), %_ASM_DX + cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user ASM_STAC 3: movl -3(%_ASM_AX),%edx @@ -75,8 +75,8 @@ ENTRY(__get_user_8) #ifdef CONFIG_X86_64 add $7,%_ASM_AX jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + mov PER_CPU_VAR(current_task), %_ASM_DX + cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user ASM_STAC 4: movq -7(%_ASM_AX),%rdx @@ -86,8 +86,8 @@ ENTRY(__get_user_8) #else add $7,%_ASM_AX jc bad_get_user_8 - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + mov PER_CPU_VAR(current_task), %_ASM_DX + cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX jae bad_get_user_8 ASM_STAC 4: movl -7(%_ASM_AX),%edx diff --git a/arch/x86/lib/hweight.S b/arch/x86/lib/hweight.S new file mode 100644 index 000000000000..02de3d74d2c5 --- /dev/null +++ b/arch/x86/lib/hweight.S @@ -0,0 +1,77 @@ +#include + +#include + +/* + * unsigned int __sw_hweight32(unsigned int w) + * %rdi: w + */ +ENTRY(__sw_hweight32) + +#ifdef CONFIG_X86_64 + movl %edi, %eax # w +#endif + __ASM_SIZE(push,) %__ASM_REG(dx) + movl %eax, %edx # w -> t + shrl %edx # t >>= 1 + andl $0x55555555, %edx # t &= 0x55555555 + subl %edx, %eax # w -= t + + movl %eax, %edx # w -> t + shrl $2, %eax # w_tmp >>= 2 + andl $0x33333333, %edx # t &= 0x33333333 + andl $0x33333333, %eax # w_tmp &= 0x33333333 + addl %edx, %eax # w = w_tmp + t + + movl %eax, %edx # w -> t + shrl $4, %edx # t >>= 4 + addl %edx, %eax # w_tmp += t + andl $0x0f0f0f0f, %eax # w_tmp &= 0x0f0f0f0f + imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101 + shrl $24, %eax # w = w_tmp >> 24 + __ASM_SIZE(pop,) %__ASM_REG(dx) + ret +ENDPROC(__sw_hweight32) + +ENTRY(__sw_hweight64) +#ifdef CONFIG_X86_64 + pushq %rdx + + movq %rdi, %rdx # w -> t + movabsq $0x5555555555555555, %rax + shrq %rdx # t >>= 1 + andq %rdx, %rax # t &= 0x5555555555555555 + movabsq $0x3333333333333333, %rdx + subq %rax, %rdi # w -= t + + movq %rdi, %rax # w -> t + shrq $2, %rdi # w_tmp >>= 2 + andq %rdx, %rax # t &= 0x3333333333333333 + andq %rdi, %rdx # w_tmp &= 0x3333333333333333 + addq %rdx, %rax # w = w_tmp + t + + movq %rax, %rdx # w -> t + shrq $4, %rdx # t >>= 4 + addq %rdx, %rax # w_tmp += t + movabsq $0x0f0f0f0f0f0f0f0f, %rdx + andq %rdx, %rax # w_tmp &= 0x0f0f0f0f0f0f0f0f + movabsq $0x0101010101010101, %rdx + imulq %rdx, %rax # w_tmp *= 0x0101010101010101 + shrq $56, %rax # w = w_tmp >> 56 + + popq %rdx + ret +#else /* CONFIG_X86_32 */ + /* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */ + pushl %ecx + + call __sw_hweight32 + movl %eax, %ecx # stash away result + movl %edx, %eax # second part of input + call __sw_hweight32 + addl %ecx, %eax # result + + popl %ecx + ret +#endif +ENDPROC(__sw_hweight64) diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c index 1a416935bac9..1088eb8f3a5f 100644 --- a/arch/x86/lib/insn.c +++ b/arch/x86/lib/insn.c @@ -155,14 +155,24 @@ found: /* * In 32-bits mode, if the [7:6] bits (mod bits of * ModRM) on the second byte are not 11b, it is - * LDS or LES. + * LDS or LES or BOUND. */ if (X86_MODRM_MOD(b2) != 3) goto vex_end; } insn->vex_prefix.bytes[0] = b; insn->vex_prefix.bytes[1] = b2; - if (inat_is_vex3_prefix(attr)) { + if (inat_is_evex_prefix(attr)) { + b2 = peek_nbyte_next(insn_byte_t, insn, 2); + insn->vex_prefix.bytes[2] = b2; + b2 = peek_nbyte_next(insn_byte_t, insn, 3); + insn->vex_prefix.bytes[3] = b2; + insn->vex_prefix.nbytes = 4; + insn->next_byte += 4; + if (insn->x86_64 && X86_VEX_W(b2)) + /* VEX.W overrides opnd_size */ + insn->opnd_bytes = 8; + } else if (inat_is_vex3_prefix(attr)) { b2 = peek_nbyte_next(insn_byte_t, insn, 2); insn->vex_prefix.bytes[2] = b2; insn->vex_prefix.nbytes = 3; @@ -221,7 +231,9 @@ void insn_get_opcode(struct insn *insn) m = insn_vex_m_bits(insn); p = insn_vex_p_bits(insn); insn->attr = inat_get_avx_attribute(op, m, p); - if (!inat_accept_vex(insn->attr) && !inat_is_group(insn->attr)) + if ((inat_must_evex(insn->attr) && !insn_is_evex(insn)) || + (!inat_accept_vex(insn->attr) && + !inat_is_group(insn->attr))) insn->attr = 0; /* This instruction is bad */ goto end; /* VEX has only 1 byte for opcode */ } diff --git a/arch/x86/lib/kaslr.c b/arch/x86/lib/kaslr.c new file mode 100644 index 000000000000..f7dfeda83e5c --- /dev/null +++ b/arch/x86/lib/kaslr.c @@ -0,0 +1,90 @@ +/* + * Entropy functions used on early boot for KASLR base and memory + * randomization. The base randomization is done in the compressed + * kernel and memory randomization is done early when the regular + * kernel starts. This file is included in the compressed kernel and + * normally linked in the regular. + */ +#include +#include +#include +#include +#include + +/* + * When built for the regular kernel, several functions need to be stubbed out + * or changed to their regular kernel equivalent. + */ +#ifndef KASLR_COMPRESSED_BOOT +#include +#include + +#define debug_putstr(v) early_printk(v) +#define has_cpuflag(f) boot_cpu_has(f) +#define get_boot_seed() kaslr_offset() +#endif + +#define I8254_PORT_CONTROL 0x43 +#define I8254_PORT_COUNTER0 0x40 +#define I8254_CMD_READBACK 0xC0 +#define I8254_SELECT_COUNTER0 0x02 +#define I8254_STATUS_NOTREADY 0x40 +static inline u16 i8254(void) +{ + u16 status, timer; + + do { + outb(I8254_PORT_CONTROL, + I8254_CMD_READBACK | I8254_SELECT_COUNTER0); + status = inb(I8254_PORT_COUNTER0); + timer = inb(I8254_PORT_COUNTER0); + timer |= inb(I8254_PORT_COUNTER0) << 8; + } while (status & I8254_STATUS_NOTREADY); + + return timer; +} + +unsigned long kaslr_get_random_long(const char *purpose) +{ +#ifdef CONFIG_X86_64 + const unsigned long mix_const = 0x5d6008cbf3848dd3UL; +#else + const unsigned long mix_const = 0x3f39e593UL; +#endif + unsigned long raw, random = get_boot_seed(); + bool use_i8254 = true; + + debug_putstr(purpose); + debug_putstr(" KASLR using"); + + if (has_cpuflag(X86_FEATURE_RDRAND)) { + debug_putstr(" RDRAND"); + if (rdrand_long(&raw)) { + random ^= raw; + use_i8254 = false; + } + } + + if (has_cpuflag(X86_FEATURE_TSC)) { + debug_putstr(" RDTSC"); + raw = rdtsc(); + + random ^= raw; + use_i8254 = false; + } + + if (use_i8254) { + debug_putstr(" i8254"); + random ^= i8254(); + } + + /* Circular multiply for better bit diffusion */ + asm("mul %3" + : "=a" (random), "=d" (raw) + : "a" (random), "rm" (mix_const)); + random += raw; + + debug_putstr("...\n"); + + return random; +} diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S index e0817a12d323..c891ece81e5b 100644 --- a/arch/x86/lib/putuser.S +++ b/arch/x86/lib/putuser.S @@ -29,14 +29,14 @@ * as they get called from within inline assembly. */ -#define ENTER GET_THREAD_INFO(%_ASM_BX) +#define ENTER mov PER_CPU_VAR(current_task), %_ASM_BX #define EXIT ASM_CLAC ; \ ret .text ENTRY(__put_user_1) ENTER - cmp TI_addr_limit(%_ASM_BX),%_ASM_CX + cmp TASK_addr_limit(%_ASM_BX),%_ASM_CX jae bad_put_user ASM_STAC 1: movb %al,(%_ASM_CX) @@ -46,7 +46,7 @@ ENDPROC(__put_user_1) ENTRY(__put_user_2) ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX + mov TASK_addr_limit(%_ASM_BX),%_ASM_BX sub $1,%_ASM_BX cmp %_ASM_BX,%_ASM_CX jae bad_put_user @@ -58,7 +58,7 @@ ENDPROC(__put_user_2) ENTRY(__put_user_4) ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX + mov TASK_addr_limit(%_ASM_BX),%_ASM_BX sub $3,%_ASM_BX cmp %_ASM_BX,%_ASM_CX jae bad_put_user @@ -70,7 +70,7 @@ ENDPROC(__put_user_4) ENTRY(__put_user_8) ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX + mov TASK_addr_limit(%_ASM_BX),%_ASM_BX sub $7,%_ASM_BX cmp %_ASM_BX,%_ASM_CX jae bad_put_user diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 0a42327a59d7..9f760cdcaf40 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -6,7 +6,7 @@ * Copyright 2002 Andi Kleen */ #include -#include +#include /* * Zero Userspace diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index d388de72eaca..ec378cd7b71e 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -13,12 +13,17 @@ # opcode: escape # escaped-name # EndTable # +# mnemonics that begin with lowercase 'v' accept a VEX or EVEX prefix +# mnemonics that begin with lowercase 'k' accept a VEX prefix +# # # GrpTable: GrpXXX # reg: mnemonic [operand1[,operand2...]] [(extra1)[,(extra2)...] [| 2nd-mnemonic ...] # EndTable # # AVX Superscripts +# (ev): this opcode requires EVEX prefix. +# (evo): this opcode is changed by EVEX prefix (EVEX opcode) # (v): this opcode requires VEX prefix. # (v1): this opcode only supports 128bit VEX. # @@ -137,7 +142,7 @@ AVXcode: # 0x60 - 0x6f 60: PUSHA/PUSHAD (i64) 61: POPA/POPAD (i64) -62: BOUND Gv,Ma (i64) +62: BOUND Gv,Ma (i64) | EVEX (Prefix) 63: ARPL Ew,Gw (i64) | MOVSXD Gv,Ev (o64) 64: SEG=FS (Prefix) 65: SEG=GS (Prefix) @@ -399,17 +404,17 @@ AVXcode: 1 3f: # 0x0f 0x40-0x4f 40: CMOVO Gv,Ev -41: CMOVNO Gv,Ev -42: CMOVB/C/NAE Gv,Ev +41: CMOVNO Gv,Ev | kandw/q Vk,Hk,Uk | kandb/d Vk,Hk,Uk (66) +42: CMOVB/C/NAE Gv,Ev | kandnw/q Vk,Hk,Uk | kandnb/d Vk,Hk,Uk (66) 43: CMOVAE/NB/NC Gv,Ev -44: CMOVE/Z Gv,Ev -45: CMOVNE/NZ Gv,Ev -46: CMOVBE/NA Gv,Ev -47: CMOVA/NBE Gv,Ev +44: CMOVE/Z Gv,Ev | knotw/q Vk,Uk | knotb/d Vk,Uk (66) +45: CMOVNE/NZ Gv,Ev | korw/q Vk,Hk,Uk | korb/d Vk,Hk,Uk (66) +46: CMOVBE/NA Gv,Ev | kxnorw/q Vk,Hk,Uk | kxnorb/d Vk,Hk,Uk (66) +47: CMOVA/NBE Gv,Ev | kxorw/q Vk,Hk,Uk | kxorb/d Vk,Hk,Uk (66) 48: CMOVS Gv,Ev 49: CMOVNS Gv,Ev -4a: CMOVP/PE Gv,Ev -4b: CMOVNP/PO Gv,Ev +4a: CMOVP/PE Gv,Ev | kaddw/q Vk,Hk,Uk | kaddb/d Vk,Hk,Uk (66) +4b: CMOVNP/PO Gv,Ev | kunpckbw Vk,Hk,Uk (66) | kunpckwd/dq Vk,Hk,Uk 4c: CMOVL/NGE Gv,Ev 4d: CMOVNL/GE Gv,Ev 4e: CMOVLE/NG Gv,Ev @@ -426,7 +431,7 @@ AVXcode: 1 58: vaddps Vps,Hps,Wps | vaddpd Vpd,Hpd,Wpd (66) | vaddss Vss,Hss,Wss (F3),(v1) | vaddsd Vsd,Hsd,Wsd (F2),(v1) 59: vmulps Vps,Hps,Wps | vmulpd Vpd,Hpd,Wpd (66) | vmulss Vss,Hss,Wss (F3),(v1) | vmulsd Vsd,Hsd,Wsd (F2),(v1) 5a: vcvtps2pd Vpd,Wps | vcvtpd2ps Vps,Wpd (66) | vcvtss2sd Vsd,Hx,Wss (F3),(v1) | vcvtsd2ss Vss,Hx,Wsd (F2),(v1) -5b: vcvtdq2ps Vps,Wdq | vcvtps2dq Vdq,Wps (66) | vcvttps2dq Vdq,Wps (F3) +5b: vcvtdq2ps Vps,Wdq | vcvtqq2ps Vps,Wqq (evo) | vcvtps2dq Vdq,Wps (66) | vcvttps2dq Vdq,Wps (F3) 5c: vsubps Vps,Hps,Wps | vsubpd Vpd,Hpd,Wpd (66) | vsubss Vss,Hss,Wss (F3),(v1) | vsubsd Vsd,Hsd,Wsd (F2),(v1) 5d: vminps Vps,Hps,Wps | vminpd Vpd,Hpd,Wpd (66) | vminss Vss,Hss,Wss (F3),(v1) | vminsd Vsd,Hsd,Wsd (F2),(v1) 5e: vdivps Vps,Hps,Wps | vdivpd Vpd,Hpd,Wpd (66) | vdivss Vss,Hss,Wss (F3),(v1) | vdivsd Vsd,Hsd,Wsd (F2),(v1) @@ -447,7 +452,7 @@ AVXcode: 1 6c: vpunpcklqdq Vx,Hx,Wx (66),(v1) 6d: vpunpckhqdq Vx,Hx,Wx (66),(v1) 6e: movd/q Pd,Ey | vmovd/q Vy,Ey (66),(v1) -6f: movq Pq,Qq | vmovdqa Vx,Wx (66) | vmovdqu Vx,Wx (F3) +6f: movq Pq,Qq | vmovdqa Vx,Wx (66) | vmovdqa32/64 Vx,Wx (66),(evo) | vmovdqu Vx,Wx (F3) | vmovdqu32/64 Vx,Wx (F3),(evo) | vmovdqu8/16 Vx,Wx (F2),(ev) # 0x0f 0x70-0x7f 70: pshufw Pq,Qq,Ib | vpshufd Vx,Wx,Ib (66),(v1) | vpshufhw Vx,Wx,Ib (F3),(v1) | vpshuflw Vx,Wx,Ib (F2),(v1) 71: Grp12 (1A) @@ -458,14 +463,14 @@ AVXcode: 1 76: pcmpeqd Pq,Qq | vpcmpeqd Vx,Hx,Wx (66),(v1) # Note: Remove (v), because vzeroall and vzeroupper becomes emms without VEX. 77: emms | vzeroupper | vzeroall -78: VMREAD Ey,Gy -79: VMWRITE Gy,Ey -7a: -7b: +78: VMREAD Ey,Gy | vcvttps2udq/pd2udq Vx,Wpd (evo) | vcvttsd2usi Gv,Wx (F2),(ev) | vcvttss2usi Gv,Wx (F3),(ev) | vcvttps2uqq/pd2uqq Vx,Wx (66),(ev) +79: VMWRITE Gy,Ey | vcvtps2udq/pd2udq Vx,Wpd (evo) | vcvtsd2usi Gv,Wx (F2),(ev) | vcvtss2usi Gv,Wx (F3),(ev) | vcvtps2uqq/pd2uqq Vx,Wx (66),(ev) +7a: vcvtudq2pd/uqq2pd Vpd,Wx (F3),(ev) | vcvtudq2ps/uqq2ps Vpd,Wx (F2),(ev) | vcvttps2qq/pd2qq Vx,Wx (66),(ev) +7b: vcvtusi2sd Vpd,Hpd,Ev (F2),(ev) | vcvtusi2ss Vps,Hps,Ev (F3),(ev) | vcvtps2qq/pd2qq Vx,Wx (66),(ev) 7c: vhaddpd Vpd,Hpd,Wpd (66) | vhaddps Vps,Hps,Wps (F2) 7d: vhsubpd Vpd,Hpd,Wpd (66) | vhsubps Vps,Hps,Wps (F2) 7e: movd/q Ey,Pd | vmovd/q Ey,Vy (66),(v1) | vmovq Vq,Wq (F3),(v1) -7f: movq Qq,Pq | vmovdqa Wx,Vx (66) | vmovdqu Wx,Vx (F3) +7f: movq Qq,Pq | vmovdqa Wx,Vx (66) | vmovdqa32/64 Wx,Vx (66),(evo) | vmovdqu Wx,Vx (F3) | vmovdqu32/64 Wx,Vx (F3),(evo) | vmovdqu8/16 Wx,Vx (F2),(ev) # 0x0f 0x80-0x8f # Note: "forced64" is Intel CPU behavior (see comment about CALL insn). 80: JO Jz (f64) @@ -485,16 +490,16 @@ AVXcode: 1 8e: JLE/JNG Jz (f64) 8f: JNLE/JG Jz (f64) # 0x0f 0x90-0x9f -90: SETO Eb -91: SETNO Eb -92: SETB/C/NAE Eb -93: SETAE/NB/NC Eb +90: SETO Eb | kmovw/q Vk,Wk | kmovb/d Vk,Wk (66) +91: SETNO Eb | kmovw/q Mv,Vk | kmovb/d Mv,Vk (66) +92: SETB/C/NAE Eb | kmovw Vk,Rv | kmovb Vk,Rv (66) | kmovq/d Vk,Rv (F2) +93: SETAE/NB/NC Eb | kmovw Gv,Uk | kmovb Gv,Uk (66) | kmovq/d Gv,Uk (F2) 94: SETE/Z Eb 95: SETNE/NZ Eb 96: SETBE/NA Eb 97: SETA/NBE Eb -98: SETS Eb -99: SETNS Eb +98: SETS Eb | kortestw/q Vk,Uk | kortestb/d Vk,Uk (66) +99: SETNS Eb | ktestw/q Vk,Uk | ktestb/d Vk,Uk (66) 9a: SETP/PE Eb 9b: SETNP/PO Eb 9c: SETL/NGE Eb @@ -564,11 +569,11 @@ d7: pmovmskb Gd,Nq | vpmovmskb Gd,Ux (66),(v1) d8: psubusb Pq,Qq | vpsubusb Vx,Hx,Wx (66),(v1) d9: psubusw Pq,Qq | vpsubusw Vx,Hx,Wx (66),(v1) da: pminub Pq,Qq | vpminub Vx,Hx,Wx (66),(v1) -db: pand Pq,Qq | vpand Vx,Hx,Wx (66),(v1) +db: pand Pq,Qq | vpand Vx,Hx,Wx (66),(v1) | vpandd/q Vx,Hx,Wx (66),(evo) dc: paddusb Pq,Qq | vpaddusb Vx,Hx,Wx (66),(v1) dd: paddusw Pq,Qq | vpaddusw Vx,Hx,Wx (66),(v1) de: pmaxub Pq,Qq | vpmaxub Vx,Hx,Wx (66),(v1) -df: pandn Pq,Qq | vpandn Vx,Hx,Wx (66),(v1) +df: pandn Pq,Qq | vpandn Vx,Hx,Wx (66),(v1) | vpandnd/q Vx,Hx,Wx (66),(evo) # 0x0f 0xe0-0xef e0: pavgb Pq,Qq | vpavgb Vx,Hx,Wx (66),(v1) e1: psraw Pq,Qq | vpsraw Vx,Hx,Wx (66),(v1) @@ -576,16 +581,16 @@ e2: psrad Pq,Qq | vpsrad Vx,Hx,Wx (66),(v1) e3: pavgw Pq,Qq | vpavgw Vx,Hx,Wx (66),(v1) e4: pmulhuw Pq,Qq | vpmulhuw Vx,Hx,Wx (66),(v1) e5: pmulhw Pq,Qq | vpmulhw Vx,Hx,Wx (66),(v1) -e6: vcvttpd2dq Vx,Wpd (66) | vcvtdq2pd Vx,Wdq (F3) | vcvtpd2dq Vx,Wpd (F2) +e6: vcvttpd2dq Vx,Wpd (66) | vcvtdq2pd Vx,Wdq (F3) | vcvtdq2pd/qq2pd Vx,Wdq (F3),(evo) | vcvtpd2dq Vx,Wpd (F2) e7: movntq Mq,Pq | vmovntdq Mx,Vx (66) e8: psubsb Pq,Qq | vpsubsb Vx,Hx,Wx (66),(v1) e9: psubsw Pq,Qq | vpsubsw Vx,Hx,Wx (66),(v1) ea: pminsw Pq,Qq | vpminsw Vx,Hx,Wx (66),(v1) -eb: por Pq,Qq | vpor Vx,Hx,Wx (66),(v1) +eb: por Pq,Qq | vpor Vx,Hx,Wx (66),(v1) | vpord/q Vx,Hx,Wx (66),(evo) ec: paddsb Pq,Qq | vpaddsb Vx,Hx,Wx (66),(v1) ed: paddsw Pq,Qq | vpaddsw Vx,Hx,Wx (66),(v1) ee: pmaxsw Pq,Qq | vpmaxsw Vx,Hx,Wx (66),(v1) -ef: pxor Pq,Qq | vpxor Vx,Hx,Wx (66),(v1) +ef: pxor Pq,Qq | vpxor Vx,Hx,Wx (66),(v1) | vpxord/q Vx,Hx,Wx (66),(evo) # 0x0f 0xf0-0xff f0: vlddqu Vx,Mx (F2) f1: psllw Pq,Qq | vpsllw Vx,Hx,Wx (66),(v1) @@ -626,81 +631,105 @@ AVXcode: 2 0e: vtestps Vx,Wx (66),(v) 0f: vtestpd Vx,Wx (66),(v) # 0x0f 0x38 0x10-0x1f -10: pblendvb Vdq,Wdq (66) -11: -12: -13: vcvtph2ps Vx,Wx,Ib (66),(v) -14: blendvps Vdq,Wdq (66) -15: blendvpd Vdq,Wdq (66) -16: vpermps Vqq,Hqq,Wqq (66),(v) +10: pblendvb Vdq,Wdq (66) | vpsrlvw Vx,Hx,Wx (66),(evo) | vpmovuswb Wx,Vx (F3),(ev) +11: vpmovusdb Wx,Vd (F3),(ev) | vpsravw Vx,Hx,Wx (66),(ev) +12: vpmovusqb Wx,Vq (F3),(ev) | vpsllvw Vx,Hx,Wx (66),(ev) +13: vcvtph2ps Vx,Wx (66),(v) | vpmovusdw Wx,Vd (F3),(ev) +14: blendvps Vdq,Wdq (66) | vpmovusqw Wx,Vq (F3),(ev) | vprorvd/q Vx,Hx,Wx (66),(evo) +15: blendvpd Vdq,Wdq (66) | vpmovusqd Wx,Vq (F3),(ev) | vprolvd/q Vx,Hx,Wx (66),(evo) +16: vpermps Vqq,Hqq,Wqq (66),(v) | vpermps/d Vqq,Hqq,Wqq (66),(evo) 17: vptest Vx,Wx (66) 18: vbroadcastss Vx,Wd (66),(v) -19: vbroadcastsd Vqq,Wq (66),(v) -1a: vbroadcastf128 Vqq,Mdq (66),(v) -1b: +19: vbroadcastsd Vqq,Wq (66),(v) | vbroadcastf32x2 Vqq,Wq (66),(evo) +1a: vbroadcastf128 Vqq,Mdq (66),(v) | vbroadcastf32x4/64x2 Vqq,Wq (66),(evo) +1b: vbroadcastf32x8/64x4 Vqq,Mdq (66),(ev) 1c: pabsb Pq,Qq | vpabsb Vx,Wx (66),(v1) 1d: pabsw Pq,Qq | vpabsw Vx,Wx (66),(v1) 1e: pabsd Pq,Qq | vpabsd Vx,Wx (66),(v1) -1f: +1f: vpabsq Vx,Wx (66),(ev) # 0x0f 0x38 0x20-0x2f -20: vpmovsxbw Vx,Ux/Mq (66),(v1) -21: vpmovsxbd Vx,Ux/Md (66),(v1) -22: vpmovsxbq Vx,Ux/Mw (66),(v1) -23: vpmovsxwd Vx,Ux/Mq (66),(v1) -24: vpmovsxwq Vx,Ux/Md (66),(v1) -25: vpmovsxdq Vx,Ux/Mq (66),(v1) -26: -27: -28: vpmuldq Vx,Hx,Wx (66),(v1) -29: vpcmpeqq Vx,Hx,Wx (66),(v1) -2a: vmovntdqa Vx,Mx (66),(v1) +20: vpmovsxbw Vx,Ux/Mq (66),(v1) | vpmovswb Wx,Vx (F3),(ev) +21: vpmovsxbd Vx,Ux/Md (66),(v1) | vpmovsdb Wx,Vd (F3),(ev) +22: vpmovsxbq Vx,Ux/Mw (66),(v1) | vpmovsqb Wx,Vq (F3),(ev) +23: vpmovsxwd Vx,Ux/Mq (66),(v1) | vpmovsdw Wx,Vd (F3),(ev) +24: vpmovsxwq Vx,Ux/Md (66),(v1) | vpmovsqw Wx,Vq (F3),(ev) +25: vpmovsxdq Vx,Ux/Mq (66),(v1) | vpmovsqd Wx,Vq (F3),(ev) +26: vptestmb/w Vk,Hx,Wx (66),(ev) | vptestnmb/w Vk,Hx,Wx (F3),(ev) +27: vptestmd/q Vk,Hx,Wx (66),(ev) | vptestnmd/q Vk,Hx,Wx (F3),(ev) +28: vpmuldq Vx,Hx,Wx (66),(v1) | vpmovm2b/w Vx,Uk (F3),(ev) +29: vpcmpeqq Vx,Hx,Wx (66),(v1) | vpmovb2m/w2m Vk,Ux (F3),(ev) +2a: vmovntdqa Vx,Mx (66),(v1) | vpbroadcastmb2q Vx,Uk (F3),(ev) 2b: vpackusdw Vx,Hx,Wx (66),(v1) -2c: vmaskmovps Vx,Hx,Mx (66),(v) -2d: vmaskmovpd Vx,Hx,Mx (66),(v) +2c: vmaskmovps Vx,Hx,Mx (66),(v) | vscalefps/d Vx,Hx,Wx (66),(evo) +2d: vmaskmovpd Vx,Hx,Mx (66),(v) | vscalefss/d Vx,Hx,Wx (66),(evo) 2e: vmaskmovps Mx,Hx,Vx (66),(v) 2f: vmaskmovpd Mx,Hx,Vx (66),(v) # 0x0f 0x38 0x30-0x3f -30: vpmovzxbw Vx,Ux/Mq (66),(v1) -31: vpmovzxbd Vx,Ux/Md (66),(v1) -32: vpmovzxbq Vx,Ux/Mw (66),(v1) -33: vpmovzxwd Vx,Ux/Mq (66),(v1) -34: vpmovzxwq Vx,Ux/Md (66),(v1) -35: vpmovzxdq Vx,Ux/Mq (66),(v1) -36: vpermd Vqq,Hqq,Wqq (66),(v) +30: vpmovzxbw Vx,Ux/Mq (66),(v1) | vpmovwb Wx,Vx (F3),(ev) +31: vpmovzxbd Vx,Ux/Md (66),(v1) | vpmovdb Wx,Vd (F3),(ev) +32: vpmovzxbq Vx,Ux/Mw (66),(v1) | vpmovqb Wx,Vq (F3),(ev) +33: vpmovzxwd Vx,Ux/Mq (66),(v1) | vpmovdw Wx,Vd (F3),(ev) +34: vpmovzxwq Vx,Ux/Md (66),(v1) | vpmovqw Wx,Vq (F3),(ev) +35: vpmovzxdq Vx,Ux/Mq (66),(v1) | vpmovqd Wx,Vq (F3),(ev) +36: vpermd Vqq,Hqq,Wqq (66),(v) | vpermd/q Vqq,Hqq,Wqq (66),(evo) 37: vpcmpgtq Vx,Hx,Wx (66),(v1) -38: vpminsb Vx,Hx,Wx (66),(v1) -39: vpminsd Vx,Hx,Wx (66),(v1) -3a: vpminuw Vx,Hx,Wx (66),(v1) -3b: vpminud Vx,Hx,Wx (66),(v1) +38: vpminsb Vx,Hx,Wx (66),(v1) | vpmovm2d/q Vx,Uk (F3),(ev) +39: vpminsd Vx,Hx,Wx (66),(v1) | vpminsd/q Vx,Hx,Wx (66),(evo) | vpmovd2m/q2m Vk,Ux (F3),(ev) +3a: vpminuw Vx,Hx,Wx (66),(v1) | vpbroadcastmw2d Vx,Uk (F3),(ev) +3b: vpminud Vx,Hx,Wx (66),(v1) | vpminud/q Vx,Hx,Wx (66),(evo) 3c: vpmaxsb Vx,Hx,Wx (66),(v1) -3d: vpmaxsd Vx,Hx,Wx (66),(v1) +3d: vpmaxsd Vx,Hx,Wx (66),(v1) | vpmaxsd/q Vx,Hx,Wx (66),(evo) 3e: vpmaxuw Vx,Hx,Wx (66),(v1) -3f: vpmaxud Vx,Hx,Wx (66),(v1) +3f: vpmaxud Vx,Hx,Wx (66),(v1) | vpmaxud/q Vx,Hx,Wx (66),(evo) # 0x0f 0x38 0x40-0x8f -40: vpmulld Vx,Hx,Wx (66),(v1) +40: vpmulld Vx,Hx,Wx (66),(v1) | vpmulld/q Vx,Hx,Wx (66),(evo) 41: vphminposuw Vdq,Wdq (66),(v1) -42: -43: -44: +42: vgetexpps/d Vx,Wx (66),(ev) +43: vgetexpss/d Vx,Hx,Wx (66),(ev) +44: vplzcntd/q Vx,Wx (66),(ev) 45: vpsrlvd/q Vx,Hx,Wx (66),(v) -46: vpsravd Vx,Hx,Wx (66),(v) +46: vpsravd Vx,Hx,Wx (66),(v) | vpsravd/q Vx,Hx,Wx (66),(evo) 47: vpsllvd/q Vx,Hx,Wx (66),(v) -# Skip 0x48-0x57 +# Skip 0x48-0x4b +4c: vrcp14ps/d Vpd,Wpd (66),(ev) +4d: vrcp14ss/d Vsd,Hpd,Wsd (66),(ev) +4e: vrsqrt14ps/d Vpd,Wpd (66),(ev) +4f: vrsqrt14ss/d Vsd,Hsd,Wsd (66),(ev) +# Skip 0x50-0x57 58: vpbroadcastd Vx,Wx (66),(v) -59: vpbroadcastq Vx,Wx (66),(v) -5a: vbroadcasti128 Vqq,Mdq (66),(v) -# Skip 0x5b-0x77 +59: vpbroadcastq Vx,Wx (66),(v) | vbroadcasti32x2 Vx,Wx (66),(evo) +5a: vbroadcasti128 Vqq,Mdq (66),(v) | vbroadcasti32x4/64x2 Vx,Wx (66),(evo) +5b: vbroadcasti32x8/64x4 Vqq,Mdq (66),(ev) +# Skip 0x5c-0x63 +64: vpblendmd/q Vx,Hx,Wx (66),(ev) +65: vblendmps/d Vx,Hx,Wx (66),(ev) +66: vpblendmb/w Vx,Hx,Wx (66),(ev) +# Skip 0x67-0x74 +75: vpermi2b/w Vx,Hx,Wx (66),(ev) +76: vpermi2d/q Vx,Hx,Wx (66),(ev) +77: vpermi2ps/d Vx,Hx,Wx (66),(ev) 78: vpbroadcastb Vx,Wx (66),(v) 79: vpbroadcastw Vx,Wx (66),(v) -# Skip 0x7a-0x7f +7a: vpbroadcastb Vx,Rv (66),(ev) +7b: vpbroadcastw Vx,Rv (66),(ev) +7c: vpbroadcastd/q Vx,Rv (66),(ev) +7d: vpermt2b/w Vx,Hx,Wx (66),(ev) +7e: vpermt2d/q Vx,Hx,Wx (66),(ev) +7f: vpermt2ps/d Vx,Hx,Wx (66),(ev) 80: INVEPT Gy,Mdq (66) 81: INVPID Gy,Mdq (66) 82: INVPCID Gy,Mdq (66) +83: vpmultishiftqb Vx,Hx,Wx (66),(ev) +88: vexpandps/d Vpd,Wpd (66),(ev) +89: vpexpandd/q Vx,Wx (66),(ev) +8a: vcompressps/d Wx,Vx (66),(ev) +8b: vpcompressd/q Wx,Vx (66),(ev) 8c: vpmaskmovd/q Vx,Hx,Mx (66),(v) +8d: vpermb/w Vx,Hx,Wx (66),(ev) 8e: vpmaskmovd/q Mx,Vx,Hx (66),(v) # 0x0f 0x38 0x90-0xbf (FMA) -90: vgatherdd/q Vx,Hx,Wx (66),(v) -91: vgatherqd/q Vx,Hx,Wx (66),(v) +90: vgatherdd/q Vx,Hx,Wx (66),(v) | vpgatherdd/q Vx,Wx (66),(evo) +91: vgatherqd/q Vx,Hx,Wx (66),(v) | vpgatherqd/q Vx,Wx (66),(evo) 92: vgatherdps/d Vx,Hx,Wx (66),(v) 93: vgatherqps/d Vx,Hx,Wx (66),(v) 94: @@ -715,6 +744,10 @@ AVXcode: 2 9d: vfnmadd132ss/d Vx,Hx,Wx (66),(v),(v1) 9e: vfnmsub132ps/d Vx,Hx,Wx (66),(v) 9f: vfnmsub132ss/d Vx,Hx,Wx (66),(v),(v1) +a0: vpscatterdd/q Wx,Vx (66),(ev) +a1: vpscatterqd/q Wx,Vx (66),(ev) +a2: vscatterdps/d Wx,Vx (66),(ev) +a3: vscatterqps/d Wx,Vx (66),(ev) a6: vfmaddsub213ps/d Vx,Hx,Wx (66),(v) a7: vfmsubadd213ps/d Vx,Hx,Wx (66),(v) a8: vfmadd213ps/d Vx,Hx,Wx (66),(v) @@ -725,6 +758,8 @@ ac: vfnmadd213ps/d Vx,Hx,Wx (66),(v) ad: vfnmadd213ss/d Vx,Hx,Wx (66),(v),(v1) ae: vfnmsub213ps/d Vx,Hx,Wx (66),(v) af: vfnmsub213ss/d Vx,Hx,Wx (66),(v),(v1) +b4: vpmadd52luq Vx,Hx,Wx (66),(ev) +b5: vpmadd52huq Vx,Hx,Wx (66),(ev) b6: vfmaddsub231ps/d Vx,Hx,Wx (66),(v) b7: vfmsubadd231ps/d Vx,Hx,Wx (66),(v) b8: vfmadd231ps/d Vx,Hx,Wx (66),(v) @@ -736,12 +771,15 @@ bd: vfnmadd231ss/d Vx,Hx,Wx (66),(v),(v1) be: vfnmsub231ps/d Vx,Hx,Wx (66),(v) bf: vfnmsub231ss/d Vx,Hx,Wx (66),(v),(v1) # 0x0f 0x38 0xc0-0xff -c8: sha1nexte Vdq,Wdq +c4: vpconflictd/q Vx,Wx (66),(ev) +c6: Grp18 (1A) +c7: Grp19 (1A) +c8: sha1nexte Vdq,Wdq | vexp2ps/d Vx,Wx (66),(ev) c9: sha1msg1 Vdq,Wdq -ca: sha1msg2 Vdq,Wdq -cb: sha256rnds2 Vdq,Wdq -cc: sha256msg1 Vdq,Wdq -cd: sha256msg2 Vdq,Wdq +ca: sha1msg2 Vdq,Wdq | vrcp28ps/d Vx,Wx (66),(ev) +cb: sha256rnds2 Vdq,Wdq | vrcp28ss/d Vx,Hx,Wx (66),(ev) +cc: sha256msg1 Vdq,Wdq | vrsqrt28ps/d Vx,Wx (66),(ev) +cd: sha256msg2 Vdq,Wdq | vrsqrt28ss/d Vx,Hx,Wx (66),(ev) db: VAESIMC Vdq,Wdq (66),(v1) dc: VAESENC Vdq,Hdq,Wdq (66),(v1) dd: VAESENCLAST Vdq,Hdq,Wdq (66),(v1) @@ -763,15 +801,15 @@ AVXcode: 3 00: vpermq Vqq,Wqq,Ib (66),(v) 01: vpermpd Vqq,Wqq,Ib (66),(v) 02: vpblendd Vx,Hx,Wx,Ib (66),(v) -03: +03: valignd/q Vx,Hx,Wx,Ib (66),(ev) 04: vpermilps Vx,Wx,Ib (66),(v) 05: vpermilpd Vx,Wx,Ib (66),(v) 06: vperm2f128 Vqq,Hqq,Wqq,Ib (66),(v) 07: -08: vroundps Vx,Wx,Ib (66) -09: vroundpd Vx,Wx,Ib (66) -0a: vroundss Vss,Wss,Ib (66),(v1) -0b: vroundsd Vsd,Wsd,Ib (66),(v1) +08: vroundps Vx,Wx,Ib (66) | vrndscaleps Vx,Wx,Ib (66),(evo) +09: vroundpd Vx,Wx,Ib (66) | vrndscalepd Vx,Wx,Ib (66),(evo) +0a: vroundss Vss,Wss,Ib (66),(v1) | vrndscaless Vx,Hx,Wx,Ib (66),(evo) +0b: vroundsd Vsd,Wsd,Ib (66),(v1) | vrndscalesd Vx,Hx,Wx,Ib (66),(evo) 0c: vblendps Vx,Hx,Wx,Ib (66) 0d: vblendpd Vx,Hx,Wx,Ib (66) 0e: vpblendw Vx,Hx,Wx,Ib (66),(v1) @@ -780,26 +818,51 @@ AVXcode: 3 15: vpextrw Rd/Mw,Vdq,Ib (66),(v1) 16: vpextrd/q Ey,Vdq,Ib (66),(v1) 17: vextractps Ed,Vdq,Ib (66),(v1) -18: vinsertf128 Vqq,Hqq,Wqq,Ib (66),(v) -19: vextractf128 Wdq,Vqq,Ib (66),(v) +18: vinsertf128 Vqq,Hqq,Wqq,Ib (66),(v) | vinsertf32x4/64x2 Vqq,Hqq,Wqq,Ib (66),(evo) +19: vextractf128 Wdq,Vqq,Ib (66),(v) | vextractf32x4/64x2 Wdq,Vqq,Ib (66),(evo) +1a: vinsertf32x8/64x4 Vqq,Hqq,Wqq,Ib (66),(ev) +1b: vextractf32x8/64x4 Wdq,Vqq,Ib (66),(ev) 1d: vcvtps2ph Wx,Vx,Ib (66),(v) +1e: vpcmpud/q Vk,Hd,Wd,Ib (66),(ev) +1f: vpcmpd/q Vk,Hd,Wd,Ib (66),(ev) 20: vpinsrb Vdq,Hdq,Ry/Mb,Ib (66),(v1) 21: vinsertps Vdq,Hdq,Udq/Md,Ib (66),(v1) 22: vpinsrd/q Vdq,Hdq,Ey,Ib (66),(v1) -38: vinserti128 Vqq,Hqq,Wqq,Ib (66),(v) -39: vextracti128 Wdq,Vqq,Ib (66),(v) +23: vshuff32x4/64x2 Vx,Hx,Wx,Ib (66),(ev) +25: vpternlogd/q Vx,Hx,Wx,Ib (66),(ev) +26: vgetmantps/d Vx,Wx,Ib (66),(ev) +27: vgetmantss/d Vx,Hx,Wx,Ib (66),(ev) +30: kshiftrb/w Vk,Uk,Ib (66),(v) +31: kshiftrd/q Vk,Uk,Ib (66),(v) +32: kshiftlb/w Vk,Uk,Ib (66),(v) +33: kshiftld/q Vk,Uk,Ib (66),(v) +38: vinserti128 Vqq,Hqq,Wqq,Ib (66),(v) | vinserti32x4/64x2 Vqq,Hqq,Wqq,Ib (66),(evo) +39: vextracti128 Wdq,Vqq,Ib (66),(v) | vextracti32x4/64x2 Wdq,Vqq,Ib (66),(evo) +3a: vinserti32x8/64x4 Vqq,Hqq,Wqq,Ib (66),(ev) +3b: vextracti32x8/64x4 Wdq,Vqq,Ib (66),(ev) +3e: vpcmpub/w Vk,Hk,Wx,Ib (66),(ev) +3f: vpcmpb/w Vk,Hk,Wx,Ib (66),(ev) 40: vdpps Vx,Hx,Wx,Ib (66) 41: vdppd Vdq,Hdq,Wdq,Ib (66),(v1) -42: vmpsadbw Vx,Hx,Wx,Ib (66),(v1) +42: vmpsadbw Vx,Hx,Wx,Ib (66),(v1) | vdbpsadbw Vx,Hx,Wx,Ib (66),(evo) +43: vshufi32x4/64x2 Vx,Hx,Wx,Ib (66),(ev) 44: vpclmulqdq Vdq,Hdq,Wdq,Ib (66),(v1) 46: vperm2i128 Vqq,Hqq,Wqq,Ib (66),(v) 4a: vblendvps Vx,Hx,Wx,Lx (66),(v) 4b: vblendvpd Vx,Hx,Wx,Lx (66),(v) 4c: vpblendvb Vx,Hx,Wx,Lx (66),(v1) +50: vrangeps/d Vx,Hx,Wx,Ib (66),(ev) +51: vrangess/d Vx,Hx,Wx,Ib (66),(ev) +54: vfixupimmps/d Vx,Hx,Wx,Ib (66),(ev) +55: vfixupimmss/d Vx,Hx,Wx,Ib (66),(ev) +56: vreduceps/d Vx,Wx,Ib (66),(ev) +57: vreducess/d Vx,Hx,Wx,Ib (66),(ev) 60: vpcmpestrm Vdq,Wdq,Ib (66),(v1) 61: vpcmpestri Vdq,Wdq,Ib (66),(v1) 62: vpcmpistrm Vdq,Wdq,Ib (66),(v1) 63: vpcmpistri Vdq,Wdq,Ib (66),(v1) +66: vfpclassps/d Vk,Wx,Ib (66),(ev) +67: vfpclassss/d Vk,Wx,Ib (66),(ev) cc: sha1rnds4 Vdq,Wdq,Ib df: VAESKEYGEN Vdq,Wdq,Ib (66),(v1) f0: RORX Gy,Ey,Ib (F2),(v) @@ -927,8 +990,10 @@ GrpTable: Grp12 EndTable GrpTable: Grp13 +0: vprord/q Hx,Wx,Ib (66),(ev) +1: vprold/q Hx,Wx,Ib (66),(ev) 2: psrld Nq,Ib (11B) | vpsrld Hx,Ux,Ib (66),(11B),(v1) -4: psrad Nq,Ib (11B) | vpsrad Hx,Ux,Ib (66),(11B),(v1) +4: psrad Nq,Ib (11B) | vpsrad Hx,Ux,Ib (66),(11B),(v1) | vpsrad/q Hx,Ux,Ib (66),(evo) 6: pslld Nq,Ib (11B) | vpslld Hx,Ux,Ib (66),(11B),(v1) EndTable @@ -963,6 +1028,20 @@ GrpTable: Grp17 3: BLSI By,Ey (v) EndTable +GrpTable: Grp18 +1: vgatherpf0dps/d Wx (66),(ev) +2: vgatherpf1dps/d Wx (66),(ev) +5: vscatterpf0dps/d Wx (66),(ev) +6: vscatterpf1dps/d Wx (66),(ev) +EndTable + +GrpTable: Grp19 +1: vgatherpf0qps/d Wx (66),(ev) +2: vgatherpf1qps/d Wx (66),(ev) +5: vscatterpf0qps/d Wx (66),(ev) +6: vscatterpf1qps/d Wx (66),(ev) +EndTable + # AMD's Prefetch Group GrpTable: GrpP 0: PREFETCH diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 62c0043a5fd5..96d2b847e09e 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -37,4 +37,5 @@ obj-$(CONFIG_NUMA_EMU) += numa_emulation.o obj-$(CONFIG_X86_INTEL_MPX) += mpx.o obj-$(CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS) += pkeys.o +obj-$(CONFIG_RANDOMIZE_MEMORY) += kaslr.o diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c index 99bfb192803f..9a17250bcbe0 100644 --- a/arch/x86/mm/dump_pagetables.c +++ b/arch/x86/mm/dump_pagetables.c @@ -72,9 +72,9 @@ static struct addr_marker address_markers[] = { { 0, "User Space" }, #ifdef CONFIG_X86_64 { 0x8000000000000000UL, "Kernel Space" }, - { PAGE_OFFSET, "Low Kernel Mapping" }, - { VMALLOC_START, "vmalloc() Area" }, - { VMEMMAP_START, "Vmemmap" }, + { 0/* PAGE_OFFSET */, "Low Kernel Mapping" }, + { 0/* VMALLOC_START */, "vmalloc() Area" }, + { 0/* VMEMMAP_START */, "Vmemmap" }, # ifdef CONFIG_X86_ESPFIX64 { ESPFIX_BASE_ADDR, "ESPfix Area", 16 }, # endif @@ -434,8 +434,16 @@ void ptdump_walk_pgd_level_checkwx(void) static int __init pt_dump_init(void) { + /* + * Various markers are not compile-time constants, so assign them + * here. + */ +#ifdef CONFIG_X86_64 + address_markers[LOW_KERNEL_NR].start_address = PAGE_OFFSET; + address_markers[VMALLOC_START_NR].start_address = VMALLOC_START; + address_markers[VMEMMAP_START_NR].start_address = VMEMMAP_START; +#endif #ifdef CONFIG_X86_32 - /* Not a compile-time constant on x86-32 */ address_markers[VMALLOC_START_NR].start_address = VMALLOC_START; address_markers[VMALLOC_END_NR].start_address = VMALLOC_END; # ifdef CONFIG_HIGHMEM diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 4bb53b89f3c5..832b98f822be 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -1,6 +1,7 @@ #include #include #include +#include typedef bool (*ex_handler_t)(const struct exception_table_entry *, struct pt_regs *, int); @@ -37,7 +38,7 @@ bool ex_handler_ext(const struct exception_table_entry *fixup, struct pt_regs *regs, int trapnr) { /* Special hack for uaccess_err */ - current_thread_info()->uaccess_err = 1; + current->thread.uaccess_err = 1; regs->ip = ex_fixup_addr(fixup); return true; } @@ -46,8 +47,9 @@ EXPORT_SYMBOL(ex_handler_ext); bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, struct pt_regs *regs, int trapnr) { - WARN_ONCE(1, "unchecked MSR access error: RDMSR from 0x%x\n", - (unsigned int)regs->cx); + if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pF)\n", + (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) + show_stack_regs(regs); /* Pretend that the read succeeded and returned 0. */ regs->ip = ex_fixup_addr(fixup); @@ -60,9 +62,10 @@ EXPORT_SYMBOL(ex_handler_rdmsr_unsafe); bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup, struct pt_regs *regs, int trapnr) { - WARN_ONCE(1, "unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x)\n", - (unsigned int)regs->cx, - (unsigned int)regs->dx, (unsigned int)regs->ax); + if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pF)\n", + (unsigned int)regs->cx, (unsigned int)regs->dx, + (unsigned int)regs->ax, regs->ip, (void *)regs->ip)) + show_stack_regs(regs); /* Pretend that the write succeeded. */ regs->ip = ex_fixup_addr(fixup); diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 7d1fa7cd2374..dc8023060456 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -439,7 +439,7 @@ static noinline int vmalloc_fault(unsigned long address) * happen within a race in page table update. In the later * case just flush: */ - pgd = pgd_offset(current->active_mm, address); + pgd = (pgd_t *)__va(read_cr3()) + pgd_index(address); pgd_ref = pgd_offset_k(address); if (pgd_none(*pgd_ref)) return -1; @@ -737,7 +737,7 @@ no_context(struct pt_regs *regs, unsigned long error_code, * In this case we need to make sure we're not recursively * faulting through the emulate_vsyscall() logic. */ - if (current_thread_info()->sig_on_uaccess_error && signal) { + if (current->thread.sig_on_uaccess_err && signal) { tsk->thread.trap_nr = X86_TRAP_PF; tsk->thread.error_code = error_code | PF_USER; tsk->thread.cr2 = address; @@ -1353,7 +1353,7 @@ good_area: * the fault. Since we never set FAULT_FLAG_RETRY_NOWAIT, if * we get VM_FAULT_RETRY back, the mmap_sem has been unlocked. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); major |= fault & VM_FAULT_MAJOR; /* diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 372aad2b3291..cc82830bc8c4 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -17,6 +17,7 @@ #include #include /* for MAX_DMA_PFN */ #include +#include /* * We need to define the tracepoints somewhere, and tlb.c @@ -590,6 +591,9 @@ void __init init_mem_mapping(void) /* the ISA range is always mapped regardless of memory holes */ init_memory_mapping(0, ISA_END_ADDRESS); + /* Init the trampoline, possibly with KASLR memory offset */ + init_trampoline(); + /* * If the allocation is in bottom-up direction, we setup direct mapping * in bottom-up, otherwise we setup direct mapping in top-down. diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index bce2e5d9edd4..53cc2256cf23 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -328,22 +328,30 @@ void __init cleanup_highmap(void) } } +/* + * Create PTE level page table mapping for physical addresses. + * It returns the last physical address mapped. + */ static unsigned long __meminit -phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end, +phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end, pgprot_t prot) { - unsigned long pages = 0, next; - unsigned long last_map_addr = end; + unsigned long pages = 0, paddr_next; + unsigned long paddr_last = paddr_end; + pte_t *pte; int i; - pte_t *pte = pte_page + pte_index(addr); + pte = pte_page + pte_index(paddr); + i = pte_index(paddr); - for (i = pte_index(addr); i < PTRS_PER_PTE; i++, addr = next, pte++) { - next = (addr & PAGE_MASK) + PAGE_SIZE; - if (addr >= end) { + for (; i < PTRS_PER_PTE; i++, paddr = paddr_next, pte++) { + paddr_next = (paddr & PAGE_MASK) + PAGE_SIZE; + if (paddr >= paddr_end) { if (!after_bootmem && - !e820_any_mapped(addr & PAGE_MASK, next, E820_RAM) && - !e820_any_mapped(addr & PAGE_MASK, next, E820_RESERVED_KERN)) + !e820_any_mapped(paddr & PAGE_MASK, paddr_next, + E820_RAM) && + !e820_any_mapped(paddr & PAGE_MASK, paddr_next, + E820_RESERVED_KERN)) set_pte(pte, __pte(0)); continue; } @@ -354,54 +362,61 @@ phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end, * pagetable pages as RO. So assume someone who pre-setup * these mappings are more intelligent. */ - if (pte_val(*pte)) { + if (!pte_none(*pte)) { if (!after_bootmem) pages++; continue; } if (0) - printk(" pte=%p addr=%lx pte=%016lx\n", - pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte); + pr_info(" pte=%p addr=%lx pte=%016lx\n", pte, paddr, + pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL).pte); pages++; - set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, prot)); - last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE; + set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot)); + paddr_last = (paddr & PAGE_MASK) + PAGE_SIZE; } update_page_count(PG_LEVEL_4K, pages); - return last_map_addr; + return paddr_last; } +/* + * Create PMD level page table mapping for physical addresses. The virtual + * and physical address have to be aligned at this level. + * It returns the last physical address mapped. + */ static unsigned long __meminit -phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, +phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end, unsigned long page_size_mask, pgprot_t prot) { - unsigned long pages = 0, next; - unsigned long last_map_addr = end; + unsigned long pages = 0, paddr_next; + unsigned long paddr_last = paddr_end; - int i = pmd_index(address); + int i = pmd_index(paddr); - for (; i < PTRS_PER_PMD; i++, address = next) { - pmd_t *pmd = pmd_page + pmd_index(address); + for (; i < PTRS_PER_PMD; i++, paddr = paddr_next) { + pmd_t *pmd = pmd_page + pmd_index(paddr); pte_t *pte; pgprot_t new_prot = prot; - next = (address & PMD_MASK) + PMD_SIZE; - if (address >= end) { + paddr_next = (paddr & PMD_MASK) + PMD_SIZE; + if (paddr >= paddr_end) { if (!after_bootmem && - !e820_any_mapped(address & PMD_MASK, next, E820_RAM) && - !e820_any_mapped(address & PMD_MASK, next, E820_RESERVED_KERN)) + !e820_any_mapped(paddr & PMD_MASK, paddr_next, + E820_RAM) && + !e820_any_mapped(paddr & PMD_MASK, paddr_next, + E820_RESERVED_KERN)) set_pmd(pmd, __pmd(0)); continue; } - if (pmd_val(*pmd)) { + if (!pmd_none(*pmd)) { if (!pmd_large(*pmd)) { spin_lock(&init_mm.page_table_lock); pte = (pte_t *)pmd_page_vaddr(*pmd); - last_map_addr = phys_pte_init(pte, address, - end, prot); + paddr_last = phys_pte_init(pte, paddr, + paddr_end, prot); spin_unlock(&init_mm.page_table_lock); continue; } @@ -420,7 +435,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, if (page_size_mask & (1 << PG_LEVEL_2M)) { if (!after_bootmem) pages++; - last_map_addr = next; + paddr_last = paddr_next; continue; } new_prot = pte_pgprot(pte_clrhuge(*(pte_t *)pmd)); @@ -430,51 +445,65 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, pages++; spin_lock(&init_mm.page_table_lock); set_pte((pte_t *)pmd, - pfn_pte((address & PMD_MASK) >> PAGE_SHIFT, + pfn_pte((paddr & PMD_MASK) >> PAGE_SHIFT, __pgprot(pgprot_val(prot) | _PAGE_PSE))); spin_unlock(&init_mm.page_table_lock); - last_map_addr = next; + paddr_last = paddr_next; continue; } pte = alloc_low_page(); - last_map_addr = phys_pte_init(pte, address, end, new_prot); + paddr_last = phys_pte_init(pte, paddr, paddr_end, new_prot); spin_lock(&init_mm.page_table_lock); pmd_populate_kernel(&init_mm, pmd, pte); spin_unlock(&init_mm.page_table_lock); } update_page_count(PG_LEVEL_2M, pages); - return last_map_addr; + return paddr_last; } +/* + * Create PUD level page table mapping for physical addresses. The virtual + * and physical address do not have to be aligned at this level. KASLR can + * randomize virtual addresses up to this level. + * It returns the last physical address mapped. + */ static unsigned long __meminit -phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, - unsigned long page_size_mask) +phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end, + unsigned long page_size_mask) { - unsigned long pages = 0, next; - unsigned long last_map_addr = end; - int i = pud_index(addr); + unsigned long pages = 0, paddr_next; + unsigned long paddr_last = paddr_end; + unsigned long vaddr = (unsigned long)__va(paddr); + int i = pud_index(vaddr); - for (; i < PTRS_PER_PUD; i++, addr = next) { - pud_t *pud = pud_page + pud_index(addr); + for (; i < PTRS_PER_PUD; i++, paddr = paddr_next) { + pud_t *pud; pmd_t *pmd; pgprot_t prot = PAGE_KERNEL; - next = (addr & PUD_MASK) + PUD_SIZE; - if (addr >= end) { + vaddr = (unsigned long)__va(paddr); + pud = pud_page + pud_index(vaddr); + paddr_next = (paddr & PUD_MASK) + PUD_SIZE; + + if (paddr >= paddr_end) { if (!after_bootmem && - !e820_any_mapped(addr & PUD_MASK, next, E820_RAM) && - !e820_any_mapped(addr & PUD_MASK, next, E820_RESERVED_KERN)) + !e820_any_mapped(paddr & PUD_MASK, paddr_next, + E820_RAM) && + !e820_any_mapped(paddr & PUD_MASK, paddr_next, + E820_RESERVED_KERN)) set_pud(pud, __pud(0)); continue; } - if (pud_val(*pud)) { + if (!pud_none(*pud)) { if (!pud_large(*pud)) { pmd = pmd_offset(pud, 0); - last_map_addr = phys_pmd_init(pmd, addr, end, - page_size_mask, prot); + paddr_last = phys_pmd_init(pmd, paddr, + paddr_end, + page_size_mask, + prot); __flush_tlb_all(); continue; } @@ -493,7 +522,7 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, if (page_size_mask & (1 << PG_LEVEL_1G)) { if (!after_bootmem) pages++; - last_map_addr = next; + paddr_last = paddr_next; continue; } prot = pte_pgprot(pte_clrhuge(*(pte_t *)pud)); @@ -503,16 +532,16 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, pages++; spin_lock(&init_mm.page_table_lock); set_pte((pte_t *)pud, - pfn_pte((addr & PUD_MASK) >> PAGE_SHIFT, + pfn_pte((paddr & PUD_MASK) >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); spin_unlock(&init_mm.page_table_lock); - last_map_addr = next; + paddr_last = paddr_next; continue; } pmd = alloc_low_page(); - last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask, - prot); + paddr_last = phys_pmd_init(pmd, paddr, paddr_end, + page_size_mask, prot); spin_lock(&init_mm.page_table_lock); pud_populate(&init_mm, pud, pmd); @@ -522,38 +551,44 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, update_page_count(PG_LEVEL_1G, pages); - return last_map_addr; + return paddr_last; } +/* + * Create page table mapping for the physical memory for specific physical + * addresses. The virtual and physical addresses have to be aligned on PMD level + * down. It returns the last physical address mapped. + */ unsigned long __meminit -kernel_physical_mapping_init(unsigned long start, - unsigned long end, +kernel_physical_mapping_init(unsigned long paddr_start, + unsigned long paddr_end, unsigned long page_size_mask) { bool pgd_changed = false; - unsigned long next, last_map_addr = end; - unsigned long addr; + unsigned long vaddr, vaddr_start, vaddr_end, vaddr_next, paddr_last; - start = (unsigned long)__va(start); - end = (unsigned long)__va(end); - addr = start; + paddr_last = paddr_end; + vaddr = (unsigned long)__va(paddr_start); + vaddr_end = (unsigned long)__va(paddr_end); + vaddr_start = vaddr; - for (; start < end; start = next) { - pgd_t *pgd = pgd_offset_k(start); + for (; vaddr < vaddr_end; vaddr = vaddr_next) { + pgd_t *pgd = pgd_offset_k(vaddr); pud_t *pud; - next = (start & PGDIR_MASK) + PGDIR_SIZE; + vaddr_next = (vaddr & PGDIR_MASK) + PGDIR_SIZE; if (pgd_val(*pgd)) { pud = (pud_t *)pgd_page_vaddr(*pgd); - last_map_addr = phys_pud_init(pud, __pa(start), - __pa(end), page_size_mask); + paddr_last = phys_pud_init(pud, __pa(vaddr), + __pa(vaddr_end), + page_size_mask); continue; } pud = alloc_low_page(); - last_map_addr = phys_pud_init(pud, __pa(start), __pa(end), - page_size_mask); + paddr_last = phys_pud_init(pud, __pa(vaddr), __pa(vaddr_end), + page_size_mask); spin_lock(&init_mm.page_table_lock); pgd_populate(&init_mm, pgd, pud); @@ -562,11 +597,11 @@ kernel_physical_mapping_init(unsigned long start, } if (pgd_changed) - sync_global_pgds(addr, end - 1, 0); + sync_global_pgds(vaddr_start, vaddr_end - 1, 0); __flush_tlb_all(); - return last_map_addr; + return paddr_last; } #ifndef CONFIG_NUMA @@ -673,7 +708,7 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) for (i = 0; i < PTRS_PER_PTE; i++) { pte = pte_start + i; - if (pte_val(*pte)) + if (!pte_none(*pte)) return; } @@ -691,7 +726,7 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud) for (i = 0; i < PTRS_PER_PMD; i++) { pmd = pmd_start + i; - if (pmd_val(*pmd)) + if (!pmd_none(*pmd)) return; } @@ -702,27 +737,6 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud) spin_unlock(&init_mm.page_table_lock); } -/* Return true if pgd is changed, otherwise return false. */ -static bool __meminit free_pud_table(pud_t *pud_start, pgd_t *pgd) -{ - pud_t *pud; - int i; - - for (i = 0; i < PTRS_PER_PUD; i++) { - pud = pud_start + i; - if (pud_val(*pud)) - return false; - } - - /* free a pud table */ - free_pagetable(pgd_page(*pgd), 0); - spin_lock(&init_mm.page_table_lock); - pgd_clear(pgd); - spin_unlock(&init_mm.page_table_lock); - - return true; -} - static void __meminit remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end, bool direct) @@ -913,7 +927,6 @@ remove_pagetable(unsigned long start, unsigned long end, bool direct) unsigned long addr; pgd_t *pgd; pud_t *pud; - bool pgd_changed = false; for (addr = start; addr < end; addr = next) { next = pgd_addr_end(addr, end); @@ -924,13 +937,8 @@ remove_pagetable(unsigned long start, unsigned long end, bool direct) pud = (pud_t *)pgd_page_vaddr(*pgd); remove_pud_table(pud, addr, next, direct); - if (free_pud_table(pud, pgd)) - pgd_changed = true; } - if (pgd_changed) - sync_global_pgds(start, end - 1, 1); - flush_tlb_all(); } diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c index 1b1110fa0057..0493c17b8a51 100644 --- a/arch/x86/mm/kasan_init_64.c +++ b/arch/x86/mm/kasan_init_64.c @@ -54,8 +54,8 @@ static int kasan_die_handler(struct notifier_block *self, void *data) { if (val == DIE_GPF) { - pr_emerg("CONFIG_KASAN_INLINE enabled"); - pr_emerg("GPF could be caused by NULL-ptr deref or user memory access"); + pr_emerg("CONFIG_KASAN_INLINE enabled\n"); + pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n"); } return NOTIFY_OK; } diff --git a/arch/x86/mm/kaslr.c b/arch/x86/mm/kaslr.c new file mode 100644 index 000000000000..26dccd6c0df1 --- /dev/null +++ b/arch/x86/mm/kaslr.c @@ -0,0 +1,172 @@ +/* + * This file implements KASLR memory randomization for x86_64. It randomizes + * the virtual address space of kernel memory regions (physical memory + * mapping, vmalloc & vmemmap) for x86_64. This security feature mitigates + * exploits relying on predictable kernel addresses. + * + * Entropy is generated using the KASLR early boot functions now shared in + * the lib directory (originally written by Kees Cook). Randomization is + * done on PGD & PUD page table levels to increase possible addresses. The + * physical memory mapping code was adapted to support PUD level virtual + * addresses. This implementation on the best configuration provides 30,000 + * possible virtual addresses in average for each memory region. An additional + * low memory page is used to ensure each CPU can start with a PGD aligned + * virtual address (for realmode). + * + * The order of each memory region is not changed. The feature looks at + * the available space for the regions based on different configuration + * options and randomizes the base and space between each. The size of the + * physical memory mapping is the available physical memory. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "mm_internal.h" + +#define TB_SHIFT 40 + +/* + * Virtual address start and end range for randomization. The end changes base + * on configuration to have the highest amount of space for randomization. + * It increases the possible random position for each randomized region. + * + * You need to add an if/def entry if you introduce a new memory region + * compatible with KASLR. Your entry must be in logical order with memory + * layout. For example, ESPFIX is before EFI because its virtual address is + * before. You also need to add a BUILD_BUG_ON in kernel_randomize_memory to + * ensure that this order is correct and won't be changed. + */ +static const unsigned long vaddr_start = __PAGE_OFFSET_BASE; +static const unsigned long vaddr_end = VMEMMAP_START; + +/* Default values */ +unsigned long page_offset_base = __PAGE_OFFSET_BASE; +EXPORT_SYMBOL(page_offset_base); +unsigned long vmalloc_base = __VMALLOC_BASE; +EXPORT_SYMBOL(vmalloc_base); + +/* + * Memory regions randomized by KASLR (except modules that use a separate logic + * earlier during boot). The list is ordered based on virtual addresses. This + * order is kept after randomization. + */ +static __initdata struct kaslr_memory_region { + unsigned long *base; + unsigned long size_tb; +} kaslr_regions[] = { + { &page_offset_base, 64/* Maximum */ }, + { &vmalloc_base, VMALLOC_SIZE_TB }, +}; + +/* Get size in bytes used by the memory region */ +static inline unsigned long get_padding(struct kaslr_memory_region *region) +{ + return (region->size_tb << TB_SHIFT); +} + +/* + * Apply no randomization if KASLR was disabled at boot or if KASAN + * is enabled. KASAN shadow mappings rely on regions being PGD aligned. + */ +static inline bool kaslr_memory_enabled(void) +{ + return kaslr_enabled() && !config_enabled(CONFIG_KASAN); +} + +/* Initialize base and padding for each memory region randomized with KASLR */ +void __init kernel_randomize_memory(void) +{ + size_t i; + unsigned long vaddr = vaddr_start; + unsigned long rand, memory_tb; + struct rnd_state rand_state; + unsigned long remain_entropy; + + if (!kaslr_memory_enabled()) + return; + + /* + * Update Physical memory mapping to available and + * add padding if needed (especially for memory hotplug support). + */ + BUG_ON(kaslr_regions[0].base != &page_offset_base); + memory_tb = ((max_pfn << PAGE_SHIFT) >> TB_SHIFT) + + CONFIG_RANDOMIZE_MEMORY_PHYSICAL_PADDING; + + /* Adapt phyiscal memory region size based on available memory */ + if (memory_tb < kaslr_regions[0].size_tb) + kaslr_regions[0].size_tb = memory_tb; + + /* Calculate entropy available between regions */ + remain_entropy = vaddr_end - vaddr_start; + for (i = 0; i < ARRAY_SIZE(kaslr_regions); i++) + remain_entropy -= get_padding(&kaslr_regions[i]); + + prandom_seed_state(&rand_state, kaslr_get_random_long("Memory")); + + for (i = 0; i < ARRAY_SIZE(kaslr_regions); i++) { + unsigned long entropy; + + /* + * Select a random virtual address using the extra entropy + * available. + */ + entropy = remain_entropy / (ARRAY_SIZE(kaslr_regions) - i); + prandom_bytes_state(&rand_state, &rand, sizeof(rand)); + entropy = (rand % (entropy + 1)) & PUD_MASK; + vaddr += entropy; + *kaslr_regions[i].base = vaddr; + + /* + * Jump the region and add a minimum padding based on + * randomization alignment. + */ + vaddr += get_padding(&kaslr_regions[i]); + vaddr = round_up(vaddr + 1, PUD_SIZE); + remain_entropy -= entropy; + } +} + +/* + * Create PGD aligned trampoline table to allow real mode initialization + * of additional CPUs. Consume only 1 low memory page. + */ +void __meminit init_trampoline(void) +{ + unsigned long paddr, paddr_next; + pgd_t *pgd; + pud_t *pud_page, *pud_page_tramp; + int i; + + if (!kaslr_memory_enabled()) { + init_trampoline_default(); + return; + } + + pud_page_tramp = alloc_low_page(); + + paddr = 0; + pgd = pgd_offset_k((unsigned long)__va(paddr)); + pud_page = (pud_t *) pgd_page_vaddr(*pgd); + + for (i = pud_index(paddr); i < PTRS_PER_PUD; i++, paddr = paddr_next) { + pud_t *pud, *pud_tramp; + unsigned long vaddr = (unsigned long)__va(paddr); + + pud_tramp = pud_page_tramp + pud_index(paddr); + pud = pud_page + pud_index(vaddr); + paddr_next = (paddr & PUD_MASK) + PUD_SIZE; + + *pud_tramp = *pud; + } + + set_pgd(&trampoline_pgd_entry, + __pgd(_KERNPG_TABLE | __pa(pud_page_tramp))); +} diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 9c086c57105c..968ac028c34e 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -1,4 +1,5 @@ /* Common code for 32 and 64-bit NUMA */ +#include #include #include #include @@ -15,7 +16,6 @@ #include #include #include -#include #include #include "numa_internal.h" diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 7a1f7bbf4105..849dc09fa4f0 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -101,7 +101,8 @@ static inline unsigned long highmap_start_pfn(void) static inline unsigned long highmap_end_pfn(void) { - return __pa_symbol(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT; + /* Do not reference physical address outside the kernel. */ + return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT; } #endif @@ -112,6 +113,12 @@ within(unsigned long addr, unsigned long start, unsigned long end) return addr >= start && addr < end; } +static inline int +within_inclusive(unsigned long addr, unsigned long start, unsigned long end) +{ + return addr >= start && addr <= end; +} + /* * Flushing functions */ @@ -746,18 +753,6 @@ static bool try_to_free_pmd_page(pmd_t *pmd) return true; } -static bool try_to_free_pud_page(pud_t *pud) -{ - int i; - - for (i = 0; i < PTRS_PER_PUD; i++) - if (!pud_none(pud[i])) - return false; - - free_page((unsigned long)pud); - return true; -} - static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end) { pte_t *pte = pte_offset_kernel(pmd, start); @@ -871,16 +866,6 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end) */ } -static void unmap_pgd_range(pgd_t *root, unsigned long addr, unsigned long end) -{ - pgd_t *pgd_entry = root + pgd_index(addr); - - unmap_pud_range(pgd_entry, addr, end); - - if (try_to_free_pud_page((pud_t *)pgd_page_vaddr(*pgd_entry))) - pgd_clear(pgd_entry); -} - static int alloc_pte_page(pmd_t *pmd) { pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK); @@ -1113,7 +1098,12 @@ static int populate_pgd(struct cpa_data *cpa, unsigned long addr) ret = populate_pud(cpa, addr, pgd_entry, pgprot); if (ret < 0) { - unmap_pgd_range(cpa->pgd, addr, + /* + * Leave the PUD page in place in case some other CPU or thread + * already found it, but remove any useless entries we just + * added to it. + */ + unmap_pud_range(pgd_entry, addr, addr + (cpa->numpages << PAGE_SHIFT)); return ret; } @@ -1185,7 +1175,7 @@ repeat: return __cpa_process_fault(cpa, address, primary); old_pte = *kpte; - if (!pte_val(old_pte)) + if (pte_none(old_pte)) return __cpa_process_fault(cpa, address, primary); if (level == PG_LEVEL_4K) { @@ -1316,7 +1306,8 @@ static int cpa_process_alias(struct cpa_data *cpa) * to touch the high mapped kernel as well: */ if (!within(vaddr, (unsigned long)_text, _brk_end) && - within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn())) { + within_inclusive(cpa->pfn, highmap_start_pfn(), + highmap_end_pfn())) { unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base; alias_cpa = *cpa; @@ -1991,12 +1982,6 @@ out: return retval; } -void kernel_unmap_pages_in_pgd(pgd_t *root, unsigned long address, - unsigned numpages) -{ - unmap_pgd_range(root, address, address + (numpages << PAGE_SHIFT)); -} - /* * The testcases use internal knowledge of the implementation that shouldn't * be exposed to the rest of the kernel. Include these directly here. diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c index fb0604f11eec..db00e3e2f3dc 100644 --- a/arch/x86/mm/pat.c +++ b/arch/x86/mm/pat.c @@ -755,11 +755,8 @@ static inline int range_is_allowed(unsigned long pfn, unsigned long size) return 1; while (cursor < to) { - if (!devmem_is_allowed(pfn)) { - pr_info("x86/PAT: Program %s tried to access /dev/mem between [mem %#010Lx-%#010Lx], PAT prevents it\n", - current->comm, from, to - 1); + if (!devmem_is_allowed(pfn)) return 0; - } cursor += PAGE_SIZE; pfn++; } diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index 4eb287e25043..3feec5af4e67 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -6,7 +6,7 @@ #include #include -#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO +#define PGALLOC_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | __GFP_ZERO) #ifdef CONFIG_HIGHPTE #define PGALLOC_USER_GFP __GFP_HIGHMEM @@ -18,7 +18,7 @@ gfp_t __userpte_alloc_gfp = PGALLOC_GFP | PGALLOC_USER_GFP; pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - return (pte_t *)__get_free_page(PGALLOC_GFP); + return (pte_t *)__get_free_page(PGALLOC_GFP & ~__GFP_ACCOUNT); } pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) @@ -207,9 +207,13 @@ static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[]) { int i; bool failed = false; + gfp_t gfp = PGALLOC_GFP; + + if (mm == &init_mm) + gfp &= ~__GFP_ACCOUNT; for(i = 0; i < PREALLOCATED_PMDS; i++) { - pmd_t *pmd = (pmd_t *)__get_free_page(PGALLOC_GFP); + pmd_t *pmd = (pmd_t *)__get_free_page(gfp); if (!pmd) failed = true; if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) { diff --git a/arch/x86/mm/pgtable_32.c b/arch/x86/mm/pgtable_32.c index 75cc0978d45d..e67ae0e6c59d 100644 --- a/arch/x86/mm/pgtable_32.c +++ b/arch/x86/mm/pgtable_32.c @@ -47,7 +47,7 @@ void set_pte_vaddr(unsigned long vaddr, pte_t pteval) return; } pte = pte_offset_kernel(pmd, vaddr); - if (pte_val(pteval)) + if (!pte_none(pteval)) set_pte_at(&init_mm, vaddr, pte, pteval); else pte_clear(&init_mm, vaddr, pte); diff --git a/arch/x86/mm/srat.c b/arch/x86/mm/srat.c index b5f821881465..b1ecff460a46 100644 --- a/arch/x86/mm/srat.c +++ b/arch/x86/mm/srat.c @@ -15,8 +15,6 @@ #include #include #include -#include -#include #include #include #include @@ -24,51 +22,6 @@ #include #include -int acpi_numa __initdata; - -static __init int setup_node(int pxm) -{ - return acpi_map_pxm_to_node(pxm); -} - -static __init void bad_srat(void) -{ - printk(KERN_ERR "SRAT: SRAT not used.\n"); - acpi_numa = -1; -} - -static __init inline int srat_disabled(void) -{ - return acpi_numa < 0; -} - -/* - * Callback for SLIT parsing. pxm_to_node() returns NUMA_NO_NODE for - * I/O localities since SRAT does not list them. I/O localities are - * not supported at this point. - */ -void __init acpi_numa_slit_init(struct acpi_table_slit *slit) -{ - int i, j; - - for (i = 0; i < slit->locality_count; i++) { - const int from_node = pxm_to_node(i); - - if (from_node == NUMA_NO_NODE) - continue; - - for (j = 0; j < slit->locality_count; j++) { - const int to_node = pxm_to_node(j); - - if (to_node == NUMA_NO_NODE) - continue; - - numa_set_distance(from_node, to_node, - slit->entry[slit->locality_count * i + j]); - } - } -} - /* Callback for Proximity Domain -> x2APIC mapping */ void __init acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) @@ -91,7 +44,7 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) pxm, apic_id); return; } - node = setup_node(pxm); + node = acpi_map_pxm_to_node(pxm); if (node < 0) { printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm); bad_srat(); @@ -104,7 +57,6 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) } set_apicid_to_node(apic_id, node); node_set(node, numa_nodes_parsed); - acpi_numa = 1; printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n", pxm, apic_id, node); } @@ -127,7 +79,7 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) pxm = pa->proximity_domain_lo; if (acpi_srat_revision >= 2) pxm |= *((unsigned int*)pa->proximity_domain_hi) << 8; - node = setup_node(pxm); + node = acpi_map_pxm_to_node(pxm); if (node < 0) { printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm); bad_srat(); @@ -146,74 +98,10 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) set_apicid_to_node(apic_id, node); node_set(node, numa_nodes_parsed); - acpi_numa = 1; printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n", pxm, apic_id, node); } -#ifdef CONFIG_MEMORY_HOTPLUG -static inline int save_add_info(void) {return 1;} -#else -static inline int save_add_info(void) {return 0;} -#endif - -/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */ -int __init -acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) -{ - u64 start, end; - u32 hotpluggable; - int node, pxm; - - if (srat_disabled()) - goto out_err; - if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) - goto out_err_bad_srat; - if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0) - goto out_err; - hotpluggable = ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE; - if (hotpluggable && !save_add_info()) - goto out_err; - - start = ma->base_address; - end = start + ma->length; - pxm = ma->proximity_domain; - if (acpi_srat_revision <= 1) - pxm &= 0xff; - - node = setup_node(pxm); - if (node < 0) { - printk(KERN_ERR "SRAT: Too many proximity domains.\n"); - goto out_err_bad_srat; - } - - if (numa_add_memblk(node, start, end) < 0) - goto out_err_bad_srat; - - node_set(node, numa_nodes_parsed); - - pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n", - node, pxm, - (unsigned long long) start, (unsigned long long) end - 1, - hotpluggable ? " hotplug" : "", - ma->flags & ACPI_SRAT_MEM_NON_VOLATILE ? " non-volatile" : ""); - - /* Mark hotplug range in memblock. */ - if (hotpluggable && memblock_mark_hotplug(start, ma->length)) - pr_warn("SRAT: Failed to mark hotplug range [mem %#010Lx-%#010Lx] in memblock\n", - (unsigned long long)start, (unsigned long long)end - 1); - - max_possible_pfn = max(max_possible_pfn, PFN_UP(end - 1)); - - return 0; -out_err_bad_srat: - bad_srat(); -out_err: - return -1; -} - -void __init acpi_numa_arch_fixup(void) {} - int __init x86_acpi_numa_init(void) { int ret; diff --git a/arch/x86/pci/acpi.c b/arch/x86/pci/acpi.c index b2a4e2a61f6b..3cd69832d7f4 100644 --- a/arch/x86/pci/acpi.c +++ b/arch/x86/pci/acpi.c @@ -396,6 +396,7 @@ int __init pci_acpi_init(void) return -ENODEV; printk(KERN_INFO "PCI: Using ACPI for IRQ routing\n"); + acpi_irq_penalty_init(); pcibios_enable_irq = acpi_pci_irq_enable; pcibios_disable_irq = acpi_pci_irq_disable; x86_init.pci.init_irq = x86_init_noop; diff --git a/arch/x86/pci/intel_mid_pci.c b/arch/x86/pci/intel_mid_pci.c index 8b93e634af84..5a18aedcb341 100644 --- a/arch/x86/pci/intel_mid_pci.c +++ b/arch/x86/pci/intel_mid_pci.c @@ -36,7 +36,8 @@ #define PCIE_CAP_OFFSET 0x100 /* Quirks for the listed devices */ -#define PCI_DEVICE_ID_INTEL_MRFL_MMC 0x1190 +#define PCI_DEVICE_ID_INTEL_MRFLD_MMC 0x1190 +#define PCI_DEVICE_ID_INTEL_MRFLD_HSU 0x1191 /* Fixed BAR fields */ #define PCIE_VNDR_CAP_ID_FIXED_BAR 0x00 /* Fixed BAR (TBD) */ @@ -224,14 +225,21 @@ static int intel_mid_pci_irq_enable(struct pci_dev *dev) /* Special treatment for IRQ0 */ if (dev->irq == 0) { + /* + * Skip HS UART common registers device since it has + * IRQ0 assigned and not used by the kernel. + */ + if (dev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU) + return -EBUSY; /* * TNG has IRQ0 assigned to eMMC controller. But there * are also other devices with bogus PCI configuration * that have IRQ0 assigned. This check ensures that - * eMMC gets it. + * eMMC gets it. The rest of devices still could be + * enabled without interrupt line being allocated. */ - if (dev->device != PCI_DEVICE_ID_INTEL_MRFL_MMC) - return -EBUSY; + if (dev->device != PCI_DEVICE_ID_INTEL_MRFLD_MMC) + return 0; } break; default: @@ -308,14 +316,39 @@ static void pci_d3delay_fixup(struct pci_dev *dev) } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, pci_d3delay_fixup); -static void mrst_power_off_unused_dev(struct pci_dev *dev) +static void mid_power_off_one_device(struct pci_dev *dev) { + u16 pmcsr; + + /* + * Update current state first, otherwise PCI core enforces PCI_D0 in + * pci_set_power_state() for devices which status was PCI_UNKNOWN. + */ + pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); + dev->current_state = (pci_power_t __force)(pmcsr & PCI_PM_CTRL_STATE_MASK); + pci_set_power_state(dev, PCI_D3hot); } -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0801, mrst_power_off_unused_dev); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0809, mrst_power_off_unused_dev); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x080C, mrst_power_off_unused_dev); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x0815, mrst_power_off_unused_dev); + +static void mid_power_off_devices(struct pci_dev *dev) +{ + int id; + + if (!pci_soc_mode) + return; + + id = intel_mid_pwr_get_lss_id(dev); + if (id < 0) + return; + + /* + * This sets only PMCSR bits. The actual power off will happen in + * arch/x86/platform/intel-mid/pwr.c. + */ + mid_power_off_one_device(dev); +} + +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, mid_power_off_devices); /* * Langwell devices reside at fixed offsets, don't try to move them. diff --git a/arch/x86/pci/vmd.c b/arch/x86/pci/vmd.c index 7792aba266df..613cac7395c4 100644 --- a/arch/x86/pci/vmd.c +++ b/arch/x86/pci/vmd.c @@ -195,7 +195,7 @@ static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info, vmdirq->virq = virq; irq_domain_set_info(domain, virq, vmdirq->irq->vmd_vector, info->chip, - vmdirq, handle_simple_irq, vmd, NULL); + vmdirq, handle_untracked_irq, vmd, NULL); return 0; } diff --git a/arch/x86/platform/atom/punit_atom_debug.c b/arch/x86/platform/atom/punit_atom_debug.c index 81c769e80614..8ff7b9355416 100644 --- a/arch/x86/platform/atom/punit_atom_debug.c +++ b/arch/x86/platform/atom/punit_atom_debug.c @@ -23,10 +23,9 @@ #include #include #include +#include #include -/* Power gate status reg */ -#define PWRGT_STATUS 0x61 /* Subsystem config/status Video processor */ #define VED_SS_PM0 0x32 /* Subsystem config/status ISP (Image Signal Processor) */ @@ -35,12 +34,16 @@ #define MIO_SS_PM 0x3B /* Shift bits for getting status for video, isp and i/o */ #define SSS_SHIFT 24 + +/* Power gate status reg */ +#define PWRGT_STATUS 0x61 /* Shift bits for getting status for graphics rendering */ #define RENDER_POS 0 /* Shift bits for getting status for media control */ #define MEDIA_POS 2 /* Shift bits for getting status for Valley View/Baytrail display */ #define VLV_DISPLAY_POS 6 + /* Subsystem config/status display for Cherry Trail SOC */ #define CHT_DSP_SSS 0x36 /* Shift bits for getting status for display */ @@ -52,6 +55,14 @@ struct punit_device { int sss_pos; }; +static const struct punit_device punit_device_tng[] = { + { "DISPLAY", CHT_DSP_SSS, SSS_SHIFT }, + { "VED", VED_SS_PM0, SSS_SHIFT }, + { "ISP", ISP_SS_PM0, SSS_SHIFT }, + { "MIO", MIO_SS_PM, SSS_SHIFT }, + { NULL } +}; + static const struct punit_device punit_device_byt[] = { { "GFX RENDER", PWRGT_STATUS, RENDER_POS }, { "GFX MEDIA", PWRGT_STATUS, MEDIA_POS }, @@ -143,8 +154,9 @@ static void punit_dbgfs_unregister(void) (kernel_ulong_t)&drv_data } static const struct x86_cpu_id intel_punit_cpu_ids[] = { - ICPU(55, punit_device_byt), /* Valleyview, Bay Trail */ - ICPU(76, punit_device_cht), /* Braswell, Cherry Trail */ + ICPU(INTEL_FAM6_ATOM_SILVERMONT1, punit_device_byt), + ICPU(INTEL_FAM6_ATOM_MERRIFIELD1, punit_device_tng), + ICPU(INTEL_FAM6_ATOM_AIRMONT, punit_device_cht), {} }; diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index f93545e7dc54..17c8bbd4e2f0 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -98,21 +98,6 @@ static efi_status_t __init phys_efi_set_virtual_address_map( return status; } -void efi_get_time(struct timespec *now) -{ - efi_status_t status; - efi_time_t eft; - efi_time_cap_t cap; - - status = efi.get_time(&eft, &cap); - if (status != EFI_SUCCESS) - pr_err("Oops: efitime: can't read time!\n"); - - now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour, - eft.minute, eft.second); - now->tv_nsec = 0; -} - void __init efi_find_mirror(void) { efi_memory_desc_t *md; @@ -978,8 +963,6 @@ static void __init __efi_enter_virtual_mode(void) * EFI mixed mode we need all of memory to be accessible when * we pass parameters to the EFI runtime services in the * thunking code. - * - * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift); */ free_pages((unsigned long)new_memmap, pg_shift); diff --git a/arch/x86/platform/efi/efi_32.c b/arch/x86/platform/efi/efi_32.c index 338402b91d2e..cef39b097649 100644 --- a/arch/x86/platform/efi/efi_32.c +++ b/arch/x86/platform/efi/efi_32.c @@ -49,9 +49,6 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) { return 0; } -void __init efi_cleanup_page_tables(unsigned long pa_memmap, unsigned num_pages) -{ -} void __init efi_map_region(efi_memory_desc_t *md) { diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 6e7242be1c87..3e12c44f88a2 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -139,7 +139,7 @@ int __init efi_alloc_page_tables(void) if (efi_enabled(EFI_OLD_MEMMAP)) return 0; - gfp_mask = GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO; + gfp_mask = GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO; efi_pgd = (pgd_t *)__get_free_page(gfp_mask); if (!efi_pgd) return -ENOMEM; @@ -285,11 +285,6 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) return 0; } -void __init efi_cleanup_page_tables(unsigned long pa_memmap, unsigned num_pages) -{ - kernel_unmap_pages_in_pgd(efi_pgd, pa_memmap, num_pages); -} - static void __init __map_region(efi_memory_desc_t *md, u64 va) { unsigned long flags = _PAGE_RW; @@ -466,22 +461,17 @@ extern efi_status_t efi64_thunk(u32, ...); #define efi_thunk(f, ...) \ ({ \ efi_status_t __s; \ - unsigned long flags; \ - u32 func; \ - \ - efi_sync_low_kernel_mappings(); \ - local_irq_save(flags); \ + unsigned long __flags; \ + u32 __func; \ \ - efi_scratch.prev_cr3 = read_cr3(); \ - write_cr3((unsigned long)efi_scratch.efi_pgt); \ - __flush_tlb_all(); \ + local_irq_save(__flags); \ + arch_efi_call_virt_setup(); \ \ - func = runtime_service32(f); \ - __s = efi64_thunk(func, __VA_ARGS__); \ + __func = runtime_service32(f); \ + __s = efi64_thunk(__func, __VA_ARGS__); \ \ - write_cr3(efi_scratch.prev_cr3); \ - __flush_tlb_all(); \ - local_irq_restore(flags); \ + arch_efi_call_virt_teardown(); \ + local_irq_restore(__flags); \ \ __s; \ }) diff --git a/arch/x86/platform/intel-mid/Makefile b/arch/x86/platform/intel-mid/Makefile index 0ce1b1913673..fa021dfab088 100644 --- a/arch/x86/platform/intel-mid/Makefile +++ b/arch/x86/platform/intel-mid/Makefile @@ -1,4 +1,4 @@ -obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o intel_mid_vrtc.o mfld.o mrfl.o +obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o intel_mid_vrtc.o mfld.o mrfld.o pwr.o # SFI specific code ifdef CONFIG_X86_INTEL_MID diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile index 91ec9f8704bf..fc135bf70511 100644 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ b/arch/x86/platform/intel-mid/device_libs/Makefile @@ -1,3 +1,5 @@ +# Family-Level Interface Shim (FLIS) +obj-$(subst m,y,$(CONFIG_PINCTRL_MERRIFIELD)) += platform_mrfld_pinctrl.o # IPC Devices obj-y += platform_ipc.o obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o @@ -8,14 +10,18 @@ obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_battery.o obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_msic_power_btn.o obj-$(subst m,y,$(CONFIG_GPIO_INTEL_PMIC)) += platform_pmic_gpio.o obj-$(subst m,y,$(CONFIG_INTEL_MFLD_THERMAL)) += platform_msic_thermal.o +# SPI Devices +obj-$(subst m,y,$(CONFIG_SPI_SPIDEV)) += platform_spidev.o # I2C Devices obj-$(subst m,y,$(CONFIG_SENSORS_EMC1403)) += platform_emc1403.o obj-$(subst m,y,$(CONFIG_SENSORS_LIS3LV02D)) += platform_lis331.o -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_max7315.o obj-$(subst m,y,$(CONFIG_INPUT_MPU3050)) += platform_mpu3050.o obj-$(subst m,y,$(CONFIG_INPUT_BMA150)) += platform_bma023.o -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_tca6416.o obj-$(subst m,y,$(CONFIG_DRM_MEDFIELD)) += platform_tc35876x.o +# I2C GPIO Expanders +obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_max7315.o +obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_pcal9555a.o +obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_tca6416.o # MISC Devices obj-$(subst m,y,$(CONFIG_KEYBOARD_GPIO)) += platform_gpio_keys.o obj-$(subst m,y,$(CONFIG_INTEL_MID_WATCHDOG)) += platform_wdt.o diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c new file mode 100644 index 000000000000..4de8a664e6a1 --- /dev/null +++ b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c @@ -0,0 +1,43 @@ +/* + * Intel Merrifield FLIS platform device initialization file + * + * Copyright (C) 2016, Intel Corporation + * + * Author: Andy Shevchenko + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ + +#include +#include +#include + +#include + +#define FLIS_BASE_ADDR 0xff0c0000 +#define FLIS_LENGTH 0x8000 + +static struct resource mrfld_pinctrl_mmio_resource = { + .start = FLIS_BASE_ADDR, + .end = FLIS_BASE_ADDR + FLIS_LENGTH - 1, + .flags = IORESOURCE_MEM, +}; + +static struct platform_device mrfld_pinctrl_device = { + .name = "pinctrl-merrifield", + .id = PLATFORM_DEVID_NONE, + .resource = &mrfld_pinctrl_mmio_resource, + .num_resources = 1, +}; + +static int __init mrfld_pinctrl_init(void) +{ + if (intel_mid_identify_cpu() == INTEL_MID_CPU_CHIP_TANGIER) + return platform_device_register(&mrfld_pinctrl_device); + + return -ENODEV; +} +arch_initcall(mrfld_pinctrl_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c b/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c new file mode 100644 index 000000000000..429a94192671 --- /dev/null +++ b/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c @@ -0,0 +1,99 @@ +/* + * PCAL9555a platform data initilization file + * + * Copyright (C) 2016, Intel Corporation + * + * Authors: Andy Shevchenko + * Dan O'Donovan + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ + +#include +#include +#include +#include +#include + +#include + +#define PCAL9555A_NUM 4 + +static struct pca953x_platform_data pcal9555a_pdata[PCAL9555A_NUM]; +static int nr; + +static void __init *pcal9555a_platform_data(void *info) +{ + struct i2c_board_info *i2c_info = info; + char *type = i2c_info->type; + struct pca953x_platform_data *pcal9555a; + char base_pin_name[SFI_NAME_LEN + 1]; + char intr_pin_name[SFI_NAME_LEN + 1]; + int gpio_base, intr; + + snprintf(base_pin_name, sizeof(base_pin_name), "%s_base", type); + snprintf(intr_pin_name, sizeof(intr_pin_name), "%s_int", type); + + gpio_base = get_gpio_by_name(base_pin_name); + intr = get_gpio_by_name(intr_pin_name); + + /* Check if the SFI record valid */ + if (gpio_base == -1) + return NULL; + + if (nr >= PCAL9555A_NUM) { + pr_err("%s: Too many instances, only %d supported\n", __func__, + PCAL9555A_NUM); + return NULL; + } + + pcal9555a = &pcal9555a_pdata[nr++]; + pcal9555a->gpio_base = gpio_base; + + if (intr >= 0) { + i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; + pcal9555a->irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; + } else { + i2c_info->irq = -1; + pcal9555a->irq_base = -1; + } + + strcpy(type, "pcal9555a"); + return pcal9555a; +} + +static const struct devs_id pcal9555a_1_dev_id __initconst = { + .name = "pcal9555a-1", + .type = SFI_DEV_TYPE_I2C, + .delay = 1, + .get_platform_data = &pcal9555a_platform_data, +}; + +static const struct devs_id pcal9555a_2_dev_id __initconst = { + .name = "pcal9555a-2", + .type = SFI_DEV_TYPE_I2C, + .delay = 1, + .get_platform_data = &pcal9555a_platform_data, +}; + +static const struct devs_id pcal9555a_3_dev_id __initconst = { + .name = "pcal9555a-3", + .type = SFI_DEV_TYPE_I2C, + .delay = 1, + .get_platform_data = &pcal9555a_platform_data, +}; + +static const struct devs_id pcal9555a_4_dev_id __initconst = { + .name = "pcal9555a-4", + .type = SFI_DEV_TYPE_I2C, + .delay = 1, + .get_platform_data = &pcal9555a_platform_data, +}; + +sfi_device(pcal9555a_1_dev_id); +sfi_device(pcal9555a_2_dev_id); +sfi_device(pcal9555a_3_dev_id); +sfi_device(pcal9555a_4_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_spidev.c b/arch/x86/platform/intel-mid/device_libs/platform_spidev.c new file mode 100644 index 000000000000..30c601b399ee --- /dev/null +++ b/arch/x86/platform/intel-mid/device_libs/platform_spidev.c @@ -0,0 +1,50 @@ +/* + * spidev platform data initilization file + * + * (C) Copyright 2014, 2016 Intel Corporation + * Authors: Andy Shevchenko + * Dan O'Donovan + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ + +#include +#include +#include +#include + +#include + +#define MRFLD_SPI_DEFAULT_DMA_BURST 8 +#define MRFLD_SPI_DEFAULT_TIMEOUT 500 + +/* GPIO pin for spidev chipselect */ +#define MRFLD_SPIDEV_GPIO_CS 111 + +static struct pxa2xx_spi_chip spidev_spi_chip = { + .dma_burst_size = MRFLD_SPI_DEFAULT_DMA_BURST, + .timeout = MRFLD_SPI_DEFAULT_TIMEOUT, + .gpio_cs = MRFLD_SPIDEV_GPIO_CS, +}; + +static void __init *spidev_platform_data(void *info) +{ + struct spi_board_info *spi_info = info; + + spi_info->mode = SPI_MODE_0; + spi_info->controller_data = &spidev_spi_chip; + + return NULL; +} + +static const struct devs_id spidev_dev_id __initconst = { + .name = "spidev", + .type = SFI_DEV_TYPE_SPI, + .delay = 0, + .get_platform_data = &spidev_platform_data, +}; + +sfi_device(spidev_dev_id); diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 90bb997ed0a2..abbf49c6e9d3 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -144,6 +145,15 @@ static void intel_mid_arch_setup(void) out: if (intel_mid_ops->arch_setup) intel_mid_ops->arch_setup(); + + /* + * Intel MID platforms are using explicitly defined regulators. + * + * Let the regulator core know that we do not have any additional + * regulators left. This lets it substitute unprovided regulators with + * dummy ones: + */ + regulator_has_full_constraints(); } /* MID systems don't have i8042 controller */ diff --git a/arch/x86/platform/intel-mid/mrfl.c b/arch/x86/platform/intel-mid/mrfld.c similarity index 97% rename from arch/x86/platform/intel-mid/mrfl.c rename to arch/x86/platform/intel-mid/mrfld.c index bd1adc621781..59253db41bbc 100644 --- a/arch/x86/platform/intel-mid/mrfl.c +++ b/arch/x86/platform/intel-mid/mrfld.c @@ -1,5 +1,5 @@ /* - * mrfl.c: Intel Merrifield platform specific setup code + * Intel Merrifield platform specific setup code * * (C) Copyright 2013 Intel Corporation * diff --git a/arch/x86/platform/intel-mid/pwr.c b/arch/x86/platform/intel-mid/pwr.c new file mode 100644 index 000000000000..5bc90dd102d4 --- /dev/null +++ b/arch/x86/platform/intel-mid/pwr.c @@ -0,0 +1,418 @@ +/* + * Intel MID Power Management Unit (PWRMU) device driver + * + * Copyright (C) 2016, Intel Corporation + * + * Author: Andy Shevchenko + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * Intel MID Power Management Unit device driver handles the South Complex PCI + * devices such as GPDMA, SPI, I2C, PWM, and so on. By default PCI core + * modifies bits in PMCSR register in the PCI configuration space. This is not + * enough on some SoCs like Intel Tangier. In such case PCI core sets a new + * power state of the device in question through a PM hook registered in struct + * pci_platform_pm_ops (see drivers/pci/pci-mid.c). + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* Registers */ +#define PM_STS 0x00 +#define PM_CMD 0x04 +#define PM_ICS 0x08 +#define PM_WKC(x) (0x10 + (x) * 4) +#define PM_WKS(x) (0x18 + (x) * 4) +#define PM_SSC(x) (0x20 + (x) * 4) +#define PM_SSS(x) (0x30 + (x) * 4) + +/* Bits in PM_STS */ +#define PM_STS_BUSY (1 << 8) + +/* Bits in PM_CMD */ +#define PM_CMD_CMD(x) ((x) << 0) +#define PM_CMD_IOC (1 << 8) +#define PM_CMD_D3cold (1 << 21) + +/* List of commands */ +#define CMD_SET_CFG 0x01 + +/* Bits in PM_ICS */ +#define PM_ICS_INT_STATUS(x) ((x) & 0xff) +#define PM_ICS_IE (1 << 8) +#define PM_ICS_IP (1 << 9) +#define PM_ICS_SW_INT_STS (1 << 10) + +/* List of interrupts */ +#define INT_INVALID 0 +#define INT_CMD_COMPLETE 1 +#define INT_CMD_ERR 2 +#define INT_WAKE_EVENT 3 +#define INT_LSS_POWER_ERR 4 +#define INT_S0iX_MSG_ERR 5 +#define INT_NO_C6 6 +#define INT_TRIGGER_ERR 7 +#define INT_INACTIVITY 8 + +/* South Complex devices */ +#define LSS_MAX_SHARED_DEVS 4 +#define LSS_MAX_DEVS 64 + +#define LSS_WS_BITS 1 /* wake state width */ +#define LSS_PWS_BITS 2 /* power state width */ + +/* Supported device IDs */ +#define PCI_DEVICE_ID_PENWELL 0x0828 +#define PCI_DEVICE_ID_TANGIER 0x11a1 + +struct mid_pwr_dev { + struct pci_dev *pdev; + pci_power_t state; +}; + +struct mid_pwr { + struct device *dev; + void __iomem *regs; + int irq; + bool available; + + struct mutex lock; + struct mid_pwr_dev lss[LSS_MAX_DEVS][LSS_MAX_SHARED_DEVS]; +}; + +static struct mid_pwr *midpwr; + +static u32 mid_pwr_get_state(struct mid_pwr *pwr, int reg) +{ + return readl(pwr->regs + PM_SSS(reg)); +} + +static void mid_pwr_set_state(struct mid_pwr *pwr, int reg, u32 value) +{ + writel(value, pwr->regs + PM_SSC(reg)); +} + +static void mid_pwr_set_wake(struct mid_pwr *pwr, int reg, u32 value) +{ + writel(value, pwr->regs + PM_WKC(reg)); +} + +static void mid_pwr_interrupt_disable(struct mid_pwr *pwr) +{ + writel(~PM_ICS_IE, pwr->regs + PM_ICS); +} + +static bool mid_pwr_is_busy(struct mid_pwr *pwr) +{ + return !!(readl(pwr->regs + PM_STS) & PM_STS_BUSY); +} + +/* Wait 500ms that the latest PWRMU command finished */ +static int mid_pwr_wait(struct mid_pwr *pwr) +{ + unsigned int count = 500000; + bool busy; + + do { + busy = mid_pwr_is_busy(pwr); + if (!busy) + return 0; + udelay(1); + } while (--count); + + return -EBUSY; +} + +static int mid_pwr_wait_for_cmd(struct mid_pwr *pwr, u8 cmd) +{ + writel(PM_CMD_CMD(cmd), pwr->regs + PM_CMD); + return mid_pwr_wait(pwr); +} + +static int __update_power_state(struct mid_pwr *pwr, int reg, int bit, int new) +{ + int curstate; + u32 power; + int ret; + + /* Check if the device is already in desired state */ + power = mid_pwr_get_state(pwr, reg); + curstate = (power >> bit) & 3; + if (curstate == new) + return 0; + + /* Update the power state */ + mid_pwr_set_state(pwr, reg, (power & ~(3 << bit)) | (new << bit)); + + /* Send command to SCU */ + ret = mid_pwr_wait_for_cmd(pwr, CMD_SET_CFG); + if (ret) + return ret; + + /* Check if the device is already in desired state */ + power = mid_pwr_get_state(pwr, reg); + curstate = (power >> bit) & 3; + if (curstate != new) + return -EAGAIN; + + return 0; +} + +static pci_power_t __find_weakest_power_state(struct mid_pwr_dev *lss, + struct pci_dev *pdev, + pci_power_t state) +{ + pci_power_t weakest = PCI_D3hot; + unsigned int j; + + /* Find device in cache or first free cell */ + for (j = 0; j < LSS_MAX_SHARED_DEVS; j++) { + if (lss[j].pdev == pdev || !lss[j].pdev) + break; + } + + /* Store the desired state in cache */ + if (j < LSS_MAX_SHARED_DEVS) { + lss[j].pdev = pdev; + lss[j].state = state; + } else { + dev_WARN(&pdev->dev, "No room for device in PWRMU LSS cache\n"); + weakest = state; + } + + /* Find the power state we may use */ + for (j = 0; j < LSS_MAX_SHARED_DEVS; j++) { + if (lss[j].state < weakest) + weakest = lss[j].state; + } + + return weakest; +} + +static int __set_power_state(struct mid_pwr *pwr, struct pci_dev *pdev, + pci_power_t state, int id, int reg, int bit) +{ + const char *name; + int ret; + + state = __find_weakest_power_state(pwr->lss[id], pdev, state); + name = pci_power_name(state); + + ret = __update_power_state(pwr, reg, bit, (__force int)state); + if (ret) { + dev_warn(&pdev->dev, "Can't set power state %s: %d\n", name, ret); + return ret; + } + + dev_vdbg(&pdev->dev, "Set power state %s\n", name); + return 0; +} + +static int mid_pwr_set_power_state(struct mid_pwr *pwr, struct pci_dev *pdev, + pci_power_t state) +{ + int id, reg, bit; + int ret; + + id = intel_mid_pwr_get_lss_id(pdev); + if (id < 0) + return id; + + reg = (id * LSS_PWS_BITS) / 32; + bit = (id * LSS_PWS_BITS) % 32; + + /* We support states between PCI_D0 and PCI_D3hot */ + if (state < PCI_D0) + state = PCI_D0; + if (state > PCI_D3hot) + state = PCI_D3hot; + + mutex_lock(&pwr->lock); + ret = __set_power_state(pwr, pdev, state, id, reg, bit); + mutex_unlock(&pwr->lock); + return ret; +} + +int intel_mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state) +{ + struct mid_pwr *pwr = midpwr; + int ret = 0; + + might_sleep(); + + if (pwr && pwr->available) + ret = mid_pwr_set_power_state(pwr, pdev, state); + dev_vdbg(&pdev->dev, "set_power_state() returns %d\n", ret); + + return 0; +} +EXPORT_SYMBOL_GPL(intel_mid_pci_set_power_state); + +int intel_mid_pwr_get_lss_id(struct pci_dev *pdev) +{ + int vndr; + u8 id; + + /* + * Mapping to PWRMU index is kept in the Logical SubSystem ID byte of + * Vendor capability. + */ + vndr = pci_find_capability(pdev, PCI_CAP_ID_VNDR); + if (!vndr) + return -EINVAL; + + /* Read the Logical SubSystem ID byte */ + pci_read_config_byte(pdev, vndr + INTEL_MID_PWR_LSS_OFFSET, &id); + if (!(id & INTEL_MID_PWR_LSS_TYPE)) + return -ENODEV; + + id &= ~INTEL_MID_PWR_LSS_TYPE; + if (id >= LSS_MAX_DEVS) + return -ERANGE; + + return id; +} + +static irqreturn_t mid_pwr_irq_handler(int irq, void *dev_id) +{ + struct mid_pwr *pwr = dev_id; + u32 ics; + + ics = readl(pwr->regs + PM_ICS); + if (!(ics & PM_ICS_IP)) + return IRQ_NONE; + + writel(ics | PM_ICS_IP, pwr->regs + PM_ICS); + + dev_warn(pwr->dev, "Unexpected IRQ: %#x\n", PM_ICS_INT_STATUS(ics)); + return IRQ_HANDLED; +} + +struct mid_pwr_device_info { + int (*set_initial_state)(struct mid_pwr *pwr); +}; + +static int mid_pwr_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct mid_pwr_device_info *info = (void *)id->driver_data; + struct device *dev = &pdev->dev; + struct mid_pwr *pwr; + int ret; + + ret = pcim_enable_device(pdev); + if (ret < 0) { + dev_err(&pdev->dev, "error: could not enable device\n"); + return ret; + } + + ret = pcim_iomap_regions(pdev, 1 << 0, pci_name(pdev)); + if (ret) { + dev_err(&pdev->dev, "I/O memory remapping failed\n"); + return ret; + } + + pwr = devm_kzalloc(dev, sizeof(*pwr), GFP_KERNEL); + if (!pwr) + return -ENOMEM; + + pwr->dev = dev; + pwr->regs = pcim_iomap_table(pdev)[0]; + pwr->irq = pdev->irq; + + mutex_init(&pwr->lock); + + /* Disable interrupts */ + mid_pwr_interrupt_disable(pwr); + + if (info && info->set_initial_state) { + ret = info->set_initial_state(pwr); + if (ret) + dev_warn(dev, "Can't set initial state: %d\n", ret); + } + + ret = devm_request_irq(dev, pdev->irq, mid_pwr_irq_handler, + IRQF_NO_SUSPEND, pci_name(pdev), pwr); + if (ret) + return ret; + + pwr->available = true; + midpwr = pwr; + + pci_set_drvdata(pdev, pwr); + return 0; +} + +static int mid_set_initial_state(struct mid_pwr *pwr) +{ + unsigned int i, j; + int ret; + + /* + * Enable wake events. + * + * PWRMU supports up to 32 sources for wake up the system. Ungate them + * all here. + */ + mid_pwr_set_wake(pwr, 0, 0xffffffff); + mid_pwr_set_wake(pwr, 1, 0xffffffff); + + /* + * Power off South Complex devices. + * + * There is a map (see a note below) of 64 devices with 2 bits per each + * on 32-bit HW registers. The following calls set all devices to one + * known initial state, i.e. PCI_D3hot. This is done in conjunction + * with PMCSR setting in arch/x86/pci/intel_mid_pci.c. + * + * NOTE: The actual device mapping is provided by a platform at run + * time using vendor capability of PCI configuration space. + */ + mid_pwr_set_state(pwr, 0, 0xffffffff); + mid_pwr_set_state(pwr, 1, 0xffffffff); + mid_pwr_set_state(pwr, 2, 0xffffffff); + mid_pwr_set_state(pwr, 3, 0xffffffff); + + /* Send command to SCU */ + ret = mid_pwr_wait_for_cmd(pwr, CMD_SET_CFG); + if (ret) + return ret; + + for (i = 0; i < LSS_MAX_DEVS; i++) { + for (j = 0; j < LSS_MAX_SHARED_DEVS; j++) + pwr->lss[i][j].state = PCI_D3hot; + } + + return 0; +} + +static const struct mid_pwr_device_info mid_info = { + .set_initial_state = mid_set_initial_state, +}; + +static const struct pci_device_id mid_pwr_pci_ids[] = { + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_PENWELL), (kernel_ulong_t)&mid_info }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_TANGIER), (kernel_ulong_t)&mid_info }, + {} +}; +MODULE_DEVICE_TABLE(pci, mid_pwr_pci_ids); + +static struct pci_driver mid_pwr_pci_driver = { + .name = "intel_mid_pwr", + .probe = mid_pwr_probe, + .id_table = mid_pwr_pci_ids, +}; + +builtin_pci_driver(mid_pwr_pci_driver); diff --git a/arch/x86/platform/intel-mid/sfi.c b/arch/x86/platform/intel-mid/sfi.c index 5ee360a951ce..1555672d436f 100644 --- a/arch/x86/platform/intel-mid/sfi.c +++ b/arch/x86/platform/intel-mid/sfi.c @@ -407,6 +407,32 @@ static void __init sfi_handle_i2c_dev(struct sfi_device_table_entry *pentry, i2c_register_board_info(pentry->host_num, &i2c_info, 1); } +static void __init sfi_handle_sd_dev(struct sfi_device_table_entry *pentry, + struct devs_id *dev) +{ + struct mid_sd_board_info sd_info; + void *pdata; + + memset(&sd_info, 0, sizeof(sd_info)); + strncpy(sd_info.name, pentry->name, SFI_NAME_LEN); + sd_info.bus_num = pentry->host_num; + sd_info.max_clk = pentry->max_freq; + sd_info.addr = pentry->addr; + pr_debug("SD bus = %d, name = %16.16s, max_clk = %d, addr = 0x%x\n", + sd_info.bus_num, + sd_info.name, + sd_info.max_clk, + sd_info.addr); + pdata = intel_mid_sfi_get_pdata(dev, &sd_info); + if (IS_ERR(pdata)) + return; + + /* Nothing we can do with this for now */ + sd_info.platform_data = pdata; + + pr_debug("Successfully registered %16.16s", sd_info.name); +} + extern struct devs_id *const __x86_intel_mid_dev_start[], *const __x86_intel_mid_dev_end[]; @@ -490,6 +516,9 @@ static int __init sfi_parse_devs(struct sfi_table_header *table) case SFI_DEV_TYPE_I2C: sfi_handle_i2c_dev(pentry, dev); break; + case SFI_DEV_TYPE_SD: + sfi_handle_sd_dev(pentry, dev); + break; case SFI_DEV_TYPE_UART: case SFI_DEV_TYPE_HSI: default: diff --git a/arch/x86/platform/uv/bios_uv.c b/arch/x86/platform/uv/bios_uv.c index 815fec6e05e2..66b2166ea4a1 100644 --- a/arch/x86/platform/uv/bios_uv.c +++ b/arch/x86/platform/uv/bios_uv.c @@ -40,8 +40,7 @@ s64 uv_bios_call(enum uv_bios_cmd which, u64 a1, u64 a2, u64 a3, u64 a4, u64 a5) */ return BIOS_STATUS_UNIMPLEMENTED; - ret = efi_call((void *)__va(tab->function), (u64)which, - a1, a2, a3, a4, a5); + ret = efi_call_virt_pointer(tab, function, (u64)which, a1, a2, a3, a4, a5); return ret; } EXPORT_SYMBOL_GPL(uv_bios_call); diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index d5f64996394a..b12c26e2e309 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -266,6 +267,35 @@ void notrace restore_processor_state(void) EXPORT_SYMBOL(restore_processor_state); #endif +#if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU) +static void resume_play_dead(void) +{ + play_dead_common(); + tboot_shutdown(TB_SHUTDOWN_WFS); + hlt_play_dead(); +} + +int hibernate_resume_nonboot_cpu_disable(void) +{ + void (*play_dead)(void) = smp_ops.play_dead; + int ret; + + /* + * Ensure that MONITOR/MWAIT will not be used in the "play dead" loop + * during hibernate image restoration, because it is likely that the + * monitored address will be actually written to at that time and then + * the "dead" CPU will attempt to execute instructions again, but the + * address in its instruction pointer may not be possible to resolve + * any more at that point (the page tables used by it previously may + * have been overwritten by hibernate image data). + */ + smp_ops.play_dead = resume_play_dead; + ret = disable_nonboot_cpus(); + smp_ops.play_dead = play_dead; + return ret; +} +#endif + /* * When bsp_check() is called in hibernate and suspend, cpu hotplug * is disabled already. So it's unnessary to handle race condition between diff --git a/arch/x86/power/hibernate_64.c b/arch/x86/power/hibernate_64.c index 009947d419a6..f2b5e6a5cf95 100644 --- a/arch/x86/power/hibernate_64.c +++ b/arch/x86/power/hibernate_64.c @@ -19,6 +19,7 @@ #include #include #include +#include /* Defined in hibernate_asm_64.S */ extern asmlinkage __visible int restore_image(void); @@ -28,6 +29,7 @@ extern asmlinkage __visible int restore_image(void); * kernel's text (this value is passed in the image header). */ unsigned long restore_jump_address __visible; +unsigned long jump_address_phys; /* * Value of the cr3 register from before the hibernation (this value is passed @@ -37,7 +39,43 @@ unsigned long restore_cr3 __visible; pgd_t *temp_level4_pgt __visible; -void *relocated_restore_code __visible; +unsigned long relocated_restore_code __visible; + +static int set_up_temporary_text_mapping(void) +{ + pmd_t *pmd; + pud_t *pud; + + /* + * The new mapping only has to cover the page containing the image + * kernel's entry point (jump_address_phys), because the switch over to + * it is carried out by relocated code running from a page allocated + * specifically for this purpose and covered by the identity mapping, so + * the temporary kernel text mapping is only needed for the final jump. + * Moreover, in that mapping the virtual address of the image kernel's + * entry point must be the same as its virtual address in the image + * kernel (restore_jump_address), so the image kernel's + * restore_registers() code doesn't find itself in a different area of + * the virtual address space after switching over to the original page + * tables used by the image kernel. + */ + pud = (pud_t *)get_safe_page(GFP_ATOMIC); + if (!pud) + return -ENOMEM; + + pmd = (pmd_t *)get_safe_page(GFP_ATOMIC); + if (!pmd) + return -ENOMEM; + + set_pmd(pmd + pmd_index(restore_jump_address), + __pmd((jump_address_phys & PMD_MASK) | __PAGE_KERNEL_LARGE_EXEC)); + set_pud(pud + pud_index(restore_jump_address), + __pud(__pa(pmd) | _KERNPG_TABLE)); + set_pgd(temp_level4_pgt + pgd_index(restore_jump_address), + __pgd(__pa(pud) | _KERNPG_TABLE)); + + return 0; +} static void *alloc_pgt_page(void *context) { @@ -59,9 +97,10 @@ static int set_up_temporary_mappings(void) if (!temp_level4_pgt) return -ENOMEM; - /* It is safe to reuse the original kernel mapping */ - set_pgd(temp_level4_pgt + pgd_index(__START_KERNEL_map), - init_level4_pgt[pgd_index(__START_KERNEL_map)]); + /* Prepare a temporary mapping for the kernel text */ + result = set_up_temporary_text_mapping(); + if (result) + return result; /* Set up the direct mapping from scratch */ for (i = 0; i < nr_pfn_mapped; i++) { @@ -78,19 +117,50 @@ static int set_up_temporary_mappings(void) return 0; } +static int relocate_restore_code(void) +{ + pgd_t *pgd; + pud_t *pud; + + relocated_restore_code = get_safe_page(GFP_ATOMIC); + if (!relocated_restore_code) + return -ENOMEM; + + memcpy((void *)relocated_restore_code, &core_restore_code, PAGE_SIZE); + + /* Make the page containing the relocated code executable */ + pgd = (pgd_t *)__va(read_cr3()) + pgd_index(relocated_restore_code); + pud = pud_offset(pgd, relocated_restore_code); + if (pud_large(*pud)) { + set_pud(pud, __pud(pud_val(*pud) & ~_PAGE_NX)); + } else { + pmd_t *pmd = pmd_offset(pud, relocated_restore_code); + + if (pmd_large(*pmd)) { + set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_NX)); + } else { + pte_t *pte = pte_offset_kernel(pmd, relocated_restore_code); + + set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_NX)); + } + } + __flush_tlb_all(); + + return 0; +} + int swsusp_arch_resume(void) { int error; /* We have got enough memory and from now on we cannot recover */ - if ((error = set_up_temporary_mappings())) + error = set_up_temporary_mappings(); + if (error) return error; - relocated_restore_code = (void *)get_safe_page(GFP_ATOMIC); - if (!relocated_restore_code) - return -ENOMEM; - memcpy(relocated_restore_code, &core_restore_code, - &restore_registers - &core_restore_code); + error = relocate_restore_code(); + if (error) + return error; restore_image(); return 0; @@ -109,11 +179,12 @@ int pfn_is_nosave(unsigned long pfn) struct restore_data_record { unsigned long jump_address; + unsigned long jump_address_phys; unsigned long cr3; unsigned long magic; }; -#define RESTORE_MAGIC 0x0123456789ABCDEFUL +#define RESTORE_MAGIC 0x123456789ABCDEF0UL /** * arch_hibernation_header_save - populate the architecture specific part @@ -126,7 +197,8 @@ int arch_hibernation_header_save(void *addr, unsigned int max_size) if (max_size < sizeof(struct restore_data_record)) return -EOVERFLOW; - rdr->jump_address = restore_jump_address; + rdr->jump_address = (unsigned long)&restore_registers; + rdr->jump_address_phys = __pa_symbol(&restore_registers); rdr->cr3 = restore_cr3; rdr->magic = RESTORE_MAGIC; return 0; @@ -142,6 +214,7 @@ int arch_hibernation_header_restore(void *addr) struct restore_data_record *rdr = addr; restore_jump_address = rdr->jump_address; + jump_address_phys = rdr->jump_address_phys; restore_cr3 = rdr->cr3; return (rdr->magic == RESTORE_MAGIC) ? 0 : -EINVAL; } diff --git a/arch/x86/power/hibernate_asm_64.S b/arch/x86/power/hibernate_asm_64.S index 4400a43b9e28..3177c2bc26f6 100644 --- a/arch/x86/power/hibernate_asm_64.S +++ b/arch/x86/power/hibernate_asm_64.S @@ -44,9 +44,6 @@ ENTRY(swsusp_arch_suspend) pushfq popq pt_regs_flags(%rax) - /* save the address of restore_registers */ - movq $restore_registers, %rax - movq %rax, restore_jump_address(%rip) /* save cr3 */ movq %cr3, %rax movq %rax, restore_cr3(%rip) @@ -57,31 +54,34 @@ ENTRY(swsusp_arch_suspend) ENDPROC(swsusp_arch_suspend) ENTRY(restore_image) - /* switch to temporary page tables */ - movq $__PAGE_OFFSET, %rdx - movq temp_level4_pgt(%rip), %rax - subq %rdx, %rax - movq %rax, %cr3 - /* Flush TLB */ - movq mmu_cr4_features(%rip), %rax - movq %rax, %rdx - andq $~(X86_CR4_PGE), %rdx - movq %rdx, %cr4; # turn off PGE - movq %cr3, %rcx; # flush TLB - movq %rcx, %cr3; - movq %rax, %cr4; # turn PGE back on - /* prepare to jump to the image kernel */ - movq restore_jump_address(%rip), %rax - movq restore_cr3(%rip), %rbx + movq restore_jump_address(%rip), %r8 + movq restore_cr3(%rip), %r9 + + /* prepare to switch to temporary page tables */ + movq temp_level4_pgt(%rip), %rax + movq mmu_cr4_features(%rip), %rbx /* prepare to copy image data to their original locations */ movq restore_pblist(%rip), %rdx + + /* jump to relocated restore code */ movq relocated_restore_code(%rip), %rcx jmpq *%rcx /* code below has been relocated to a safe page */ ENTRY(core_restore_code) + /* switch to temporary page tables */ + movq $__PAGE_OFFSET, %rcx + subq %rcx, %rax + movq %rax, %cr3 + /* flush TLB */ + movq %rbx, %rcx + andq $~(X86_CR4_PGE), %rcx + movq %rcx, %cr4; # turn off PGE + movq %cr3, %rcx; # flush TLB + movq %rcx, %cr3; + movq %rbx, %cr4; # turn PGE back on .Lloop: testq %rdx, %rdx jz .Ldone @@ -96,24 +96,17 @@ ENTRY(core_restore_code) /* progress to the next pbe */ movq pbe_next(%rdx), %rdx jmp .Lloop + .Ldone: /* jump to the restore_registers address from the image header */ - jmpq *%rax - /* - * NOTE: This assumes that the boot kernel's text mapping covers the - * image kernel's page containing restore_registers and the address of - * this page is the same as in the image kernel's text mapping (it - * should always be true, because the text mapping is linear, starting - * from 0, and is supposed to cover the entire kernel text for every - * kernel). - * - * code below belongs to the image kernel - */ + jmpq *%r8 + /* code below belongs to the image kernel */ + .align PAGE_SIZE ENTRY(restore_registers) FRAME_BEGIN /* go back to the original page tables */ - movq %rbx, %cr3 + movq %r9, %cr3 /* Flush TLB, including "global" things (vmalloc) */ movq mmu_cr4_features(%rip), %rax diff --git a/arch/x86/ras/mce_amd_inj.c b/arch/x86/ras/mce_amd_inj.c index e69f4701a076..1104515d5ad2 100644 --- a/arch/x86/ras/mce_amd_inj.c +++ b/arch/x86/ras/mce_amd_inj.c @@ -241,6 +241,31 @@ static void toggle_nb_mca_mst_cpu(u16 nid) __func__, PCI_FUNC(F3->devfn), NBCFG); } +static void prepare_msrs(void *info) +{ + struct mce i_mce = *(struct mce *)info; + u8 b = i_mce.bank; + + wrmsrl(MSR_IA32_MCG_STATUS, i_mce.mcgstatus); + + if (boot_cpu_has(X86_FEATURE_SMCA)) { + if (i_mce.inject_flags == DFR_INT_INJ) { + wrmsrl(MSR_AMD64_SMCA_MCx_DESTAT(b), i_mce.status); + wrmsrl(MSR_AMD64_SMCA_MCx_DEADDR(b), i_mce.addr); + } else { + wrmsrl(MSR_AMD64_SMCA_MCx_STATUS(b), i_mce.status); + wrmsrl(MSR_AMD64_SMCA_MCx_ADDR(b), i_mce.addr); + } + + wrmsrl(MSR_AMD64_SMCA_MCx_MISC(b), i_mce.misc); + } else { + wrmsrl(MSR_IA32_MCx_STATUS(b), i_mce.status); + wrmsrl(MSR_IA32_MCx_ADDR(b), i_mce.addr); + wrmsrl(MSR_IA32_MCx_MISC(b), i_mce.misc); + } + +} + static void do_inject(void) { u64 mcg_status = 0; @@ -287,36 +312,9 @@ static void do_inject(void) toggle_hw_mce_inject(cpu, true); - wrmsr_on_cpu(cpu, MSR_IA32_MCG_STATUS, - (u32)mcg_status, (u32)(mcg_status >> 32)); - - if (boot_cpu_has(X86_FEATURE_SMCA)) { - if (inj_type == DFR_INT_INJ) { - wrmsr_on_cpu(cpu, MSR_AMD64_SMCA_MCx_DESTAT(b), - (u32)i_mce.status, (u32)(i_mce.status >> 32)); - - wrmsr_on_cpu(cpu, MSR_AMD64_SMCA_MCx_DEADDR(b), - (u32)i_mce.addr, (u32)(i_mce.addr >> 32)); - } else { - wrmsr_on_cpu(cpu, MSR_AMD64_SMCA_MCx_STATUS(b), - (u32)i_mce.status, (u32)(i_mce.status >> 32)); - - wrmsr_on_cpu(cpu, MSR_AMD64_SMCA_MCx_ADDR(b), - (u32)i_mce.addr, (u32)(i_mce.addr >> 32)); - } - - wrmsr_on_cpu(cpu, MSR_AMD64_SMCA_MCx_MISC(b), - (u32)i_mce.misc, (u32)(i_mce.misc >> 32)); - } else { - wrmsr_on_cpu(cpu, MSR_IA32_MCx_STATUS(b), - (u32)i_mce.status, (u32)(i_mce.status >> 32)); - - wrmsr_on_cpu(cpu, MSR_IA32_MCx_ADDR(b), - (u32)i_mce.addr, (u32)(i_mce.addr >> 32)); - - wrmsr_on_cpu(cpu, MSR_IA32_MCx_MISC(b), - (u32)i_mce.misc, (u32)(i_mce.misc >> 32)); - } + i_mce.mcgstatus = mcg_status; + i_mce.inject_flags = inj_type; + smp_call_function_single(cpu, prepare_msrs, &i_mce, 0); toggle_hw_mce_inject(cpu, false); diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c index 0b7a63d98440..705e3fffb4a1 100644 --- a/arch/x86/realmode/init.c +++ b/arch/x86/realmode/init.c @@ -8,6 +8,9 @@ struct real_mode_header *real_mode_header; u32 *trampoline_cr4_features; +/* Hold the pgd entry used on booting additional CPUs */ +pgd_t trampoline_pgd_entry; + void __init reserve_real_mode(void) { phys_addr_t mem; @@ -84,7 +87,7 @@ void __init setup_real_mode(void) *trampoline_cr4_features = __read_cr4(); trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd); - trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd; + trampoline_pgd[0] = trampoline_pgd_entry.pgd; trampoline_pgd[511] = init_level4_pgt[511].pgd; #endif } diff --git a/arch/x86/tools/gen-insn-attr-x86.awk b/arch/x86/tools/gen-insn-attr-x86.awk index 093a892026f9..a3d2c62fd805 100644 --- a/arch/x86/tools/gen-insn-attr-x86.awk +++ b/arch/x86/tools/gen-insn-attr-x86.awk @@ -72,12 +72,14 @@ BEGIN { lprefix_expr = "\\((66|F2|F3)\\)" max_lprefix = 4 - # All opcodes starting with lower-case 'v' or with (v1) superscript + # All opcodes starting with lower-case 'v', 'k' or with (v1) superscript # accepts VEX prefix - vexok_opcode_expr = "^v.*" + vexok_opcode_expr = "^[vk].*" vexok_expr = "\\(v1\\)" # All opcodes with (v) superscript supports *only* VEX prefix vexonly_expr = "\\(v\\)" + # All opcodes with (ev) superscript supports *only* EVEX prefix + evexonly_expr = "\\(ev\\)" prefix_expr = "\\(Prefix\\)" prefix_num["Operand-Size"] = "INAT_PFX_OPNDSZ" @@ -95,6 +97,7 @@ BEGIN { prefix_num["Address-Size"] = "INAT_PFX_ADDRSZ" prefix_num["VEX+1byte"] = "INAT_PFX_VEX2" prefix_num["VEX+2byte"] = "INAT_PFX_VEX3" + prefix_num["EVEX"] = "INAT_PFX_EVEX" clear_vars() } @@ -319,7 +322,9 @@ function convert_operands(count,opnd, i,j,imm,mod) flags = add_flags(flags, "INAT_MODRM") # check VEX codes - if (match(ext, vexonly_expr)) + if (match(ext, evexonly_expr)) + flags = add_flags(flags, "INAT_VEXOK | INAT_EVEXONLY") + else if (match(ext, vexonly_expr)) flags = add_flags(flags, "INAT_VEXOK | INAT_VEXONLY") else if (match(ext, vexok_expr) || match(opcode, vexok_opcode_expr)) flags = add_flags(flags, "INAT_VEXOK") diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index db52a7fafcc2..44c88ad1841a 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -177,7 +177,6 @@ static struct apic xen_pv_apic = { .get_apic_id = xen_get_apic_id, .set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */ - .apic_id_mask = 0xFF << 24, /* Used by verify_local_APIC. Match with what xen_get_apic_id does. */ .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, diff --git a/arch/x86/xen/efi.c b/arch/x86/xen/efi.c index be14cc3e48d5..3be012115853 100644 --- a/arch/x86/xen/efi.c +++ b/arch/x86/xen/efi.c @@ -20,10 +20,121 @@ #include #include +#include #include +#include #include #include +#include + +static efi_char16_t vendor[100] __initdata; + +static efi_system_table_t efi_systab_xen __initdata = { + .hdr = { + .signature = EFI_SYSTEM_TABLE_SIGNATURE, + .revision = 0, /* Initialized later. */ + .headersize = 0, /* Ignored by Linux Kernel. */ + .crc32 = 0, /* Ignored by Linux Kernel. */ + .reserved = 0 + }, + .fw_vendor = EFI_INVALID_TABLE_ADDR, /* Initialized later. */ + .fw_revision = 0, /* Initialized later. */ + .con_in_handle = EFI_INVALID_TABLE_ADDR, /* Not used under Xen. */ + .con_in = EFI_INVALID_TABLE_ADDR, /* Not used under Xen. */ + .con_out_handle = EFI_INVALID_TABLE_ADDR, /* Not used under Xen. */ + .con_out = EFI_INVALID_TABLE_ADDR, /* Not used under Xen. */ + .stderr_handle = EFI_INVALID_TABLE_ADDR, /* Not used under Xen. */ + .stderr = EFI_INVALID_TABLE_ADDR, /* Not used under Xen. */ + .runtime = (efi_runtime_services_t *)EFI_INVALID_TABLE_ADDR, + /* Not used under Xen. */ + .boottime = (efi_boot_services_t *)EFI_INVALID_TABLE_ADDR, + /* Not used under Xen. */ + .nr_tables = 0, /* Initialized later. */ + .tables = EFI_INVALID_TABLE_ADDR /* Initialized later. */ +}; + +static const struct efi efi_xen __initconst = { + .systab = NULL, /* Initialized later. */ + .runtime_version = 0, /* Initialized later. */ + .mps = EFI_INVALID_TABLE_ADDR, + .acpi = EFI_INVALID_TABLE_ADDR, + .acpi20 = EFI_INVALID_TABLE_ADDR, + .smbios = EFI_INVALID_TABLE_ADDR, + .smbios3 = EFI_INVALID_TABLE_ADDR, + .sal_systab = EFI_INVALID_TABLE_ADDR, + .boot_info = EFI_INVALID_TABLE_ADDR, + .hcdp = EFI_INVALID_TABLE_ADDR, + .uga = EFI_INVALID_TABLE_ADDR, + .uv_systab = EFI_INVALID_TABLE_ADDR, + .fw_vendor = EFI_INVALID_TABLE_ADDR, + .runtime = EFI_INVALID_TABLE_ADDR, + .config_table = EFI_INVALID_TABLE_ADDR, + .get_time = xen_efi_get_time, + .set_time = xen_efi_set_time, + .get_wakeup_time = xen_efi_get_wakeup_time, + .set_wakeup_time = xen_efi_set_wakeup_time, + .get_variable = xen_efi_get_variable, + .get_next_variable = xen_efi_get_next_variable, + .set_variable = xen_efi_set_variable, + .query_variable_info = xen_efi_query_variable_info, + .update_capsule = xen_efi_update_capsule, + .query_capsule_caps = xen_efi_query_capsule_caps, + .get_next_high_mono_count = xen_efi_get_next_high_mono_count, + .reset_system = NULL, /* Functionality provided by Xen. */ + .set_virtual_address_map = NULL, /* Not used under Xen. */ + .flags = 0 /* Initialized later. */ +}; + +static efi_system_table_t __init *xen_efi_probe(void) +{ + struct xen_platform_op op = { + .cmd = XENPF_firmware_info, + .u.firmware_info = { + .type = XEN_FW_EFI_INFO, + .index = XEN_FW_EFI_CONFIG_TABLE + } + }; + union xenpf_efi_info *info = &op.u.firmware_info.u.efi_info; + + if (!xen_initial_domain() || HYPERVISOR_platform_op(&op) < 0) + return NULL; + + /* Here we know that Xen runs on EFI platform. */ + + efi = efi_xen; + + efi_systab_xen.tables = info->cfg.addr; + efi_systab_xen.nr_tables = info->cfg.nent; + + op.cmd = XENPF_firmware_info; + op.u.firmware_info.type = XEN_FW_EFI_INFO; + op.u.firmware_info.index = XEN_FW_EFI_VENDOR; + info->vendor.bufsz = sizeof(vendor); + set_xen_guest_handle(info->vendor.name, vendor); + + if (HYPERVISOR_platform_op(&op) == 0) { + efi_systab_xen.fw_vendor = __pa_symbol(vendor); + efi_systab_xen.fw_revision = info->vendor.revision; + } else + efi_systab_xen.fw_vendor = __pa_symbol(L"UNKNOWN"); + + op.cmd = XENPF_firmware_info; + op.u.firmware_info.type = XEN_FW_EFI_INFO; + op.u.firmware_info.index = XEN_FW_EFI_VERSION; + + if (HYPERVISOR_platform_op(&op) == 0) + efi_systab_xen.hdr.revision = info->version; + + op.cmd = XENPF_firmware_info; + op.u.firmware_info.type = XEN_FW_EFI_INFO; + op.u.firmware_info.index = XEN_FW_EFI_RT_VERSION; + + if (HYPERVISOR_platform_op(&op) == 0) + efi.runtime_version = info->version; + + return &efi_systab_xen; +} void __init xen_efi_init(void) { diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 760789ae8562..69b4b6d29738 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -118,6 +119,10 @@ DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu); */ DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info); +/* Linux <-> Xen vCPU id mapping */ +DEFINE_PER_CPU(int, xen_vcpu_id) = -1; +EXPORT_PER_CPU_SYMBOL(xen_vcpu_id); + enum xen_domain_type xen_domain_type = XEN_NATIVE; EXPORT_SYMBOL_GPL(xen_domain_type); @@ -179,7 +184,7 @@ static void clamp_max_cpus(void) #endif } -static void xen_vcpu_setup(int cpu) +void xen_vcpu_setup(int cpu) { struct vcpu_register_vcpu_info info; int err; @@ -202,8 +207,9 @@ static void xen_vcpu_setup(int cpu) if (per_cpu(xen_vcpu, cpu) == &per_cpu(xen_vcpu_info, cpu)) return; } - if (cpu < MAX_VIRT_CPUS) - per_cpu(xen_vcpu,cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu]; + if (xen_vcpu_nr(cpu) < MAX_VIRT_CPUS) + per_cpu(xen_vcpu, cpu) = + &HYPERVISOR_shared_info->vcpu_info[xen_vcpu_nr(cpu)]; if (!have_vcpu_info_placement) { if (cpu >= MAX_VIRT_CPUS) @@ -223,7 +229,8 @@ static void xen_vcpu_setup(int cpu) hypervisor has no unregister variant and this hypercall does not allow to over-write info.mfn and info.offset. */ - err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, &info); + err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, xen_vcpu_nr(cpu), + &info); if (err) { printk(KERN_DEBUG "register_vcpu_info failed: err=%d\n", err); @@ -247,10 +254,11 @@ void xen_vcpu_restore(void) for_each_possible_cpu(cpu) { bool other_cpu = (cpu != smp_processor_id()); - bool is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL); + bool is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up, xen_vcpu_nr(cpu), + NULL); if (other_cpu && is_up && - HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL)) + HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL)) BUG(); xen_setup_runstate_info(cpu); @@ -259,7 +267,7 @@ void xen_vcpu_restore(void) xen_vcpu_setup(cpu); if (other_cpu && is_up && - HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL)) + HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL)) BUG(); } } @@ -521,9 +529,7 @@ static void set_aliased_prot(void *v, pgprot_t prot) preempt_disable(); - pagefault_disable(); /* Avoid warnings due to being atomic. */ - __get_user(dummy, (unsigned char __user __force *)v); - pagefault_enable(); + probe_kernel_read(&dummy, v, 1); if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0)) BUG(); @@ -590,7 +596,7 @@ static void xen_load_gdt(const struct desc_ptr *dtr) { unsigned long va = dtr->address; unsigned int size = dtr->size + 1; - unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + unsigned pages = DIV_ROUND_UP(size, PAGE_SIZE); unsigned long frames[pages]; int f; @@ -639,7 +645,7 @@ static void __init xen_load_gdt_boot(const struct desc_ptr *dtr) { unsigned long va = dtr->address; unsigned int size = dtr->size + 1; - unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + unsigned pages = DIV_ROUND_UP(size, PAGE_SIZE); unsigned long frames[pages]; int f; @@ -1137,8 +1143,11 @@ void xen_setup_vcpu_info_placement(void) { int cpu; - for_each_possible_cpu(cpu) + for_each_possible_cpu(cpu) { + /* Set up direct vCPU id mapping for PV guests. */ + per_cpu(xen_vcpu_id, cpu) = cpu; xen_vcpu_setup(cpu); + } /* xen_vcpu_setup managed to place the vcpu_info within the * percpu area for all cpus, so make use of it. Note that for @@ -1729,6 +1738,9 @@ asmlinkage __visible void __init xen_start_kernel(void) #endif xen_raw_console_write("about to get started...\n"); + /* Let's presume PV guests always boot on vCPU with id 0. */ + per_cpu(xen_vcpu_id, 0) = 0; + xen_setup_runstate_info(0); xen_efi_init(); @@ -1770,9 +1782,10 @@ void __ref xen_hvm_init_shared_info(void) * in that case multiple vcpus might be online. */ for_each_online_cpu(cpu) { /* Leave it to be NULL. */ - if (cpu >= MAX_VIRT_CPUS) + if (xen_vcpu_nr(cpu) >= MAX_VIRT_CPUS) continue; - per_cpu(xen_vcpu, cpu) = &HYPERVISOR_shared_info->vcpu_info[cpu]; + per_cpu(xen_vcpu, cpu) = + &HYPERVISOR_shared_info->vcpu_info[xen_vcpu_nr(cpu)]; } } @@ -1797,6 +1810,12 @@ static void __init init_hvm_pv_info(void) xen_setup_features(); + cpuid(base + 4, &eax, &ebx, &ecx, &edx); + if (eax & XEN_HVM_CPUID_VCPU_ID_PRESENT) + this_cpu_write(xen_vcpu_id, ebx); + else + this_cpu_write(xen_vcpu_id, smp_processor_id()); + pv_info.name = "Xen HVM"; xen_domain_type = XEN_HVM_DOMAIN; @@ -1808,6 +1827,10 @@ static int xen_hvm_cpu_notify(struct notifier_block *self, unsigned long action, int cpu = (long)hcpu; switch (action) { case CPU_UP_PREPARE: + if (cpu_acpi_id(cpu) != U32_MAX) + per_cpu(xen_vcpu_id, cpu) = cpu_acpi_id(cpu); + else + per_cpu(xen_vcpu_id, cpu) = cpu; xen_vcpu_setup(cpu); if (xen_have_vector_callback) { if (xen_feature(XENFEAT_hvm_safe_pvclock)) diff --git a/arch/x86/xen/grant-table.c b/arch/x86/xen/grant-table.c index e079500b17f3..de4144c24f1c 100644 --- a/arch/x86/xen/grant-table.c +++ b/arch/x86/xen/grant-table.c @@ -111,63 +111,18 @@ int arch_gnttab_init(unsigned long nr_shared) } #ifdef CONFIG_XEN_PVH -#include #include -#include -static int __init xlated_setup_gnttab_pages(void) -{ - struct page **pages; - xen_pfn_t *pfns; - void *vaddr; - int rc; - unsigned int i; - unsigned long nr_grant_frames = gnttab_max_grant_frames(); - - BUG_ON(nr_grant_frames == 0); - pages = kcalloc(nr_grant_frames, sizeof(pages[0]), GFP_KERNEL); - if (!pages) - return -ENOMEM; - - pfns = kcalloc(nr_grant_frames, sizeof(pfns[0]), GFP_KERNEL); - if (!pfns) { - kfree(pages); - return -ENOMEM; - } - rc = alloc_xenballooned_pages(nr_grant_frames, pages); - if (rc) { - pr_warn("%s Couldn't balloon alloc %ld pfns rc:%d\n", __func__, - nr_grant_frames, rc); - kfree(pages); - kfree(pfns); - return rc; - } - for (i = 0; i < nr_grant_frames; i++) - pfns[i] = page_to_pfn(pages[i]); - - vaddr = vmap(pages, nr_grant_frames, 0, PAGE_KERNEL); - if (!vaddr) { - pr_warn("%s Couldn't map %ld pfns rc:%d\n", __func__, - nr_grant_frames, rc); - free_xenballooned_pages(nr_grant_frames, pages); - kfree(pages); - kfree(pfns); - return -ENOMEM; - } - kfree(pages); - - xen_auto_xlat_grant_frames.pfn = pfns; - xen_auto_xlat_grant_frames.count = nr_grant_frames; - xen_auto_xlat_grant_frames.vaddr = vaddr; - - return 0; -} - +#include static int __init xen_pvh_gnttab_setup(void) { if (!xen_pvh_domain()) return -ENODEV; - return xlated_setup_gnttab_pages(); + xen_auto_xlat_grant_frames.count = gnttab_max_grant_frames(); + + return xen_xlate_map_ballooned_pages(&xen_auto_xlat_grant_frames.pfn, + &xen_auto_xlat_grant_frames.vaddr, + xen_auto_xlat_grant_frames.count); } /* Call it _before_ __gnttab_init as we need to initialize the * xen_auto_xlat_grant_frames first. */ diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c index a1207cb6472a..33e92955e09d 100644 --- a/arch/x86/xen/irq.c +++ b/arch/x86/xen/irq.c @@ -109,7 +109,8 @@ static void xen_safe_halt(void) static void xen_halt(void) { if (irqs_disabled()) - HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL); + HYPERVISOR_vcpu_op(VCPUOP_down, + xen_vcpu_nr(smp_processor_id()), NULL); else xen_safe_halt(); } diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index 478a2de543a5..67433714b791 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c @@ -1113,7 +1113,7 @@ static void __init xen_cleanhighmap(unsigned long vaddr, /* NOTE: The loop is more greedy than the cleanup_highmap variant. * We include the PMD passed in on _both_ boundaries. */ - for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PAGE_SIZE)); + for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PTRS_PER_PMD)); pmd++, vaddr += PMD_SIZE) { if (pmd_none(*pmd)) continue; @@ -1551,41 +1551,6 @@ static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd) #endif } -#ifdef CONFIG_X86_32 -static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) -{ - /* If there's an existing pte, then don't allow _PAGE_RW to be set */ - if (pte_val_ma(*ptep) & _PAGE_PRESENT) - pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) & - pte_val_ma(pte)); - - return pte; -} -#else /* CONFIG_X86_64 */ -static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) -{ - unsigned long pfn; - - if (xen_feature(XENFEAT_writable_page_tables) || - xen_feature(XENFEAT_auto_translated_physmap) || - xen_start_info->mfn_list >= __START_KERNEL_map) - return pte; - - /* - * Pages belonging to the initial p2m list mapped outside the default - * address range must be mapped read-only. This region contains the - * page tables for mapping the p2m list, too, and page tables MUST be - * mapped read-only. - */ - pfn = pte_pfn(pte); - if (pfn >= xen_start_info->first_p2m_pfn && - pfn < xen_start_info->first_p2m_pfn + xen_start_info->nr_p2m_frames) - pte = __pte_ma(pte_val_ma(pte) & ~_PAGE_RW); - - return pte; -} -#endif /* CONFIG_X86_64 */ - /* * Init-time set_pte while constructing initial pagetables, which * doesn't allow RO page table pages to be remapped RW. @@ -1600,13 +1565,37 @@ static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) * so always write the PTE directly and rely on Xen trapping and * emulating any updates as necessary. */ -static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) +__visible pte_t xen_make_pte_init(pteval_t pte) { - if (pte_mfn(pte) != INVALID_P2M_ENTRY) - pte = mask_rw_pte(ptep, pte); - else - pte = __pte_ma(0); +#ifdef CONFIG_X86_64 + unsigned long pfn; + + /* + * Pages belonging to the initial p2m list mapped outside the default + * address range must be mapped read-only. This region contains the + * page tables for mapping the p2m list, too, and page tables MUST be + * mapped read-only. + */ + pfn = (pte & PTE_PFN_MASK) >> PAGE_SHIFT; + if (xen_start_info->mfn_list < __START_KERNEL_map && + pfn >= xen_start_info->first_p2m_pfn && + pfn < xen_start_info->first_p2m_pfn + xen_start_info->nr_p2m_frames) + pte &= ~_PAGE_RW; +#endif + pte = pte_pfn_to_mfn(pte); + return native_make_pte(pte); +} +PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte_init); +static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) +{ +#ifdef CONFIG_X86_32 + /* If there's an existing pte, then don't allow _PAGE_RW to be set */ + if (pte_mfn(pte) != INVALID_P2M_ENTRY + && pte_val_ma(*ptep) & _PAGE_PRESENT) + pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) & + pte_val_ma(pte)); +#endif native_set_pte(ptep, pte); } @@ -2407,6 +2396,7 @@ static void __init xen_post_allocator_init(void) pv_mmu_ops.alloc_pud = xen_alloc_pud; pv_mmu_ops.release_pud = xen_release_pud; #endif + pv_mmu_ops.make_pte = PV_CALLEE_SAVE(xen_make_pte); #ifdef CONFIG_X86_64 pv_mmu_ops.write_cr3 = &xen_write_cr3; @@ -2455,7 +2445,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = { .pte_val = PV_CALLEE_SAVE(xen_pte_val), .pgd_val = PV_CALLEE_SAVE(xen_pgd_val), - .make_pte = PV_CALLEE_SAVE(xen_make_pte), + .make_pte = PV_CALLEE_SAVE(xen_make_pte_init), .make_pgd = PV_CALLEE_SAVE(xen_make_pgd), #ifdef CONFIG_X86_PAE diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index cab9f766bb06..dd2a49a8aacc 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c @@ -182,7 +182,7 @@ static void * __ref alloc_p2m_page(void) if (unlikely(!slab_is_available())) return alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE); - return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT); + return (void *)__get_free_page(GFP_KERNEL); } static void __ref free_p2m_page(void *p) diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 9466354d3e49..32bdc2c90297 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -547,7 +547,7 @@ void xen_pmu_init(int cpu) return; fail: - pr_warn_once("Could not initialize VPMU for cpu %d, error %d\n", + pr_info_once("Could not initialize VPMU for cpu %d, error %d\n", cpu, err); free_pages((unsigned long)xenpmu_data, 0); } diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 719cf291dcdf..0b4d04c8ab4d 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -322,6 +322,13 @@ static void __init xen_smp_prepare_boot_cpu(void) xen_filter_cpu_maps(); xen_setup_vcpu_info_placement(); } + + /* + * Setup vcpu_info for boot CPU. + */ + if (xen_hvm_domain()) + xen_vcpu_setup(0); + /* * The alternative logic (which patches the unlock/lock) runs before * the smp bootup up code is activated. Hence we need to set this up @@ -454,7 +461,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) #endif ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs); ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir)); - if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt)) + if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt)) BUG(); kfree(ctxt); @@ -492,7 +499,7 @@ static int xen_cpu_up(unsigned int cpu, struct task_struct *idle) if (rc) return rc; - rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL); + rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL); BUG_ON(rc); while (cpu_report_state(cpu) != CPU_ONLINE) @@ -520,7 +527,8 @@ static int xen_cpu_disable(void) static void xen_cpu_die(unsigned int cpu) { - while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) { + while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, + xen_vcpu_nr(cpu), NULL)) { __set_current_state(TASK_UNINTERRUPTIBLE); schedule_timeout(HZ/10); } @@ -536,7 +544,7 @@ static void xen_cpu_die(unsigned int cpu) static void xen_play_dead(void) /* used only with HOTPLUG_CPU */ { play_dead_common(); - HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL); + HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL); cpu_bringup(); /* * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down) @@ -576,7 +584,7 @@ static void stop_self(void *v) set_cpu_online(cpu, false); - HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL); + HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL); BUG(); } diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index 6deba5bc7e34..67356d29d74d 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -11,8 +11,6 @@ #include #include #include -#include -#include #include #include #include @@ -31,44 +29,6 @@ /* Xen may fire a timer up to this many ns early */ #define TIMER_SLOP 100000 -#define NS_PER_TICK (1000000000LL / HZ) - -/* snapshots of runstate info */ -static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot); - -/* unused ns of stolen time */ -static DEFINE_PER_CPU(u64, xen_residual_stolen); - -static void do_stolen_accounting(void) -{ - struct vcpu_runstate_info state; - struct vcpu_runstate_info *snap; - s64 runnable, offline, stolen; - cputime_t ticks; - - xen_get_runstate_snapshot(&state); - - WARN_ON(state.state != RUNSTATE_running); - - snap = this_cpu_ptr(&xen_runstate_snapshot); - - /* work out how much time the VCPU has not been runn*ing* */ - runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable]; - offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline]; - - *snap = state; - - /* Add the appropriate number of ticks of stolen time, - including any left-overs from last time. */ - stolen = runnable + offline + __this_cpu_read(xen_residual_stolen); - - if (stolen < 0) - stolen = 0; - - ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen); - __this_cpu_write(xen_residual_stolen, stolen); - account_steal_ticks(ticks); -} /* Get the TSC speed from Xen */ static unsigned long xen_tsc_khz(void) @@ -263,8 +223,10 @@ static int xen_vcpuop_shutdown(struct clock_event_device *evt) { int cpu = smp_processor_id(); - if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) || - HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) + if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, xen_vcpu_nr(cpu), + NULL) || + HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu), + NULL)) BUG(); return 0; @@ -274,7 +236,8 @@ static int xen_vcpuop_set_oneshot(struct clock_event_device *evt) { int cpu = smp_processor_id(); - if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) + if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu), + NULL)) BUG(); return 0; @@ -293,7 +256,8 @@ static int xen_vcpuop_set_next_event(unsigned long delta, /* Get an event anyway, even if the timeout is already expired */ single.flags = 0; - ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single); + ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, xen_vcpu_nr(cpu), + &single); BUG_ON(ret != 0); return ret; @@ -335,8 +299,6 @@ static irqreturn_t xen_timer_interrupt(int irq, void *dev_id) ret = IRQ_HANDLED; } - do_stolen_accounting(); - return ret; } @@ -394,13 +356,15 @@ void xen_timer_resume(void) return; for_each_online_cpu(cpu) { - if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) + if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, + xen_vcpu_nr(cpu), NULL)) BUG(); } } static const struct pv_time_ops xen_time_ops __initconst = { .sched_clock = xen_clocksource_read, + .steal_clock = xen_steal_clock, }; static void __init xen_time_init(void) @@ -414,7 +378,8 @@ static void __init xen_time_init(void) clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC); - if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) { + if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu), + NULL) == 0) { /* Successfully turned off 100Hz tick, so we have the vcpuop-based timer interface */ printk(KERN_DEBUG "Xen: using vcpuop timer interface\n"); @@ -431,6 +396,8 @@ static void __init xen_time_init(void) xen_setup_timer(cpu); xen_setup_cpu_clockevents(); + xen_time_setup_guest(); + if (xen_initial_domain()) pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier); } diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 4140b070f2e9..3cbce3b085e7 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -76,6 +76,7 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id); bool xen_vcpu_stolen(int vcpu); +void xen_vcpu_setup(int cpu); void xen_setup_vcpu_info_placement(void); #ifdef CONFIG_SMP diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index fd8017ce298a..e7a23f2a519a 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -98,6 +98,26 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ return result; \ } +#define ATOMIC_FETCH_OP(op) \ +static inline int atomic_fetch_##op(int i, atomic_t * v) \ +{ \ + unsigned long tmp; \ + int result; \ + \ + __asm__ __volatile__( \ + "1: l32i %1, %3, 0\n" \ + " wsr %1, scompare1\n" \ + " " #op " %0, %1, %2\n" \ + " s32c1i %0, %3, 0\n" \ + " bne %0, %1, 1b\n" \ + : "=&a" (result), "=&a" (tmp) \ + : "a" (i), "a" (v) \ + : "memory" \ + ); \ + \ + return result; \ +} + #else /* XCHAL_HAVE_S32C1I */ #define ATOMIC_OP(op) \ @@ -138,18 +158,42 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ return vval; \ } +#define ATOMIC_FETCH_OP(op) \ +static inline int atomic_fetch_##op(int i, atomic_t * v) \ +{ \ + unsigned int tmp, vval; \ + \ + __asm__ __volatile__( \ + " rsil a15,"__stringify(TOPLEVEL)"\n" \ + " l32i %0, %3, 0\n" \ + " " #op " %1, %0, %2\n" \ + " s32i %1, %3, 0\n" \ + " wsr a15, ps\n" \ + " rsync\n" \ + : "=&a" (vval), "=&a" (tmp) \ + : "a" (i), "a" (v) \ + : "a15", "memory" \ + ); \ + \ + return vval; \ +} + #endif /* XCHAL_HAVE_S32C1I */ -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/xtensa/include/asm/pgalloc.h b/arch/xtensa/include/asm/pgalloc.h index d38eb9237e64..1065bc8bcae5 100644 --- a/arch/xtensa/include/asm/pgalloc.h +++ b/arch/xtensa/include/asm/pgalloc.h @@ -44,7 +44,7 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, pte_t *ptep; int i; - ptep = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); + ptep = (pte_t *)__get_free_page(GFP_KERNEL); if (!ptep) return NULL; for (i = 0; i < 1024; i++) diff --git a/arch/xtensa/include/asm/spinlock.h b/arch/xtensa/include/asm/spinlock.h index 1d95fa5dcd10..a36221cf6363 100644 --- a/arch/xtensa/include/asm/spinlock.h +++ b/arch/xtensa/include/asm/spinlock.h @@ -11,6 +11,9 @@ #ifndef _XTENSA_SPINLOCK_H #define _XTENSA_SPINLOCK_H +#include +#include + /* * spinlock * @@ -29,8 +32,11 @@ */ #define arch_spin_is_locked(x) ((x)->slock != 0) -#define arch_spin_unlock_wait(lock) \ - do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) + +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->slock, !VAL); +} #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c index 7f4a1fdb1502..2725e08ef353 100644 --- a/arch/xtensa/mm/fault.c +++ b/arch/xtensa/mm/fault.c @@ -110,7 +110,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; diff --git a/arch/xtensa/platforms/xt2000/setup.c b/arch/xtensa/platforms/xt2000/setup.c index 5f4bd71971d6..4904c5c16918 100644 --- a/arch/xtensa/platforms/xt2000/setup.c +++ b/arch/xtensa/platforms/xt2000/setup.c @@ -113,7 +113,6 @@ void platform_heartbeat(void) } //#define RS_TABLE_SIZE 2 -//#define STD_COM_FLAGS (UPF_BOOT_AUTOCONF|UPF_SKIP_TEST) #define _SERIAL_PORT(_base,_irq) \ { \ diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 711e4d8de6fa..f70cc3bdfd01 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -26,6 +26,7 @@ #include #include #include +#include "blk.h" #define BIP_INLINE_VECS 4 @@ -53,7 +54,6 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, { struct bio_integrity_payload *bip; struct bio_set *bs = bio->bi_pool; - unsigned long idx = BIO_POOL_NONE; unsigned inline_vecs; if (!bs || !bs->bio_integrity_pool) { @@ -71,17 +71,19 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, memset(bip, 0, sizeof(*bip)); if (nr_vecs > inline_vecs) { + unsigned long idx = 0; + bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx, bs->bvec_integrity_pool); if (!bip->bip_vec) goto err; bip->bip_max_vcnt = bvec_nr_vecs(idx); + bip->bip_slab = idx; } else { bip->bip_vec = bip->bip_inline_vecs; bip->bip_max_vcnt = inline_vecs; } - bip->bip_slab = idx; bip->bip_bio = bio; bio->bi_integrity = bip; bio->bi_rw |= REQ_INTEGRITY; @@ -110,9 +112,7 @@ void bio_integrity_free(struct bio *bio) bip->bip_vec->bv_offset); if (bs && bs->bio_integrity_pool) { - if (bip->bip_slab != BIO_POOL_NONE) - bvec_free(bs->bvec_integrity_pool, bip->bip_vec, - bip->bip_slab); + bvec_free(bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab); mempool_free(bip, bs->bio_integrity_pool); } else { diff --git a/block/bio.c b/block/bio.c index 0e4aa42bc30d..54ee3846c3a5 100644 --- a/block/bio.c +++ b/block/bio.c @@ -43,7 +43,7 @@ * unsigned short */ #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) } -static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = { +static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = { BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES), }; #undef BV @@ -160,11 +160,15 @@ unsigned int bvec_nr_vecs(unsigned short idx) void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx) { - BIO_BUG_ON(idx >= BIOVEC_NR_POOLS); + if (!idx) + return; + idx--; + + BIO_BUG_ON(idx >= BVEC_POOL_NR); - if (idx == BIOVEC_MAX_IDX) + if (idx == BVEC_POOL_MAX) { mempool_free(bv, pool); - else { + } else { struct biovec_slab *bvs = bvec_slabs + idx; kmem_cache_free(bvs->slab, bv); @@ -206,7 +210,7 @@ struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx, * idx now points to the pool we want to allocate from. only the * 1-vec entry pool is mempool backed. */ - if (*idx == BIOVEC_MAX_IDX) { + if (*idx == BVEC_POOL_MAX) { fallback: bvl = mempool_alloc(pool, gfp_mask); } else { @@ -226,11 +230,12 @@ fallback: */ bvl = kmem_cache_alloc(bvs->slab, __gfp_mask); if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) { - *idx = BIOVEC_MAX_IDX; + *idx = BVEC_POOL_MAX; goto fallback; } } + (*idx)++; return bvl; } @@ -250,8 +255,7 @@ static void bio_free(struct bio *bio) __bio_free(bio); if (bs) { - if (bio_flagged(bio, BIO_OWNS_VEC)) - bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio)); + bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio)); /* * If we have front padding, adjust the bio pointer before freeing @@ -420,7 +424,6 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) gfp_t saved_gfp = gfp_mask; unsigned front_pad; unsigned inline_vecs; - unsigned long idx = BIO_POOL_NONE; struct bio_vec *bvl = NULL; struct bio *bio; void *p; @@ -480,6 +483,8 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) bio_init(bio); if (nr_iovecs > inline_vecs) { + unsigned long idx = 0; + bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool); if (!bvl && gfp_mask != saved_gfp) { punt_bios_to_rescuer(bs); @@ -490,13 +495,12 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) if (unlikely(!bvl)) goto err_free; - bio_set_flag(bio, BIO_OWNS_VEC); + bio->bi_flags |= idx << BVEC_POOL_OFFSET; } else if (nr_iovecs) { bvl = bio->bi_inline_vecs; } bio->bi_pool = bs; - bio->bi_flags |= idx << BIO_POOL_OFFSET; bio->bi_max_vecs = nr_iovecs; bio->bi_io_vec = bvl; return bio; @@ -568,7 +572,7 @@ EXPORT_SYMBOL(bio_phys_segments); */ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) { - BUG_ON(bio->bi_pool && BIO_POOL_IDX(bio) != BIO_POOL_NONE); + BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio)); /* * most users will be overriding ->bi_bdev with a new target, @@ -656,16 +660,15 @@ struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask, bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs); if (!bio) return NULL; - bio->bi_bdev = bio_src->bi_bdev; bio->bi_rw = bio_src->bi_rw; bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) goto integrity_clone; - if (bio->bi_rw & REQ_WRITE_SAME) { + if (bio_op(bio) == REQ_OP_WRITE_SAME) { bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0]; goto integrity_clone; } @@ -854,21 +857,20 @@ static void submit_bio_wait_endio(struct bio *bio) /** * submit_bio_wait - submit a bio, and wait until it completes - * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) * @bio: The &struct bio which describes the I/O * * Simple wrapper around submit_bio(). Returns 0 on success, or the error from * bio_endio() on failure. */ -int submit_bio_wait(int rw, struct bio *bio) +int submit_bio_wait(struct bio *bio) { struct submit_bio_ret ret; - rw |= REQ_SYNC; init_completion(&ret.event); bio->bi_private = &ret; bio->bi_end_io = submit_bio_wait_endio; - submit_bio(rw, bio); + bio->bi_rw |= REQ_SYNC; + submit_bio(bio); wait_for_completion_io(&ret.event); return ret.error; @@ -1099,7 +1101,6 @@ int bio_uncopy_user(struct bio *bio) bio_put(bio); return ret; } -EXPORT_SYMBOL(bio_uncopy_user); /** * bio_copy_user_iov - copy user data to bio @@ -1167,7 +1168,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q, goto out_bmd; if (iter->type & WRITE) - bio->bi_rw |= REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); ret = 0; @@ -1337,7 +1338,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, * set data direction, and check if mapped pages need bouncing */ if (iter->type & WRITE) - bio->bi_rw |= REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio_set_flag(bio, BIO_USER_MAPPED); @@ -1394,7 +1395,6 @@ void bio_unmap_user(struct bio *bio) __bio_unmap_user(bio); bio_put(bio); } -EXPORT_SYMBOL(bio_unmap_user); static void bio_map_kern_endio(struct bio *bio) { @@ -1530,7 +1530,7 @@ struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len, bio->bi_private = data; } else { bio->bi_end_io = bio_copy_kern_endio; - bio->bi_rw |= REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); } return bio; @@ -1540,7 +1540,6 @@ cleanup: bio_put(bio); return ERR_PTR(-ENOMEM); } -EXPORT_SYMBOL(bio_copy_kern); /* * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions @@ -1785,7 +1784,7 @@ struct bio *bio_split(struct bio *bio, int sectors, * Discards need a mutable bio_vec to accommodate the payload * required by the DSM TRIM and UNMAP commands. */ - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) split = bio_clone_bioset(bio, gfp, bs); else split = bio_clone_fast(bio, gfp, bs); @@ -1834,7 +1833,7 @@ EXPORT_SYMBOL_GPL(bio_trim); */ mempool_t *biovec_create_pool(int pool_entries) { - struct biovec_slab *bp = bvec_slabs + BIOVEC_MAX_IDX; + struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX; return mempool_create_slab_pool(pool_entries, bp->slab); } @@ -2011,7 +2010,7 @@ static void __init biovec_init_slabs(void) { int i; - for (i = 0; i < BIOVEC_NR_POOLS; i++) { + for (i = 0; i < BVEC_POOL_NR; i++) { int size; struct biovec_slab *bvs = bvec_slabs + i; diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 66e6f1aae02e..dd38e5ced4a3 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -905,7 +905,7 @@ static int blkcg_print_stat(struct seq_file *sf, void *v) return 0; } -struct cftype blkcg_files[] = { +static struct cftype blkcg_files[] = { { .name = "stat", .flags = CFTYPE_NOT_ON_ROOT, @@ -914,7 +914,7 @@ struct cftype blkcg_files[] = { { } /* terminate */ }; -struct cftype blkcg_legacy_files[] = { +static struct cftype blkcg_legacy_files[] = { { .name = "reset_stats", .write_u64 = blkcg_reset_stats, diff --git a/block/blk-core.c b/block/blk-core.c index 2475b1c72773..a687e9cc16c2 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -959,10 +959,10 @@ static void __freed_request(struct request_list *rl, int sync) * A request has just been released. Account for it, update the full and * congestion status, wake up any waiters. Called under q->queue_lock. */ -static void freed_request(struct request_list *rl, unsigned int flags) +static void freed_request(struct request_list *rl, int op, unsigned int flags) { struct request_queue *q = rl->q; - int sync = rw_is_sync(flags); + int sync = rw_is_sync(op, flags); q->nr_rqs[sync]--; rl->count[sync]--; @@ -1029,7 +1029,7 @@ static bool blk_rq_should_init_elevator(struct bio *bio) * Flush requests do not use the elevator so skip initialization. * This allows a request to share the flush and elevator data. */ - if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) + if (bio->bi_rw & (REQ_PREFLUSH | REQ_FUA)) return false; return true; @@ -1054,7 +1054,8 @@ static struct io_context *rq_ioc(struct bio *bio) /** * __get_request - get a free request * @rl: request list to allocate from - * @rw_flags: RW and SYNC flags + * @op: REQ_OP_READ/REQ_OP_WRITE + * @op_flags: rq_flag_bits * @bio: bio to allocate request for (can be %NULL) * @gfp_mask: allocation mask * @@ -1065,21 +1066,22 @@ static struct io_context *rq_ioc(struct bio *bio) * Returns ERR_PTR on failure, with @q->queue_lock held. * Returns request pointer on success, with @q->queue_lock *not held*. */ -static struct request *__get_request(struct request_list *rl, int rw_flags, - struct bio *bio, gfp_t gfp_mask) +static struct request *__get_request(struct request_list *rl, int op, + int op_flags, struct bio *bio, + gfp_t gfp_mask) { struct request_queue *q = rl->q; struct request *rq; struct elevator_type *et = q->elevator->type; struct io_context *ioc = rq_ioc(bio); struct io_cq *icq = NULL; - const bool is_sync = rw_is_sync(rw_flags) != 0; + const bool is_sync = rw_is_sync(op, op_flags) != 0; int may_queue; if (unlikely(blk_queue_dying(q))) return ERR_PTR(-ENODEV); - may_queue = elv_may_queue(q, rw_flags); + may_queue = elv_may_queue(q, op, op_flags); if (may_queue == ELV_MQUEUE_NO) goto rq_starved; @@ -1123,7 +1125,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, /* * Decide whether the new request will be managed by elevator. If - * so, mark @rw_flags and increment elvpriv. Non-zero elvpriv will + * so, mark @op_flags and increment elvpriv. Non-zero elvpriv will * prevent the current elevator from being destroyed until the new * request is freed. This guarantees icq's won't be destroyed and * makes creating new ones safe. @@ -1132,14 +1134,14 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, * it will be created after releasing queue_lock. */ if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) { - rw_flags |= REQ_ELVPRIV; + op_flags |= REQ_ELVPRIV; q->nr_rqs_elvpriv++; if (et->icq_cache && ioc) icq = ioc_lookup_icq(ioc, q); } if (blk_queue_io_stat(q)) - rw_flags |= REQ_IO_STAT; + op_flags |= REQ_IO_STAT; spin_unlock_irq(q->queue_lock); /* allocate and init request */ @@ -1149,10 +1151,10 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, blk_rq_init(q, rq); blk_rq_set_rl(rq, rl); - rq->cmd_flags = rw_flags | REQ_ALLOCED; + req_set_op_attrs(rq, op, op_flags | REQ_ALLOCED); /* init elvpriv */ - if (rw_flags & REQ_ELVPRIV) { + if (op_flags & REQ_ELVPRIV) { if (unlikely(et->icq_cache && !icq)) { if (ioc) icq = ioc_create_icq(ioc, q, gfp_mask); @@ -1178,7 +1180,7 @@ out: if (ioc_batching(q, ioc)) ioc->nr_batch_requests--; - trace_block_getrq(q, bio, rw_flags & 1); + trace_block_getrq(q, bio, op); return rq; fail_elvpriv: @@ -1208,7 +1210,7 @@ fail_alloc: * queue, but this is pretty rare. */ spin_lock_irq(q->queue_lock); - freed_request(rl, rw_flags); + freed_request(rl, op, op_flags); /* * in the very unlikely event that allocation failed and no @@ -1226,7 +1228,8 @@ rq_starved: /** * get_request - get a free request * @q: request_queue to allocate request from - * @rw_flags: RW and SYNC flags + * @op: REQ_OP_READ/REQ_OP_WRITE + * @op_flags: rq_flag_bits * @bio: bio to allocate request for (can be %NULL) * @gfp_mask: allocation mask * @@ -1237,17 +1240,18 @@ rq_starved: * Returns ERR_PTR on failure, with @q->queue_lock held. * Returns request pointer on success, with @q->queue_lock *not held*. */ -static struct request *get_request(struct request_queue *q, int rw_flags, - struct bio *bio, gfp_t gfp_mask) +static struct request *get_request(struct request_queue *q, int op, + int op_flags, struct bio *bio, + gfp_t gfp_mask) { - const bool is_sync = rw_is_sync(rw_flags) != 0; + const bool is_sync = rw_is_sync(op, op_flags) != 0; DEFINE_WAIT(wait); struct request_list *rl; struct request *rq; rl = blk_get_rl(q, bio); /* transferred to @rq on success */ retry: - rq = __get_request(rl, rw_flags, bio, gfp_mask); + rq = __get_request(rl, op, op_flags, bio, gfp_mask); if (!IS_ERR(rq)) return rq; @@ -1260,7 +1264,7 @@ retry: prepare_to_wait_exclusive(&rl->wait[is_sync], &wait, TASK_UNINTERRUPTIBLE); - trace_block_sleeprq(q, bio, rw_flags & 1); + trace_block_sleeprq(q, bio, op); spin_unlock_irq(q->queue_lock); io_schedule(); @@ -1289,11 +1293,16 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw, create_io_context(gfp_mask, q->node); spin_lock_irq(q->queue_lock); - rq = get_request(q, rw, NULL, gfp_mask); - if (IS_ERR(rq)) + rq = get_request(q, rw, 0, NULL, gfp_mask); + if (IS_ERR(rq)) { spin_unlock_irq(q->queue_lock); - /* q->queue_lock is unlocked at this point */ + return rq; + } + /* q->queue_lock is unlocked at this point */ + rq->__data_len = 0; + rq->__sector = (sector_t) -1; + rq->bio = rq->biotail = NULL; return rq; } @@ -1308,63 +1317,6 @@ struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask) } EXPORT_SYMBOL(blk_get_request); -/** - * blk_make_request - given a bio, allocate a corresponding struct request. - * @q: target request queue - * @bio: The bio describing the memory mappings that will be submitted for IO. - * It may be a chained-bio properly constructed by block/bio layer. - * @gfp_mask: gfp flags to be used for memory allocation - * - * blk_make_request is the parallel of generic_make_request for BLOCK_PC - * type commands. Where the struct request needs to be farther initialized by - * the caller. It is passed a &struct bio, which describes the memory info of - * the I/O transfer. - * - * The caller of blk_make_request must make sure that bi_io_vec - * are set to describe the memory buffers. That bio_data_dir() will return - * the needed direction of the request. (And all bio's in the passed bio-chain - * are properly set accordingly) - * - * If called under none-sleepable conditions, mapped bio buffers must not - * need bouncing, by calling the appropriate masked or flagged allocator, - * suitable for the target device. Otherwise the call to blk_queue_bounce will - * BUG. - * - * WARNING: When allocating/cloning a bio-chain, careful consideration should be - * given to how you allocate bios. In particular, you cannot use - * __GFP_DIRECT_RECLAIM for anything but the first bio in the chain. Otherwise - * you risk waiting for IO completion of a bio that hasn't been submitted yet, - * thus resulting in a deadlock. Alternatively bios should be allocated using - * bio_kmalloc() instead of bio_alloc(), as that avoids the mempool deadlock. - * If possible a big IO should be split into smaller parts when allocation - * fails. Partial allocation should not be an error, or you risk a live-lock. - */ -struct request *blk_make_request(struct request_queue *q, struct bio *bio, - gfp_t gfp_mask) -{ - struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask); - - if (IS_ERR(rq)) - return rq; - - blk_rq_set_block_pc(rq); - - for_each_bio(bio) { - struct bio *bounce_bio = bio; - int ret; - - blk_queue_bounce(q, &bounce_bio); - ret = blk_rq_append_bio(q, rq, bounce_bio); - if (unlikely(ret)) { - blk_put_request(rq); - return ERR_PTR(ret); - } - } - - return rq; -} -EXPORT_SYMBOL(blk_make_request); - /** * blk_rq_set_block_pc - initialize a request to type BLOCK_PC * @rq: request to be initialized @@ -1373,9 +1325,6 @@ EXPORT_SYMBOL(blk_make_request); void blk_rq_set_block_pc(struct request *rq) { rq->cmd_type = REQ_TYPE_BLOCK_PC; - rq->__data_len = 0; - rq->__sector = (sector_t) -1; - rq->bio = rq->biotail = NULL; memset(rq->__cmd, 0, sizeof(rq->__cmd)); } EXPORT_SYMBOL(blk_rq_set_block_pc); @@ -1491,13 +1440,14 @@ void __blk_put_request(struct request_queue *q, struct request *req) */ if (req->cmd_flags & REQ_ALLOCED) { unsigned int flags = req->cmd_flags; + int op = req_op(req); struct request_list *rl = blk_rq_rl(req); BUG_ON(!list_empty(&req->queuelist)); BUG_ON(ELV_ON_HASH(req)); blk_free_request(rl, req); - freed_request(rl, flags); + freed_request(rl, op, flags); blk_put_rl(rl); } } @@ -1712,7 +1662,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) { const bool sync = !!(bio->bi_rw & REQ_SYNC); struct blk_plug *plug; - int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT; + int el_ret, rw_flags = 0, where = ELEVATOR_INSERT_SORT; struct request *req; unsigned int request_count = 0; @@ -1731,7 +1681,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) return BLK_QC_T_NONE; } - if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { + if (bio->bi_rw & (REQ_PREFLUSH | REQ_FUA)) { spin_lock_irq(q->queue_lock); where = ELEVATOR_INSERT_FLUSH; goto get_rq; @@ -1772,15 +1722,19 @@ get_rq: * but we need to set it earlier to expose the sync flag to the * rq allocator and io schedulers. */ - rw_flags = bio_data_dir(bio); if (sync) rw_flags |= REQ_SYNC; + /* + * Add in META/PRIO flags, if set, before we get to the IO scheduler + */ + rw_flags |= (bio->bi_rw & (REQ_META | REQ_PRIO)); + /* * Grab a free request. This is might sleep but can not fail. * Returns with the queue unlocked. */ - req = get_request(q, rw_flags, bio, GFP_NOIO); + req = get_request(q, bio_data_dir(bio), rw_flags, bio, GFP_NOIO); if (IS_ERR(req)) { bio->bi_error = PTR_ERR(req); bio_endio(bio); @@ -1849,7 +1803,7 @@ static void handle_bad_sector(struct bio *bio) char b[BDEVNAME_SIZE]; printk(KERN_INFO "attempt to access beyond end of device\n"); - printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n", + printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n", bdevname(bio->bi_bdev, b), bio->bi_rw, (unsigned long long)bio_end_sector(bio), @@ -1964,25 +1918,30 @@ generic_make_request_checks(struct bio *bio) * drivers without flush support don't have to worry * about them. */ - if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && + if ((bio->bi_rw & (REQ_PREFLUSH | REQ_FUA)) && !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) { - bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA); + bio->bi_rw &= ~(REQ_PREFLUSH | REQ_FUA); if (!nr_sectors) { err = 0; goto end_io; } } - if ((bio->bi_rw & REQ_DISCARD) && - (!blk_queue_discard(q) || - ((bio->bi_rw & REQ_SECURE) && !blk_queue_secdiscard(q)))) { - err = -EOPNOTSUPP; - goto end_io; - } - - if (bio->bi_rw & REQ_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) { - err = -EOPNOTSUPP; - goto end_io; + switch (bio_op(bio)) { + case REQ_OP_DISCARD: + if (!blk_queue_discard(q)) + goto not_supported; + break; + case REQ_OP_SECURE_ERASE: + if (!blk_queue_secure_erase(q)) + goto not_supported; + break; + case REQ_OP_WRITE_SAME: + if (!bdev_write_same(bio->bi_bdev)) + goto not_supported; + break; + default: + break; } /* @@ -1999,6 +1958,8 @@ generic_make_request_checks(struct bio *bio) trace_block_bio_queue(q, bio); return true; +not_supported: + err = -EOPNOTSUPP; end_io: bio->bi_error = err; bio_endio(bio); @@ -2094,7 +2055,6 @@ EXPORT_SYMBOL(generic_make_request); /** * submit_bio - submit a bio to the block device layer for I/O - * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) * @bio: The &struct bio which describes the I/O * * submit_bio() is very similar in purpose to generic_make_request(), and @@ -2102,10 +2062,8 @@ EXPORT_SYMBOL(generic_make_request); * interfaces; @bio must be presetup and ready for I/O. * */ -blk_qc_t submit_bio(int rw, struct bio *bio) +blk_qc_t submit_bio(struct bio *bio) { - bio->bi_rw |= rw; - /* * If it's a regular read/write or a barrier with data attached, * go through the normal accounting stuff before submission. @@ -2113,12 +2071,12 @@ blk_qc_t submit_bio(int rw, struct bio *bio) if (bio_has_data(bio)) { unsigned int count; - if (unlikely(rw & REQ_WRITE_SAME)) + if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) count = bdev_logical_block_size(bio->bi_bdev) >> 9; else count = bio_sectors(bio); - if (rw & WRITE) { + if (op_is_write(bio_op(bio))) { count_vm_events(PGPGOUT, count); } else { task_io_account_read(bio->bi_iter.bi_size); @@ -2129,7 +2087,7 @@ blk_qc_t submit_bio(int rw, struct bio *bio) char b[BDEVNAME_SIZE]; printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", current->comm, task_pid_nr(current), - (rw & WRITE) ? "WRITE" : "READ", + op_is_write(bio_op(bio)) ? "WRITE" : "READ", (unsigned long long)bio->bi_iter.bi_sector, bdevname(bio->bi_bdev, b), count); @@ -2160,7 +2118,7 @@ EXPORT_SYMBOL(submit_bio); static int blk_cloned_rq_check_limits(struct request_queue *q, struct request *rq) { - if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, rq->cmd_flags)) { + if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) { printk(KERN_ERR "%s: over max size limit.\n", __func__); return -EIO; } @@ -2216,7 +2174,7 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq) */ BUG_ON(blk_queued_rq(rq)); - if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA)) + if (rq->cmd_flags & (REQ_PREFLUSH | REQ_FUA)) where = ELEVATOR_INSERT_FLUSH; add_acct_request(q, rq, where); @@ -2979,8 +2937,7 @@ EXPORT_SYMBOL_GPL(__blk_end_request_err); void blk_rq_bio_prep(struct request_queue *q, struct request *rq, struct bio *bio) { - /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */ - rq->cmd_flags |= bio->bi_rw & REQ_WRITE; + req_set_op(rq, bio_op(bio)); if (bio_has_data(bio)) rq->nr_phys_segments = bio_phys_segments(q, bio); @@ -3065,7 +3022,8 @@ EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); static void __blk_rq_prep_clone(struct request *dst, struct request *src) { dst->cpu = src->cpu; - dst->cmd_flags |= (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE; + req_set_op_attrs(dst, req_op(src), + (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE); dst->cmd_type = src->cmd_type; dst->__sector = blk_rq_pos(src); dst->__data_len = blk_rq_bytes(src); @@ -3310,7 +3268,7 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) /* * rq is already accounted, so use raw insert */ - if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA)) + if (rq->cmd_flags & (REQ_PREFLUSH | REQ_FUA)) __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH); else __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE); @@ -3377,6 +3335,7 @@ bool blk_poll(struct request_queue *q, blk_qc_t cookie) return false; } +EXPORT_SYMBOL_GPL(blk_poll); #ifdef CONFIG_PM /** diff --git a/block/blk-exec.c b/block/blk-exec.c index 3fec8a29d0fa..7ea04325d02f 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -62,7 +62,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, /* * don't check dying flag for MQ because the request won't - * be resued after dying flag is set + * be reused after dying flag is set */ if (q->mq_ops) { blk_mq_insert_request(rq, at_head, true, false); diff --git a/block/blk-flush.c b/block/blk-flush.c index b1c91d229e5e..d308def812db 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -10,8 +10,8 @@ * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request * properties and hardware capability. * - * If a request doesn't have data, only REQ_FLUSH makes sense, which - * indicates a simple flush request. If there is data, REQ_FLUSH indicates + * If a request doesn't have data, only REQ_PREFLUSH makes sense, which + * indicates a simple flush request. If there is data, REQ_PREFLUSH indicates * that the device cache should be flushed before the data is executed, and * REQ_FUA means that the data must be on non-volatile media on request * completion. @@ -20,16 +20,16 @@ * difference. The requests are either completed immediately if there's no * data or executed as normal requests otherwise. * - * If the device has writeback cache and supports FUA, REQ_FLUSH is + * If the device has writeback cache and supports FUA, REQ_PREFLUSH is * translated to PREFLUSH but REQ_FUA is passed down directly with DATA. * - * If the device has writeback cache and doesn't support FUA, REQ_FLUSH is - * translated to PREFLUSH and REQ_FUA to POSTFLUSH. + * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH + * is translated to PREFLUSH and REQ_FUA to POSTFLUSH. * * The actual execution of flush is double buffered. Whenever a request * needs to execute PRE or POSTFLUSH, it queues at * fq->flush_queue[fq->flush_pending_idx]. Once certain criteria are met, a - * flush is issued and the pending_idx is toggled. When the flush + * REQ_OP_FLUSH is issued and the pending_idx is toggled. When the flush * completes, all the requests which were pending are proceeded to the next * step. This allows arbitrary merging of different types of FLUSH/FUA * requests. @@ -103,7 +103,7 @@ static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq) policy |= REQ_FSEQ_DATA; if (fflags & (1UL << QUEUE_FLAG_WC)) { - if (rq->cmd_flags & REQ_FLUSH) + if (rq->cmd_flags & REQ_PREFLUSH) policy |= REQ_FSEQ_PREFLUSH; if (!(fflags & (1UL << QUEUE_FLAG_FUA)) && (rq->cmd_flags & REQ_FUA)) @@ -330,7 +330,7 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) } flush_rq->cmd_type = REQ_TYPE_FS; - flush_rq->cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ; + req_set_op_attrs(flush_rq, REQ_OP_FLUSH, WRITE_FLUSH | REQ_FLUSH_SEQ); flush_rq->rq_disk = first_rq->rq_disk; flush_rq->end_io = flush_end_io; @@ -391,9 +391,9 @@ void blk_insert_flush(struct request *rq) /* * @policy now records what operations need to be done. Adjust - * REQ_FLUSH and FUA for the driver. + * REQ_PREFLUSH and FUA for the driver. */ - rq->cmd_flags &= ~REQ_FLUSH; + rq->cmd_flags &= ~REQ_PREFLUSH; if (!(fflags & (1UL << QUEUE_FLAG_FUA))) rq->cmd_flags &= ~REQ_FUA; @@ -485,8 +485,9 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, bio = bio_alloc(gfp_mask, 0); bio->bi_bdev = bdev; + bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH); - ret = submit_bio_wait(WRITE_FLUSH, bio); + ret = submit_bio_wait(bio); /* * The driver must store the error location in ->bi_sector, if diff --git a/block/blk-lib.c b/block/blk-lib.c index 9e29dc351695..083e56f72308 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -9,33 +9,46 @@ #include "blk.h" -static struct bio *next_bio(struct bio *bio, int rw, unsigned int nr_pages, +static struct bio *next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp) { struct bio *new = bio_alloc(gfp, nr_pages); if (bio) { bio_chain(bio, new); - submit_bio(rw, bio); + submit_bio(bio); } return new; } int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, - sector_t nr_sects, gfp_t gfp_mask, int type, struct bio **biop) + sector_t nr_sects, gfp_t gfp_mask, int flags, + struct bio **biop) { struct request_queue *q = bdev_get_queue(bdev); struct bio *bio = *biop; unsigned int granularity; + enum req_op op; int alignment; if (!q) return -ENXIO; - if (!blk_queue_discard(q)) - return -EOPNOTSUPP; - if ((type & REQ_SECURE) && !blk_queue_secdiscard(q)) - return -EOPNOTSUPP; + + if (flags & BLKDEV_DISCARD_SECURE) { + if (flags & BLKDEV_DISCARD_ZERO) + return -EOPNOTSUPP; + if (!blk_queue_secure_erase(q)) + return -EOPNOTSUPP; + op = REQ_OP_SECURE_ERASE; + } else { + if (!blk_queue_discard(q)) + return -EOPNOTSUPP; + if ((flags & BLKDEV_DISCARD_ZERO) && + !q->limits.discard_zeroes_data) + return -EOPNOTSUPP; + op = REQ_OP_DISCARD; + } /* Zero-sector (unknown) and one-sector granularities are the same. */ granularity = max(q->limits.discard_granularity >> 9, 1U); @@ -62,9 +75,10 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, req_sects = end_sect - sector; } - bio = next_bio(bio, type, 1, gfp_mask); + bio = next_bio(bio, 1, gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; + bio_set_op_attrs(bio, op, 0); bio->bi_iter.bi_size = req_sects << 9; nr_sects -= req_sects; @@ -98,20 +112,16 @@ EXPORT_SYMBOL(__blkdev_issue_discard); int blkdev_issue_discard(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, unsigned long flags) { - int type = REQ_WRITE | REQ_DISCARD; struct bio *bio = NULL; struct blk_plug plug; int ret; - if (flags & BLKDEV_DISCARD_SECURE) - type |= REQ_SECURE; - blk_start_plug(&plug); - ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, type, + ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags, &bio); if (!ret && bio) { - ret = submit_bio_wait(type, bio); - if (ret == -EOPNOTSUPP) + ret = submit_bio_wait(bio); + if (ret == -EOPNOTSUPP && !(flags & BLKDEV_DISCARD_ZERO)) ret = 0; bio_put(bio); } @@ -148,13 +158,14 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, max_write_same_sectors = UINT_MAX >> 9; while (nr_sects) { - bio = next_bio(bio, REQ_WRITE | REQ_WRITE_SAME, 1, gfp_mask); + bio = next_bio(bio, 1, gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; bio->bi_vcnt = 1; bio->bi_io_vec->bv_page = page; bio->bi_io_vec->bv_offset = 0; bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev); + bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0); if (nr_sects > max_write_same_sectors) { bio->bi_iter.bi_size = max_write_same_sectors << 9; @@ -167,10 +178,10 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, } if (bio) { - ret = submit_bio_wait(REQ_WRITE | REQ_WRITE_SAME, bio); + ret = submit_bio_wait(bio); bio_put(bio); } - return ret != -EOPNOTSUPP ? ret : 0; + return ret; } EXPORT_SYMBOL(blkdev_issue_write_same); @@ -193,11 +204,11 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, unsigned int sz; while (nr_sects != 0) { - bio = next_bio(bio, WRITE, - min(nr_sects, (sector_t)BIO_MAX_PAGES), + bio = next_bio(bio, min(nr_sects, (sector_t)BIO_MAX_PAGES), gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); while (nr_sects != 0) { sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects); @@ -210,7 +221,7 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, } if (bio) { - ret = submit_bio_wait(WRITE, bio); + ret = submit_bio_wait(bio); bio_put(bio); return ret; } @@ -241,11 +252,11 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, bool discard) { - struct request_queue *q = bdev_get_queue(bdev); - - if (discard && blk_queue_discard(q) && q->limits.discard_zeroes_data && - blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, 0) == 0) - return 0; + if (discard) { + if (!blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, + BLKDEV_DISCARD_ZERO)) + return 0; + } if (bdev_write_same(bdev) && blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, diff --git a/block/blk-map.c b/block/blk-map.c index b9f88b7751fb..b8657fa8dc9a 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -9,21 +9,26 @@ #include "blk.h" -int blk_rq_append_bio(struct request_queue *q, struct request *rq, - struct bio *bio) +/* + * Append a bio to a passthrough request. Only works can be merged into + * the request based on the driver constraints. + */ +int blk_rq_append_bio(struct request *rq, struct bio *bio) { - if (!rq->bio) - blk_rq_bio_prep(q, rq, bio); - else if (!ll_back_merge_fn(q, rq, bio)) - return -EINVAL; - else { + if (!rq->bio) { + blk_rq_bio_prep(rq->q, rq, bio); + } else { + if (!ll_back_merge_fn(rq->q, rq, bio)) + return -EINVAL; + rq->biotail->bi_next = bio; rq->biotail = bio; - rq->__data_len += bio->bi_iter.bi_size; } + return 0; } +EXPORT_SYMBOL(blk_rq_append_bio); static int __blk_rq_unmap_user(struct bio *bio) { @@ -71,7 +76,7 @@ static int __blk_rq_map_user_iov(struct request *rq, */ bio_get(bio); - ret = blk_rq_append_bio(q, rq, bio); + ret = blk_rq_append_bio(rq, bio); if (ret) { bio_endio(bio); __blk_rq_unmap_user(orig_bio); @@ -224,12 +229,12 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, return PTR_ERR(bio); if (!reading) - bio->bi_rw |= REQ_WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); if (do_copy) rq->cmd_flags |= REQ_COPY_USER; - ret = blk_rq_append_bio(q, rq, bio); + ret = blk_rq_append_bio(rq, bio); if (unlikely(ret)) { /* request is too big */ bio_put(bio); diff --git a/block/blk-merge.c b/block/blk-merge.c index 261353166dcf..41cbd4878958 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -172,9 +172,9 @@ void blk_queue_split(struct request_queue *q, struct bio **bio, struct bio *split, *res; unsigned nsegs; - if ((*bio)->bi_rw & REQ_DISCARD) + if (bio_op(*bio) == REQ_OP_DISCARD) split = blk_bio_discard_split(q, *bio, bs, &nsegs); - else if ((*bio)->bi_rw & REQ_WRITE_SAME) + else if (bio_op(*bio) == REQ_OP_WRITE_SAME) split = blk_bio_write_same_split(q, *bio, bs, &nsegs); else split = blk_bio_segment_split(q, *bio, q->bio_split, &nsegs); @@ -213,10 +213,10 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, * This should probably be returning 0, but blk_add_request_payload() * (Christoph!!!!) */ - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) return 1; - if (bio->bi_rw & REQ_WRITE_SAME) + if (bio_op(bio) == REQ_OP_WRITE_SAME) return 1; fbio = bio; @@ -385,7 +385,7 @@ static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, nsegs = 0; cluster = blk_queue_cluster(q); - if (bio->bi_rw & REQ_DISCARD) { + if (bio_op(bio) == REQ_OP_DISCARD) { /* * This is a hack - drivers should be neither modifying the * biovec, nor relying on bi_vcnt - but because of @@ -400,7 +400,7 @@ static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, return 0; } - if (bio->bi_rw & REQ_WRITE_SAME) { + if (bio_op(bio) == REQ_OP_WRITE_SAME) { single_segment: *sg = sglist; bvec = bio_iovec(bio); @@ -439,7 +439,7 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq, } if (q->dma_drain_size && q->dma_drain_needed(rq)) { - if (rq->cmd_flags & REQ_WRITE) + if (op_is_write(req_op(rq))) memset(q->dma_drain_buffer, 0, q->dma_drain_size); sg_unmark_end(sg); @@ -500,7 +500,7 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req, integrity_req_gap_back_merge(req, bio)) return 0; if (blk_rq_sectors(req) + bio_sectors(bio) > - blk_rq_get_max_sectors(req)) { + blk_rq_get_max_sectors(req, blk_rq_pos(req))) { req->cmd_flags |= REQ_NOMERGE; if (req == q->last_merge) q->last_merge = NULL; @@ -524,7 +524,7 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req, integrity_req_gap_front_merge(req, bio)) return 0; if (blk_rq_sectors(req) + bio_sectors(bio) > - blk_rq_get_max_sectors(req)) { + blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) { req->cmd_flags |= REQ_NOMERGE; if (req == q->last_merge) q->last_merge = NULL; @@ -570,7 +570,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, * Will it become too large? */ if ((blk_rq_sectors(req) + blk_rq_sectors(next)) > - blk_rq_get_max_sectors(req)) + blk_rq_get_max_sectors(req, blk_rq_pos(req))) return 0; total_phys_segments = req->nr_phys_segments + next->nr_phys_segments; @@ -649,7 +649,7 @@ static int attempt_merge(struct request_queue *q, struct request *req, if (!rq_mergeable(req) || !rq_mergeable(next)) return 0; - if (!blk_check_merge_flags(req->cmd_flags, next->cmd_flags)) + if (req_op(req) != req_op(next)) return 0; /* @@ -663,7 +663,7 @@ static int attempt_merge(struct request_queue *q, struct request *req, || req_no_special_merge(next)) return 0; - if (req->cmd_flags & REQ_WRITE_SAME && + if (req_op(req) == REQ_OP_WRITE_SAME && !blk_write_same_mergeable(req->bio, next->bio)) return 0; @@ -743,6 +743,12 @@ int attempt_front_merge(struct request_queue *q, struct request *rq) int blk_attempt_req_merge(struct request_queue *q, struct request *rq, struct request *next) { + struct elevator_queue *e = q->elevator; + + if (e->type->ops.elevator_allow_rq_merge_fn) + if (!e->type->ops.elevator_allow_rq_merge_fn(q, rq, next)) + return 0; + return attempt_merge(q, rq, next); } @@ -751,7 +757,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) if (!rq_mergeable(rq) || !bio_mergeable(bio)) return false; - if (!blk_check_merge_flags(rq->cmd_flags, bio->bi_rw)) + if (req_op(rq) != bio_op(bio)) return false; /* different data direction or already started, don't merge */ @@ -767,7 +773,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) return false; /* must be using the same buffer */ - if (rq->cmd_flags & REQ_WRITE_SAME && + if (req_op(rq) == REQ_OP_WRITE_SAME && !blk_write_same_mergeable(rq->bio, bio)) return false; diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 56a0c37a3d06..729bac3a673b 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -485,6 +485,32 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, } EXPORT_SYMBOL(blk_mq_tagset_busy_iter); +int blk_mq_reinit_tagset(struct blk_mq_tag_set *set) +{ + int i, j, ret = 0; + + if (!set->ops->reinit_request) + goto out; + + for (i = 0; i < set->nr_hw_queues; i++) { + struct blk_mq_tags *tags = set->tags[i]; + + for (j = 0; j < tags->nr_tags; j++) { + if (!tags->rqs[j]) + continue; + + ret = set->ops->reinit_request(set->driver_data, + tags->rqs[j]); + if (ret) + goto out; + } + } + +out: + return ret; +} +EXPORT_SYMBOL_GPL(blk_mq_reinit_tagset); + void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, void *priv) { diff --git a/block/blk-mq.c b/block/blk-mq.c index f9b9049b1284..576e7112f807 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -159,16 +159,17 @@ bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) EXPORT_SYMBOL(blk_mq_can_queue); static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, - struct request *rq, unsigned int rw_flags) + struct request *rq, int op, + unsigned int op_flags) { if (blk_queue_io_stat(q)) - rw_flags |= REQ_IO_STAT; + op_flags |= REQ_IO_STAT; INIT_LIST_HEAD(&rq->queuelist); /* csd/requeue_work/fifo_time is initialized before use */ rq->q = q; rq->mq_ctx = ctx; - rq->cmd_flags |= rw_flags; + req_set_op_attrs(rq, op, op_flags); /* do not touch atomic flags, it needs atomic ops against the timer */ rq->cpu = -1; INIT_HLIST_NODE(&rq->hash); @@ -203,11 +204,11 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, rq->end_io_data = NULL; rq->next_rq = NULL; - ctx->rq_dispatched[rw_is_sync(rw_flags)]++; + ctx->rq_dispatched[rw_is_sync(op, op_flags)]++; } static struct request * -__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw) +__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int op, int op_flags) { struct request *rq; unsigned int tag; @@ -222,7 +223,7 @@ __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw) } rq->tag = tag; - blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw); + blk_mq_rq_ctx_init(data->q, data->ctx, rq, op, op_flags); return rq; } @@ -246,7 +247,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, hctx = q->mq_ops->map_queue(q, ctx->cpu); blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw); + rq = __blk_mq_alloc_request(&alloc_data, rw, 0); if (!rq && !(flags & BLK_MQ_REQ_NOWAIT)) { __blk_mq_run_hw_queue(hctx); blk_mq_put_ctx(ctx); @@ -254,7 +255,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw); + rq = __blk_mq_alloc_request(&alloc_data, rw, 0); ctx = alloc_data.ctx; } blk_mq_put_ctx(ctx); @@ -262,10 +263,53 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, blk_queue_exit(q); return ERR_PTR(-EWOULDBLOCK); } + + rq->__data_len = 0; + rq->__sector = (sector_t) -1; + rq->bio = rq->biotail = NULL; return rq; } EXPORT_SYMBOL(blk_mq_alloc_request); +struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw, + unsigned int flags, unsigned int hctx_idx) +{ + struct blk_mq_hw_ctx *hctx; + struct blk_mq_ctx *ctx; + struct request *rq; + struct blk_mq_alloc_data alloc_data; + int ret; + + /* + * If the tag allocator sleeps we could get an allocation for a + * different hardware context. No need to complicate the low level + * allocator for this for the rare use case of a command tied to + * a specific queue. + */ + if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT))) + return ERR_PTR(-EINVAL); + + if (hctx_idx >= q->nr_hw_queues) + return ERR_PTR(-EIO); + + ret = blk_queue_enter(q, true); + if (ret) + return ERR_PTR(ret); + + hctx = q->queue_hw_ctx[hctx_idx]; + ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask)); + + blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); + rq = __blk_mq_alloc_request(&alloc_data, rw, 0); + if (!rq) { + blk_queue_exit(q); + return ERR_PTR(-EWOULDBLOCK); + } + + return rq; +} +EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); + static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, struct request *rq) { @@ -784,7 +828,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) switch (ret) { case BLK_MQ_RQ_QUEUE_OK: queued++; - continue; + break; case BLK_MQ_RQ_QUEUE_BUSY: list_add(&rq->queuelist, &rq_list); __blk_mq_requeue_request(rq); @@ -1169,28 +1213,29 @@ static struct request *blk_mq_map_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx; struct blk_mq_ctx *ctx; struct request *rq; - int rw = bio_data_dir(bio); + int op = bio_data_dir(bio); + int op_flags = 0; struct blk_mq_alloc_data alloc_data; blk_queue_enter_live(q); ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); - if (rw_is_sync(bio->bi_rw)) - rw |= REQ_SYNC; + if (rw_is_sync(bio_op(bio), bio->bi_rw)) + op_flags |= REQ_SYNC; - trace_block_getrq(q, bio, rw); + trace_block_getrq(q, bio, op); blk_mq_set_alloc_data(&alloc_data, q, BLK_MQ_REQ_NOWAIT, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw); + rq = __blk_mq_alloc_request(&alloc_data, op, op_flags); if (unlikely(!rq)) { __blk_mq_run_hw_queue(hctx); blk_mq_put_ctx(ctx); - trace_block_sleeprq(q, bio, rw); + trace_block_sleeprq(q, bio, op); ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); blk_mq_set_alloc_data(&alloc_data, q, 0, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw); + rq = __blk_mq_alloc_request(&alloc_data, op, op_flags); ctx = alloc_data.ctx; hctx = alloc_data.hctx; } @@ -1244,8 +1289,8 @@ static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie) */ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) { - const int is_sync = rw_is_sync(bio->bi_rw); - const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA); + const int is_sync = rw_is_sync(bio_op(bio), bio->bi_rw); + const int is_flush_fua = bio->bi_rw & (REQ_PREFLUSH | REQ_FUA); struct blk_map_ctx data; struct request *rq; unsigned int request_count = 0; @@ -1338,8 +1383,8 @@ done: */ static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio) { - const int is_sync = rw_is_sync(bio->bi_rw); - const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA); + const int is_sync = rw_is_sync(bio_op(bio), bio->bi_rw); + const int is_flush_fua = bio->bi_rw & (REQ_PREFLUSH | REQ_FUA); struct blk_plug *plug; unsigned int request_count = 0; struct blk_map_ctx data; diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 99205965f559..f87a7e747d36 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -379,6 +379,11 @@ static ssize_t queue_wc_store(struct request_queue *q, const char *page, return count; } +static ssize_t queue_dax_show(struct request_queue *q, char *page) +{ + return queue_var_show(blk_queue_dax(q), page); +} + static struct queue_sysfs_entry queue_requests_entry = { .attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR }, .show = queue_requests_show, @@ -516,6 +521,11 @@ static struct queue_sysfs_entry queue_wc_entry = { .store = queue_wc_store, }; +static struct queue_sysfs_entry queue_dax_entry = { + .attr = {.name = "dax", .mode = S_IRUGO }, + .show = queue_dax_show, +}; + static struct attribute *default_attrs[] = { &queue_requests_entry.attr, &queue_ra_entry.attr, @@ -542,6 +552,7 @@ static struct attribute *default_attrs[] = { &queue_random_entry.attr, &queue_poll_entry.attr, &queue_wc_entry.attr, + &queue_dax_entry.attr, NULL, }; diff --git a/block/blk.h b/block/blk.h index 70e4aee9cdcb..c37492f5edaa 100644 --- a/block/blk.h +++ b/block/blk.h @@ -64,8 +64,6 @@ void blk_exit_rl(struct request_list *rl); void init_request_from_bio(struct request *req, struct bio *bio); void blk_rq_bio_prep(struct request_queue *q, struct request *rq, struct bio *bio); -int blk_rq_append_bio(struct request_queue *q, struct request *rq, - struct bio *bio); void blk_queue_bypass_start(struct request_queue *q); void blk_queue_bypass_end(struct request_queue *q); void blk_dequeue_request(struct request *rq); diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 4a349787bc62..acabba198de9 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include @@ -22,28 +22,28 @@ */ /* max queue in one round of service */ static const int cfq_quantum = 8; -static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 }; +static const u64 cfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 }; /* maximum backwards seek, in KiB */ static const int cfq_back_max = 16 * 1024; /* penalty of a backwards seek */ static const int cfq_back_penalty = 2; -static const int cfq_slice_sync = HZ / 10; -static int cfq_slice_async = HZ / 25; +static const u64 cfq_slice_sync = NSEC_PER_SEC / 10; +static u64 cfq_slice_async = NSEC_PER_SEC / 25; static const int cfq_slice_async_rq = 2; -static int cfq_slice_idle = HZ / 125; -static int cfq_group_idle = HZ / 125; -static const int cfq_target_latency = HZ * 3/10; /* 300 ms */ +static u64 cfq_slice_idle = NSEC_PER_SEC / 125; +static u64 cfq_group_idle = NSEC_PER_SEC / 125; +static const u64 cfq_target_latency = (u64)NSEC_PER_SEC * 3/10; /* 300 ms */ static const int cfq_hist_divisor = 4; /* * offset from end of service tree */ -#define CFQ_IDLE_DELAY (HZ / 5) +#define CFQ_IDLE_DELAY (NSEC_PER_SEC / 5) /* * below this threshold, we consider thinktime immediate */ -#define CFQ_MIN_TT (2) +#define CFQ_MIN_TT (2 * NSEC_PER_SEC / HZ) #define CFQ_SLICE_SCALE (5) #define CFQ_HW_QUEUE_MIN (5) @@ -73,11 +73,11 @@ static struct kmem_cache *cfq_pool; #define CFQ_WEIGHT_LEGACY_MAX 1000 struct cfq_ttime { - unsigned long last_end_request; + u64 last_end_request; - unsigned long ttime_total; + u64 ttime_total; + u64 ttime_mean; unsigned long ttime_samples; - unsigned long ttime_mean; }; /* @@ -94,7 +94,7 @@ struct cfq_rb_root { struct cfq_ttime ttime; }; #define CFQ_RB_ROOT (struct cfq_rb_root) { .rb = RB_ROOT, \ - .ttime = {.last_end_request = jiffies,},} + .ttime = {.last_end_request = ktime_get_ns(),},} /* * Per process-grouping structure @@ -109,7 +109,7 @@ struct cfq_queue { /* service_tree member */ struct rb_node rb_node; /* service_tree key */ - unsigned long rb_key; + u64 rb_key; /* prio tree member */ struct rb_node p_node; /* prio tree root we belong to, if any */ @@ -126,13 +126,13 @@ struct cfq_queue { struct list_head fifo; /* time when queue got scheduled in to dispatch first request. */ - unsigned long dispatch_start; - unsigned int allocated_slice; - unsigned int slice_dispatch; + u64 dispatch_start; + u64 allocated_slice; + u64 slice_dispatch; /* time when first request from queue completed and slice started. */ - unsigned long slice_start; - unsigned long slice_end; - long slice_resid; + u64 slice_start; + u64 slice_end; + s64 slice_resid; /* pending priority requests */ int prio_pending; @@ -141,7 +141,7 @@ struct cfq_queue { /* io prio of this group */ unsigned short ioprio, org_ioprio; - unsigned short ioprio_class; + unsigned short ioprio_class, org_ioprio_class; pid_t pid; @@ -290,7 +290,7 @@ struct cfq_group { struct cfq_rb_root service_trees[2][3]; struct cfq_rb_root service_tree_idle; - unsigned long saved_wl_slice; + u64 saved_wl_slice; enum wl_type_t saved_wl_type; enum wl_class_t saved_wl_class; @@ -329,7 +329,7 @@ struct cfq_data { */ enum wl_class_t serving_wl_class; enum wl_type_t serving_wl_type; - unsigned long workload_expires; + u64 workload_expires; struct cfq_group *serving_group; /* @@ -362,7 +362,7 @@ struct cfq_data { /* * idle window management */ - struct timer_list idle_slice_timer; + struct hrtimer idle_slice_timer; struct work_struct unplug_work; struct cfq_queue *active_queue; @@ -374,22 +374,22 @@ struct cfq_data { * tunables, see top of file */ unsigned int cfq_quantum; - unsigned int cfq_fifo_expire[2]; unsigned int cfq_back_penalty; unsigned int cfq_back_max; - unsigned int cfq_slice[2]; unsigned int cfq_slice_async_rq; - unsigned int cfq_slice_idle; - unsigned int cfq_group_idle; unsigned int cfq_latency; - unsigned int cfq_target_latency; + u64 cfq_fifo_expire[2]; + u64 cfq_slice[2]; + u64 cfq_slice_idle; + u64 cfq_group_idle; + u64 cfq_target_latency; /* * Fallback dummy cfqq for extreme OOM conditions */ struct cfq_queue oom_cfqq; - unsigned long last_delayed_sync; + u64 last_delayed_sync; }; static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd); @@ -667,15 +667,16 @@ static inline void cfqg_put(struct cfq_group *cfqg) } while (0) static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, - struct cfq_group *curr_cfqg, int rw) + struct cfq_group *curr_cfqg, int op, + int op_flags) { - blkg_rwstat_add(&cfqg->stats.queued, rw, 1); + blkg_rwstat_add(&cfqg->stats.queued, op, op_flags, 1); cfqg_stats_end_empty_time(&cfqg->stats); cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg); } static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, - unsigned long time, unsigned long unaccounted_time) + uint64_t time, unsigned long unaccounted_time) { blkg_stat_add(&cfqg->stats.time, time); #ifdef CONFIG_DEBUG_BLK_CGROUP @@ -683,26 +684,30 @@ static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, #endif } -static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) +static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op, + int op_flags) { - blkg_rwstat_add(&cfqg->stats.queued, rw, -1); + blkg_rwstat_add(&cfqg->stats.queued, op, op_flags, -1); } -static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) +static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op, + int op_flags) { - blkg_rwstat_add(&cfqg->stats.merged, rw, 1); + blkg_rwstat_add(&cfqg->stats.merged, op, op_flags, 1); } static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, - uint64_t start_time, uint64_t io_start_time, int rw) + uint64_t start_time, uint64_t io_start_time, int op, + int op_flags) { struct cfqg_stats *stats = &cfqg->stats; unsigned long long now = sched_clock(); if (time_after64(now, io_start_time)) - blkg_rwstat_add(&stats->service_time, rw, now - io_start_time); + blkg_rwstat_add(&stats->service_time, op, op_flags, + now - io_start_time); if (time_after64(io_start_time, start_time)) - blkg_rwstat_add(&stats->wait_time, rw, + blkg_rwstat_add(&stats->wait_time, op, op_flags, io_start_time - start_time); } @@ -781,13 +786,16 @@ static inline void cfqg_put(struct cfq_group *cfqg) { } #define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do {} while (0) static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, - struct cfq_group *curr_cfqg, int rw) { } + struct cfq_group *curr_cfqg, int op, int op_flags) { } static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, - unsigned long time, unsigned long unaccounted_time) { } -static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) { } -static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) { } + uint64_t time, unsigned long unaccounted_time) { } +static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op, + int op_flags) { } +static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op, + int op_flags) { } static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, - uint64_t start_time, uint64_t io_start_time, int rw) { } + uint64_t start_time, uint64_t io_start_time, int op, + int op_flags) { } #endif /* CONFIG_CFQ_GROUP_IOSCHED */ @@ -807,7 +815,7 @@ static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd, struct cfq_ttime *ttime, bool group_idle) { - unsigned long slice; + u64 slice; if (!sample_valid(ttime->ttime_samples)) return false; if (group_idle) @@ -930,17 +938,18 @@ static inline void cfq_schedule_dispatch(struct cfq_data *cfqd) * if a queue is marked sync and has sync io queued. A sync queue with async * io only, should not get full sync slice length. */ -static inline int cfq_prio_slice(struct cfq_data *cfqd, bool sync, +static inline u64 cfq_prio_slice(struct cfq_data *cfqd, bool sync, unsigned short prio) { - const int base_slice = cfqd->cfq_slice[sync]; + u64 base_slice = cfqd->cfq_slice[sync]; + u64 slice = div_u64(base_slice, CFQ_SLICE_SCALE); WARN_ON(prio >= IOPRIO_BE_NR); - return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - prio)); + return base_slice + (slice * (4 - prio)); } -static inline int +static inline u64 cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) { return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio); @@ -958,15 +967,14 @@ cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) * * The result is also in fixed point w/ CFQ_SERVICE_SHIFT. */ -static inline u64 cfqg_scale_charge(unsigned long charge, +static inline u64 cfqg_scale_charge(u64 charge, unsigned int vfraction) { u64 c = charge << CFQ_SERVICE_SHIFT; /* make it fixed point */ /* charge / vfraction */ c <<= CFQ_SERVICE_SHIFT; - do_div(c, vfraction); - return c; + return div_u64(c, vfraction); } static inline u64 max_vdisktime(u64 min_vdisktime, u64 vdisktime) @@ -1019,16 +1027,16 @@ static inline unsigned cfq_group_get_avg_queues(struct cfq_data *cfqd, return cfqg->busy_queues_avg[rt]; } -static inline unsigned +static inline u64 cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg) { return cfqd->cfq_target_latency * cfqg->vfraction >> CFQ_SERVICE_SHIFT; } -static inline unsigned +static inline u64 cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) { - unsigned slice = cfq_prio_to_slice(cfqd, cfqq); + u64 slice = cfq_prio_to_slice(cfqd, cfqq); if (cfqd->cfq_latency) { /* * interested queues (we consider only the ones with the same @@ -1036,20 +1044,22 @@ cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) */ unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg, cfq_class_rt(cfqq)); - unsigned sync_slice = cfqd->cfq_slice[1]; - unsigned expect_latency = sync_slice * iq; - unsigned group_slice = cfq_group_slice(cfqd, cfqq->cfqg); + u64 sync_slice = cfqd->cfq_slice[1]; + u64 expect_latency = sync_slice * iq; + u64 group_slice = cfq_group_slice(cfqd, cfqq->cfqg); if (expect_latency > group_slice) { - unsigned base_low_slice = 2 * cfqd->cfq_slice_idle; + u64 base_low_slice = 2 * cfqd->cfq_slice_idle; + u64 low_slice; + /* scale low_slice according to IO priority * and sync vs async */ - unsigned low_slice = - min(slice, base_low_slice * slice / sync_slice); + low_slice = div64_u64(base_low_slice*slice, sync_slice); + low_slice = min(slice, low_slice); /* the adapted slice value is scaled to fit all iqs * into the target latency */ - slice = max(slice * group_slice / expect_latency, - low_slice); + slice = div64_u64(slice*group_slice, expect_latency); + slice = max(slice, low_slice); } } return slice; @@ -1058,12 +1068,13 @@ cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) static inline void cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) { - unsigned slice = cfq_scaled_cfqq_slice(cfqd, cfqq); + u64 slice = cfq_scaled_cfqq_slice(cfqd, cfqq); + u64 now = ktime_get_ns(); - cfqq->slice_start = jiffies; - cfqq->slice_end = jiffies + slice; + cfqq->slice_start = now; + cfqq->slice_end = now + slice; cfqq->allocated_slice = slice; - cfq_log_cfqq(cfqd, cfqq, "set_slice=%lu", cfqq->slice_end - jiffies); + cfq_log_cfqq(cfqd, cfqq, "set_slice=%llu", cfqq->slice_end - now); } /* @@ -1075,7 +1086,7 @@ static inline bool cfq_slice_used(struct cfq_queue *cfqq) { if (cfq_cfqq_slice_new(cfqq)) return false; - if (time_before(jiffies, cfqq->slice_end)) + if (ktime_get_ns() < cfqq->slice_end) return false; return true; @@ -1241,8 +1252,8 @@ cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq, return cfq_choose_req(cfqd, next, prev, blk_rq_pos(last)); } -static unsigned long cfq_slice_offset(struct cfq_data *cfqd, - struct cfq_queue *cfqq) +static u64 cfq_slice_offset(struct cfq_data *cfqd, + struct cfq_queue *cfqq) { /* * just an approximation, should be ok. @@ -1435,31 +1446,32 @@ cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg) cfqg_stats_update_dequeue(cfqg); } -static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq, - unsigned int *unaccounted_time) +static inline u64 cfq_cfqq_slice_usage(struct cfq_queue *cfqq, + u64 *unaccounted_time) { - unsigned int slice_used; + u64 slice_used; + u64 now = ktime_get_ns(); /* * Queue got expired before even a single request completed or * got expired immediately after first request completion. */ - if (!cfqq->slice_start || cfqq->slice_start == jiffies) { + if (!cfqq->slice_start || cfqq->slice_start == now) { /* * Also charge the seek time incurred to the group, otherwise * if there are mutiple queues in the group, each can dispatch * a single request on seeky media and cause lots of seek time * and group will never know it. */ - slice_used = max_t(unsigned, (jiffies - cfqq->dispatch_start), - 1); + slice_used = max_t(u64, (now - cfqq->dispatch_start), + jiffies_to_nsecs(1)); } else { - slice_used = jiffies - cfqq->slice_start; + slice_used = now - cfqq->slice_start; if (slice_used > cfqq->allocated_slice) { *unaccounted_time = slice_used - cfqq->allocated_slice; slice_used = cfqq->allocated_slice; } - if (time_after(cfqq->slice_start, cfqq->dispatch_start)) + if (cfqq->slice_start > cfqq->dispatch_start) *unaccounted_time += cfqq->slice_start - cfqq->dispatch_start; } @@ -1471,10 +1483,11 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, struct cfq_queue *cfqq) { struct cfq_rb_root *st = &cfqd->grp_service_tree; - unsigned int used_sl, charge, unaccounted_sl = 0; + u64 used_sl, charge, unaccounted_sl = 0; int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg) - cfqg->service_tree_idle.count; unsigned int vfr; + u64 now = ktime_get_ns(); BUG_ON(nr_sync < 0); used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl); @@ -1496,9 +1509,8 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, cfq_group_service_tree_add(st, cfqg); /* This group is being expired. Save the context */ - if (time_after(cfqd->workload_expires, jiffies)) { - cfqg->saved_wl_slice = cfqd->workload_expires - - jiffies; + if (cfqd->workload_expires > now) { + cfqg->saved_wl_slice = cfqd->workload_expires - now; cfqg->saved_wl_type = cfqd->serving_wl_type; cfqg->saved_wl_class = cfqd->serving_wl_class; } else @@ -1507,7 +1519,7 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime, st->min_vdisktime); cfq_log_cfqq(cfqq->cfqd, cfqq, - "sl_used=%u disp=%u charge=%u iops=%u sect=%lu", + "sl_used=%llu disp=%llu charge=%llu iops=%u sect=%lu", used_sl, cfqq->slice_dispatch, charge, iops_mode(cfqd), cfqq->nr_sectors); cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl); @@ -1530,7 +1542,7 @@ static void cfq_init_cfqg_base(struct cfq_group *cfqg) *st = CFQ_RB_ROOT; RB_CLEAR_NODE(&cfqg->rb_node); - cfqg->ttime.last_end_request = jiffies; + cfqg->ttime.last_end_request = ktime_get_ns(); } #ifdef CONFIG_CFQ_GROUP_IOSCHED @@ -2213,10 +2225,11 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, { struct rb_node **p, *parent; struct cfq_queue *__cfqq; - unsigned long rb_key; + u64 rb_key; struct cfq_rb_root *st; int left; int new_cfqq = 1; + u64 now = ktime_get_ns(); st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq)); if (cfq_class_idle(cfqq)) { @@ -2226,7 +2239,7 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, __cfqq = rb_entry(parent, struct cfq_queue, rb_node); rb_key += __cfqq->rb_key; } else - rb_key += jiffies; + rb_key += now; } else if (!add_front) { /* * Get our rb key offset. Subtract any residual slice @@ -2234,13 +2247,13 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, * count indicates slice overrun, and this should position * the next service time further away in the tree. */ - rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies; + rb_key = cfq_slice_offset(cfqd, cfqq) + now; rb_key -= cfqq->slice_resid; cfqq->slice_resid = 0; } else { - rb_key = -HZ; + rb_key = -NSEC_PER_SEC; __cfqq = cfq_rb_first(st); - rb_key += __cfqq ? __cfqq->rb_key : jiffies; + rb_key += __cfqq ? __cfqq->rb_key : now; } if (!RB_EMPTY_NODE(&cfqq->rb_node)) { @@ -2266,7 +2279,7 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, /* * sort by key, that represents service time. */ - if (time_before(rb_key, __cfqq->rb_key)) + if (rb_key < __cfqq->rb_key) p = &parent->rb_left; else { p = &parent->rb_right; @@ -2461,10 +2474,10 @@ static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq) { elv_rb_del(&cfqq->sort_list, rq); cfqq->queued[rq_is_sync(rq)]--; - cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags); + cfqg_stats_update_io_remove(RQ_CFQG(rq), req_op(rq), rq->cmd_flags); cfq_add_rq_rb(rq); cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group, - rq->cmd_flags); + req_op(rq), rq->cmd_flags); } static struct request * @@ -2517,7 +2530,7 @@ static void cfq_remove_request(struct request *rq) cfq_del_rq_rb(rq); cfqq->cfqd->rq_queued--; - cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags); + cfqg_stats_update_io_remove(RQ_CFQG(rq), req_op(rq), rq->cmd_flags); if (rq->cmd_flags & REQ_PRIO) { WARN_ON(!cfqq->prio_pending); cfqq->prio_pending--; @@ -2531,7 +2544,7 @@ static int cfq_merge(struct request_queue *q, struct request **req, struct request *__rq; __rq = cfq_find_rq_fmerge(cfqd, bio); - if (__rq && elv_rq_merge_ok(__rq, bio)) { + if (__rq && elv_bio_merge_ok(__rq, bio)) { *req = __rq; return ELEVATOR_FRONT_MERGE; } @@ -2552,7 +2565,7 @@ static void cfq_merged_request(struct request_queue *q, struct request *req, static void cfq_bio_merged(struct request_queue *q, struct request *req, struct bio *bio) { - cfqg_stats_update_io_merged(RQ_CFQG(req), bio->bi_rw); + cfqg_stats_update_io_merged(RQ_CFQG(req), bio_op(bio), bio->bi_rw); } static void @@ -2566,7 +2579,7 @@ cfq_merged_requests(struct request_queue *q, struct request *rq, * reposition in fifo if next is older than rq */ if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && - time_before(next->fifo_time, rq->fifo_time) && + next->fifo_time < rq->fifo_time && cfqq == RQ_CFQQ(next)) { list_move(&rq->queuelist, &next->queuelist); rq->fifo_time = next->fifo_time; @@ -2575,7 +2588,7 @@ cfq_merged_requests(struct request_queue *q, struct request *rq, if (cfqq->next_rq == next) cfqq->next_rq = rq; cfq_remove_request(next); - cfqg_stats_update_io_merged(RQ_CFQG(rq), next->cmd_flags); + cfqg_stats_update_io_merged(RQ_CFQG(rq), req_op(next), next->cmd_flags); cfqq = RQ_CFQQ(next); /* @@ -2588,8 +2601,8 @@ cfq_merged_requests(struct request_queue *q, struct request *rq, cfq_del_cfqq_rr(cfqd, cfqq); } -static int cfq_allow_merge(struct request_queue *q, struct request *rq, - struct bio *bio) +static int cfq_allow_bio_merge(struct request_queue *q, struct request *rq, + struct bio *bio) { struct cfq_data *cfqd = q->elevator->elevator_data; struct cfq_io_cq *cic; @@ -2613,9 +2626,15 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq, return cfqq == RQ_CFQQ(rq); } +static int cfq_allow_rq_merge(struct request_queue *q, struct request *rq, + struct request *next) +{ + return RQ_CFQQ(rq) == RQ_CFQQ(next); +} + static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq) { - del_timer(&cfqd->idle_slice_timer); + hrtimer_try_to_cancel(&cfqd->idle_slice_timer); cfqg_stats_update_idle_time(cfqq->cfqg); } @@ -2627,7 +2646,7 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd, cfqd->serving_wl_class, cfqd->serving_wl_type); cfqg_stats_update_avg_queue_size(cfqq->cfqg); cfqq->slice_start = 0; - cfqq->dispatch_start = jiffies; + cfqq->dispatch_start = ktime_get_ns(); cfqq->allocated_slice = 0; cfqq->slice_end = 0; cfqq->slice_dispatch = 0; @@ -2676,8 +2695,8 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq, if (cfq_cfqq_slice_new(cfqq)) cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq); else - cfqq->slice_resid = cfqq->slice_end - jiffies; - cfq_log_cfqq(cfqd, cfqq, "resid=%ld", cfqq->slice_resid); + cfqq->slice_resid = cfqq->slice_end - ktime_get_ns(); + cfq_log_cfqq(cfqd, cfqq, "resid=%lld", cfqq->slice_resid); } cfq_group_served(cfqd, cfqq->cfqg, cfqq); @@ -2911,7 +2930,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) struct cfq_queue *cfqq = cfqd->active_queue; struct cfq_rb_root *st = cfqq->service_tree; struct cfq_io_cq *cic; - unsigned long sl, group_idle = 0; + u64 sl, group_idle = 0; + u64 now = ktime_get_ns(); /* * SSD device without seek penalty, disable idling. But only do so @@ -2954,8 +2974,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) * time slice. */ if (sample_valid(cic->ttime.ttime_samples) && - (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) { - cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%lu", + (cfqq->slice_end - now < cic->ttime.ttime_mean)) { + cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%llu", cic->ttime.ttime_mean); return; } @@ -2976,9 +2996,10 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) else sl = cfqd->cfq_slice_idle; - mod_timer(&cfqd->idle_slice_timer, jiffies + sl); + hrtimer_start(&cfqd->idle_slice_timer, ns_to_ktime(sl), + HRTIMER_MODE_REL); cfqg_stats_set_start_idle_time(cfqq->cfqg); - cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl, + cfq_log_cfqq(cfqd, cfqq, "arm_idle: %llu group_idle: %d", sl, group_idle ? 1 : 0); } @@ -3018,7 +3039,7 @@ static struct request *cfq_check_fifo(struct cfq_queue *cfqq) return NULL; rq = rq_entry_fifo(cfqq->fifo.next); - if (time_before(jiffies, rq->fifo_time)) + if (ktime_get_ns() < rq->fifo_time) rq = NULL; cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq); @@ -3096,14 +3117,14 @@ static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd, struct cfq_queue *queue; int i; bool key_valid = false; - unsigned long lowest_key = 0; + u64 lowest_key = 0; enum wl_type_t cur_best = SYNC_NOIDLE_WORKLOAD; for (i = 0; i <= SYNC_WORKLOAD; ++i) { /* select the one with lowest rb_key */ queue = cfq_rb_first(st_for(cfqg, wl_class, i)); if (queue && - (!key_valid || time_before(queue->rb_key, lowest_key))) { + (!key_valid || queue->rb_key < lowest_key)) { lowest_key = queue->rb_key; cur_best = i; key_valid = true; @@ -3116,11 +3137,12 @@ static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd, static void choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg) { - unsigned slice; + u64 slice; unsigned count; struct cfq_rb_root *st; - unsigned group_slice; + u64 group_slice; enum wl_class_t original_class = cfqd->serving_wl_class; + u64 now = ktime_get_ns(); /* Choose next priority. RT > BE > IDLE */ if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg)) @@ -3129,7 +3151,7 @@ choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg) cfqd->serving_wl_class = BE_WORKLOAD; else { cfqd->serving_wl_class = IDLE_WORKLOAD; - cfqd->workload_expires = jiffies + 1; + cfqd->workload_expires = now + jiffies_to_nsecs(1); return; } @@ -3147,7 +3169,7 @@ choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg) /* * check workload expiration, and that we still have other queues ready */ - if (count && !time_after(jiffies, cfqd->workload_expires)) + if (count && !(now > cfqd->workload_expires)) return; new_workload: @@ -3164,13 +3186,13 @@ new_workload: */ group_slice = cfq_group_slice(cfqd, cfqg); - slice = group_slice * count / + slice = div_u64(group_slice * count, max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_wl_class], cfq_group_busy_queues_wl(cfqd->serving_wl_class, cfqd, - cfqg)); + cfqg))); if (cfqd->serving_wl_type == ASYNC_WORKLOAD) { - unsigned int tmp; + u64 tmp; /* * Async queues are currently system wide. Just taking @@ -3181,19 +3203,19 @@ new_workload: */ tmp = cfqd->cfq_target_latency * cfqg_busy_async_queues(cfqd, cfqg); - tmp = tmp/cfqd->busy_queues; - slice = min_t(unsigned, slice, tmp); + tmp = div_u64(tmp, cfqd->busy_queues); + slice = min_t(u64, slice, tmp); /* async workload slice is scaled down according to * the sync/async slice ratio. */ - slice = slice * cfqd->cfq_slice[0] / cfqd->cfq_slice[1]; + slice = div64_u64(slice*cfqd->cfq_slice[0], cfqd->cfq_slice[1]); } else /* sync workload slice is at least 2 * cfq_slice_idle */ slice = max(slice, 2 * cfqd->cfq_slice_idle); - slice = max_t(unsigned, slice, CFQ_MIN_TT); - cfq_log(cfqd, "workload slice:%d", slice); - cfqd->workload_expires = jiffies + slice; + slice = max_t(u64, slice, CFQ_MIN_TT); + cfq_log(cfqd, "workload slice:%llu", slice); + cfqd->workload_expires = now + slice; } static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd) @@ -3211,16 +3233,17 @@ static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd) static void cfq_choose_cfqg(struct cfq_data *cfqd) { struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd); + u64 now = ktime_get_ns(); cfqd->serving_group = cfqg; /* Restore the workload type data */ if (cfqg->saved_wl_slice) { - cfqd->workload_expires = jiffies + cfqg->saved_wl_slice; + cfqd->workload_expires = now + cfqg->saved_wl_slice; cfqd->serving_wl_type = cfqg->saved_wl_type; cfqd->serving_wl_class = cfqg->saved_wl_class; } else - cfqd->workload_expires = jiffies - 1; + cfqd->workload_expires = now - 1; choose_wl_class_and_type(cfqd, cfqg); } @@ -3232,6 +3255,7 @@ static void cfq_choose_cfqg(struct cfq_data *cfqd) static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) { struct cfq_queue *cfqq, *new_cfqq = NULL; + u64 now = ktime_get_ns(); cfqq = cfqd->active_queue; if (!cfqq) @@ -3292,7 +3316,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) * flight or is idling for a new request, allow either of these * conditions to happen (or time out) before selecting a new queue. */ - if (timer_pending(&cfqd->idle_slice_timer)) { + if (hrtimer_active(&cfqd->idle_slice_timer)) { cfqq = NULL; goto keep_queue; } @@ -3303,7 +3327,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) **/ if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) && (cfq_cfqq_slice_new(cfqq) || - (cfqq->slice_end - jiffies > jiffies - cfqq->slice_start))) { + (cfqq->slice_end - now > now - cfqq->slice_start))) { cfq_clear_cfqq_deep(cfqq); cfq_clear_cfqq_idle_window(cfqq); } @@ -3381,11 +3405,12 @@ static int cfq_forced_dispatch(struct cfq_data *cfqd) static inline bool cfq_slice_used_soon(struct cfq_data *cfqd, struct cfq_queue *cfqq) { + u64 now = ktime_get_ns(); + /* the queue hasn't finished any request, can't estimate */ if (cfq_cfqq_slice_new(cfqq)) return true; - if (time_after(jiffies + cfqd->cfq_slice_idle * cfqq->dispatched, - cfqq->slice_end)) + if (now + cfqd->cfq_slice_idle * cfqq->dispatched > cfqq->slice_end) return true; return false; @@ -3460,10 +3485,10 @@ static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq) * based on the last sync IO we serviced */ if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) { - unsigned long last_sync = jiffies - cfqd->last_delayed_sync; + u64 last_sync = ktime_get_ns() - cfqd->last_delayed_sync; unsigned int depth; - depth = last_sync / cfqd->cfq_slice[1]; + depth = div64_u64(last_sync, cfqd->cfq_slice[1]); if (!depth && !cfqq->dispatched) depth = 1; if (depth < max_dispatch) @@ -3546,7 +3571,7 @@ static int cfq_dispatch_requests(struct request_queue *q, int force) if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) && cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) || cfq_class_idle(cfqq))) { - cfqq->slice_end = jiffies + 1; + cfqq->slice_end = ktime_get_ns() + 1; cfq_slice_expired(cfqd, 0); } @@ -3624,7 +3649,7 @@ static void cfq_init_icq(struct io_cq *icq) { struct cfq_io_cq *cic = icq_to_cic(icq); - cic->ttime.last_end_request = jiffies; + cic->ttime.last_end_request = ktime_get_ns(); } static void cfq_exit_icq(struct io_cq *icq) @@ -3682,6 +3707,7 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic) * elevate the priority of this queue */ cfqq->org_ioprio = cfqq->ioprio; + cfqq->org_ioprio_class = cfqq->ioprio_class; cfq_clear_cfqq_prio_changed(cfqq); } @@ -3845,14 +3871,15 @@ out: } static void -__cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle) +__cfq_update_io_thinktime(struct cfq_ttime *ttime, u64 slice_idle) { - unsigned long elapsed = jiffies - ttime->last_end_request; + u64 elapsed = ktime_get_ns() - ttime->last_end_request; elapsed = min(elapsed, 2UL * slice_idle); ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8; - ttime->ttime_total = (7*ttime->ttime_total + 256*elapsed) / 8; - ttime->ttime_mean = (ttime->ttime_total + 128) / ttime->ttime_samples; + ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed, 8); + ttime->ttime_mean = div64_ul(ttime->ttime_total + 128, + ttime->ttime_samples); } static void @@ -4105,10 +4132,10 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq) cfq_log_cfqq(cfqd, cfqq, "insert_request"); cfq_init_prio_data(cfqq, RQ_CIC(rq)); - rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]; + rq->fifo_time = ktime_get_ns() + cfqd->cfq_fifo_expire[rq_is_sync(rq)]; list_add_tail(&rq->queuelist, &cfqq->fifo); cfq_add_rq_rb(rq); - cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, + cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq), rq->cmd_flags); cfq_rq_enqueued(cfqd, cfqq, rq); } @@ -4153,6 +4180,7 @@ static void cfq_update_hw_tag(struct cfq_data *cfqd) static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) { struct cfq_io_cq *cic = cfqd->active_cic; + u64 now = ktime_get_ns(); /* If the queue already has requests, don't wait */ if (!RB_EMPTY_ROOT(&cfqq->sort_list)) @@ -4171,7 +4199,7 @@ static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) /* if slice left is less than think time, wait busy */ if (cic && sample_valid(cic->ttime.ttime_samples) - && (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) + && (cfqq->slice_end - now < cic->ttime.ttime_mean)) return true; /* @@ -4181,7 +4209,7 @@ static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) * case where think time is less than a jiffy, mark the queue wait * busy if only 1 jiffy is left in the slice. */ - if (cfqq->slice_end - jiffies == 1) + if (cfqq->slice_end - now <= jiffies_to_nsecs(1)) return true; return false; @@ -4192,9 +4220,8 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) struct cfq_queue *cfqq = RQ_CFQQ(rq); struct cfq_data *cfqd = cfqq->cfqd; const int sync = rq_is_sync(rq); - unsigned long now; + u64 now = ktime_get_ns(); - now = jiffies; cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d", !!(rq->cmd_flags & REQ_NOIDLE)); @@ -4206,7 +4233,8 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfqq->dispatched--; (RQ_CFQG(rq))->dispatched--; cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq), - rq_io_start_time_ns(rq), rq->cmd_flags); + rq_io_start_time_ns(rq), req_op(rq), + rq->cmd_flags); cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--; @@ -4222,7 +4250,16 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfqq_type(cfqq)); st->ttime.last_end_request = now; - if (!time_after(rq->start_time + cfqd->cfq_fifo_expire[1], now)) + /* + * We have to do this check in jiffies since start_time is in + * jiffies and it is not trivial to convert to ns. If + * cfq_fifo_expire[1] ever comes close to 1 jiffie, this test + * will become problematic but so far we are fine (the default + * is 128 ms). + */ + if (!time_after(rq->start_time + + nsecs_to_jiffies(cfqd->cfq_fifo_expire[1]), + jiffies)) cfqd->last_delayed_sync = now; } @@ -4247,10 +4284,10 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) * the queue. */ if (cfq_should_wait_busy(cfqd, cfqq)) { - unsigned long extend_sl = cfqd->cfq_slice_idle; + u64 extend_sl = cfqd->cfq_slice_idle; if (!cfqd->cfq_slice_idle) extend_sl = cfqd->cfq_group_idle; - cfqq->slice_end = jiffies + extend_sl; + cfqq->slice_end = now + extend_sl; cfq_mark_cfqq_wait_busy(cfqq); cfq_log_cfqq(cfqd, cfqq, "will busy wait"); } @@ -4275,6 +4312,24 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfq_schedule_dispatch(cfqd); } +static void cfqq_boost_on_prio(struct cfq_queue *cfqq, int op_flags) +{ + /* + * If REQ_PRIO is set, boost class and prio level, if it's below + * BE/NORM. If prio is not set, restore the potentially boosted + * class/prio level. + */ + if (!(op_flags & REQ_PRIO)) { + cfqq->ioprio_class = cfqq->org_ioprio_class; + cfqq->ioprio = cfqq->org_ioprio; + } else { + if (cfq_class_idle(cfqq)) + cfqq->ioprio_class = IOPRIO_CLASS_BE; + if (cfqq->ioprio > IOPRIO_NORM) + cfqq->ioprio = IOPRIO_NORM; + } +} + static inline int __cfq_may_queue(struct cfq_queue *cfqq) { if (cfq_cfqq_wait_request(cfqq) && !cfq_cfqq_must_alloc_slice(cfqq)) { @@ -4285,7 +4340,7 @@ static inline int __cfq_may_queue(struct cfq_queue *cfqq) return ELV_MQUEUE_MAY; } -static int cfq_may_queue(struct request_queue *q, int rw) +static int cfq_may_queue(struct request_queue *q, int op, int op_flags) { struct cfq_data *cfqd = q->elevator->elevator_data; struct task_struct *tsk = current; @@ -4302,9 +4357,10 @@ static int cfq_may_queue(struct request_queue *q, int rw) if (!cic) return ELV_MQUEUE_MAY; - cfqq = cic_to_cfqq(cic, rw_is_sync(rw)); + cfqq = cic_to_cfqq(cic, rw_is_sync(op, op_flags)); if (cfqq) { cfq_init_prio_data(cfqq, cic); + cfqq_boost_on_prio(cfqq, op_flags); return __cfq_may_queue(cfqq); } @@ -4435,9 +4491,10 @@ static void cfq_kick_queue(struct work_struct *work) /* * Timer running if the active_queue is currently idling inside its time slice */ -static void cfq_idle_slice_timer(unsigned long data) +static enum hrtimer_restart cfq_idle_slice_timer(struct hrtimer *timer) { - struct cfq_data *cfqd = (struct cfq_data *) data; + struct cfq_data *cfqd = container_of(timer, struct cfq_data, + idle_slice_timer); struct cfq_queue *cfqq; unsigned long flags; int timed_out = 1; @@ -4486,11 +4543,12 @@ out_kick: cfq_schedule_dispatch(cfqd); out_cont: spin_unlock_irqrestore(cfqd->queue->queue_lock, flags); + return HRTIMER_NORESTART; } static void cfq_shutdown_timer_wq(struct cfq_data *cfqd) { - del_timer_sync(&cfqd->idle_slice_timer); + hrtimer_cancel(&cfqd->idle_slice_timer); cancel_work_sync(&cfqd->unplug_work); } @@ -4586,9 +4644,9 @@ static int cfq_init_queue(struct request_queue *q, struct elevator_type *e) cfqg_put(cfqd->root_group); spin_unlock_irq(q->queue_lock); - init_timer(&cfqd->idle_slice_timer); + hrtimer_init(&cfqd->idle_slice_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); cfqd->idle_slice_timer.function = cfq_idle_slice_timer; - cfqd->idle_slice_timer.data = (unsigned long) cfqd; INIT_WORK(&cfqd->unplug_work, cfq_kick_queue); @@ -4609,7 +4667,7 @@ static int cfq_init_queue(struct request_queue *q, struct elevator_type *e) * we optimistically start assuming sync ops weren't delayed in last * second, in order to have larger depth for async operations. */ - cfqd->last_delayed_sync = jiffies - HZ; + cfqd->last_delayed_sync = ktime_get_ns() - NSEC_PER_SEC; return 0; out_free: @@ -4652,9 +4710,9 @@ cfq_var_store(unsigned int *var, const char *page, size_t count) static ssize_t __FUNC(struct elevator_queue *e, char *page) \ { \ struct cfq_data *cfqd = e->elevator_data; \ - unsigned int __data = __VAR; \ + u64 __data = __VAR; \ if (__CONV) \ - __data = jiffies_to_msecs(__data); \ + __data = div_u64(__data, NSEC_PER_MSEC); \ return cfq_var_show(__data, (page)); \ } SHOW_FUNCTION(cfq_quantum_show, cfqd->cfq_quantum, 0); @@ -4671,6 +4729,21 @@ SHOW_FUNCTION(cfq_low_latency_show, cfqd->cfq_latency, 0); SHOW_FUNCTION(cfq_target_latency_show, cfqd->cfq_target_latency, 1); #undef SHOW_FUNCTION +#define USEC_SHOW_FUNCTION(__FUNC, __VAR) \ +static ssize_t __FUNC(struct elevator_queue *e, char *page) \ +{ \ + struct cfq_data *cfqd = e->elevator_data; \ + u64 __data = __VAR; \ + __data = div_u64(__data, NSEC_PER_USEC); \ + return cfq_var_show(__data, (page)); \ +} +USEC_SHOW_FUNCTION(cfq_slice_idle_us_show, cfqd->cfq_slice_idle); +USEC_SHOW_FUNCTION(cfq_group_idle_us_show, cfqd->cfq_group_idle); +USEC_SHOW_FUNCTION(cfq_slice_sync_us_show, cfqd->cfq_slice[1]); +USEC_SHOW_FUNCTION(cfq_slice_async_us_show, cfqd->cfq_slice[0]); +USEC_SHOW_FUNCTION(cfq_target_latency_us_show, cfqd->cfq_target_latency); +#undef USEC_SHOW_FUNCTION + #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \ static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) \ { \ @@ -4682,7 +4755,7 @@ static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) else if (__data > (MAX)) \ __data = (MAX); \ if (__CONV) \ - *(__PTR) = msecs_to_jiffies(__data); \ + *(__PTR) = (u64)__data * NSEC_PER_MSEC; \ else \ *(__PTR) = __data; \ return ret; \ @@ -4705,6 +4778,26 @@ STORE_FUNCTION(cfq_low_latency_store, &cfqd->cfq_latency, 0, 1, 0); STORE_FUNCTION(cfq_target_latency_store, &cfqd->cfq_target_latency, 1, UINT_MAX, 1); #undef STORE_FUNCTION +#define USEC_STORE_FUNCTION(__FUNC, __PTR, MIN, MAX) \ +static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) \ +{ \ + struct cfq_data *cfqd = e->elevator_data; \ + unsigned int __data; \ + int ret = cfq_var_store(&__data, (page), count); \ + if (__data < (MIN)) \ + __data = (MIN); \ + else if (__data > (MAX)) \ + __data = (MAX); \ + *(__PTR) = (u64)__data * NSEC_PER_USEC; \ + return ret; \ +} +USEC_STORE_FUNCTION(cfq_slice_idle_us_store, &cfqd->cfq_slice_idle, 0, UINT_MAX); +USEC_STORE_FUNCTION(cfq_group_idle_us_store, &cfqd->cfq_group_idle, 0, UINT_MAX); +USEC_STORE_FUNCTION(cfq_slice_sync_us_store, &cfqd->cfq_slice[1], 1, UINT_MAX); +USEC_STORE_FUNCTION(cfq_slice_async_us_store, &cfqd->cfq_slice[0], 1, UINT_MAX); +USEC_STORE_FUNCTION(cfq_target_latency_us_store, &cfqd->cfq_target_latency, 1, UINT_MAX); +#undef USEC_STORE_FUNCTION + #define CFQ_ATTR(name) \ __ATTR(name, S_IRUGO|S_IWUSR, cfq_##name##_show, cfq_##name##_store) @@ -4715,12 +4808,17 @@ static struct elv_fs_entry cfq_attrs[] = { CFQ_ATTR(back_seek_max), CFQ_ATTR(back_seek_penalty), CFQ_ATTR(slice_sync), + CFQ_ATTR(slice_sync_us), CFQ_ATTR(slice_async), + CFQ_ATTR(slice_async_us), CFQ_ATTR(slice_async_rq), CFQ_ATTR(slice_idle), + CFQ_ATTR(slice_idle_us), CFQ_ATTR(group_idle), + CFQ_ATTR(group_idle_us), CFQ_ATTR(low_latency), CFQ_ATTR(target_latency), + CFQ_ATTR(target_latency_us), __ATTR_NULL }; @@ -4729,7 +4827,8 @@ static struct elevator_type iosched_cfq = { .elevator_merge_fn = cfq_merge, .elevator_merged_fn = cfq_merged_request, .elevator_merge_req_fn = cfq_merged_requests, - .elevator_allow_merge_fn = cfq_allow_merge, + .elevator_allow_bio_merge_fn = cfq_allow_bio_merge, + .elevator_allow_rq_merge_fn = cfq_allow_rq_merge, .elevator_bio_merged_fn = cfq_bio_merged, .elevator_dispatch_fn = cfq_dispatch_requests, .elevator_add_req_fn = cfq_insert_request, @@ -4776,18 +4875,7 @@ static int __init cfq_init(void) { int ret; - /* - * could be 0 on HZ < 1000 setups - */ - if (!cfq_slice_async) - cfq_slice_async = 1; - if (!cfq_slice_idle) - cfq_slice_idle = 1; - #ifdef CONFIG_CFQ_GROUP_IOSCHED - if (!cfq_group_idle) - cfq_group_idle = 1; - ret = blkcg_policy_register(&blkcg_policy_cfq); if (ret) return ret; diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c index d0dd7882d8c7..55e0bb6d7da7 100644 --- a/block/deadline-iosched.c +++ b/block/deadline-iosched.c @@ -137,7 +137,7 @@ deadline_merge(struct request_queue *q, struct request **req, struct bio *bio) if (__rq) { BUG_ON(sector != blk_rq_pos(__rq)); - if (elv_rq_merge_ok(__rq, bio)) { + if (elv_bio_merge_ok(__rq, bio)) { ret = ELEVATOR_FRONT_MERGE; goto out; } @@ -173,7 +173,8 @@ deadline_merged_requests(struct request_queue *q, struct request *req, * and move into next position (next will be deleted) in fifo */ if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) { - if (time_before(next->fifo_time, req->fifo_time)) { + if (time_before((unsigned long)next->fifo_time, + (unsigned long)req->fifo_time)) { list_move(&req->queuelist, &next->queuelist); req->fifo_time = next->fifo_time; } @@ -227,7 +228,7 @@ static inline int deadline_check_fifo(struct deadline_data *dd, int ddir) /* * rq is expired! */ - if (time_after_eq(jiffies, rq->fifo_time)) + if (time_after_eq(jiffies, (unsigned long)rq->fifo_time)) return 1; return 0; diff --git a/block/elevator.c b/block/elevator.c index c3555c9c672f..7096c22041e7 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -53,13 +53,13 @@ static LIST_HEAD(elv_list); * Query io scheduler to see if the current process issuing bio may be * merged with rq. */ -static int elv_iosched_allow_merge(struct request *rq, struct bio *bio) +static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio) { struct request_queue *q = rq->q; struct elevator_queue *e = q->elevator; - if (e->type->ops.elevator_allow_merge_fn) - return e->type->ops.elevator_allow_merge_fn(q, rq, bio); + if (e->type->ops.elevator_allow_bio_merge_fn) + return e->type->ops.elevator_allow_bio_merge_fn(q, rq, bio); return 1; } @@ -67,17 +67,17 @@ static int elv_iosched_allow_merge(struct request *rq, struct bio *bio) /* * can we safely merge with this request? */ -bool elv_rq_merge_ok(struct request *rq, struct bio *bio) +bool elv_bio_merge_ok(struct request *rq, struct bio *bio) { if (!blk_rq_merge_ok(rq, bio)) - return 0; + return false; - if (!elv_iosched_allow_merge(rq, bio)) - return 0; + if (!elv_iosched_allow_bio_merge(rq, bio)) + return false; - return 1; + return true; } -EXPORT_SYMBOL(elv_rq_merge_ok); +EXPORT_SYMBOL(elv_bio_merge_ok); static struct elevator_type *elevator_find(const char *name) { @@ -366,8 +366,7 @@ void elv_dispatch_sort(struct request_queue *q, struct request *rq) list_for_each_prev(entry, &q->queue_head) { struct request *pos = list_entry_rq(entry); - if ((rq->cmd_flags & REQ_DISCARD) != - (pos->cmd_flags & REQ_DISCARD)) + if ((req_op(rq) == REQ_OP_DISCARD) != (req_op(pos) == REQ_OP_DISCARD)) break; if (rq_data_dir(rq) != rq_data_dir(pos)) break; @@ -426,7 +425,7 @@ int elv_merge(struct request_queue *q, struct request **req, struct bio *bio) /* * First try one-hit cache. */ - if (q->last_merge && elv_rq_merge_ok(q->last_merge, bio)) { + if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) { ret = blk_try_merge(q->last_merge, bio); if (ret != ELEVATOR_NO_MERGE) { *req = q->last_merge; @@ -441,7 +440,7 @@ int elv_merge(struct request_queue *q, struct request **req, struct bio *bio) * See if our hash lookup can find a potential backmerge. */ __rq = elv_rqhash_find(q, bio->bi_iter.bi_sector); - if (__rq && elv_rq_merge_ok(__rq, bio)) { + if (__rq && elv_bio_merge_ok(__rq, bio)) { *req = __rq; return ELEVATOR_BACK_MERGE; } @@ -717,12 +716,12 @@ void elv_put_request(struct request_queue *q, struct request *rq) e->type->ops.elevator_put_req_fn(rq); } -int elv_may_queue(struct request_queue *q, int rw) +int elv_may_queue(struct request_queue *q, int op, int op_flags) { struct elevator_queue *e = q->elevator; if (e->type->ops.elevator_may_queue_fn) - return e->type->ops.elevator_may_queue_fn(q, rw); + return e->type->ops.elevator_may_queue_fn(q, op, op_flags); return ELV_MQUEUE_MAY; } diff --git a/block/genhd.c b/block/genhd.c index 9f42526b4d62..3c9dede4e04f 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -506,7 +506,7 @@ static int exact_lock(dev_t devt, void *data) return 0; } -static void register_disk(struct gendisk *disk) +static void register_disk(struct device *parent, struct gendisk *disk) { struct device *ddev = disk_to_dev(disk); struct block_device *bdev; @@ -514,7 +514,7 @@ static void register_disk(struct gendisk *disk) struct hd_struct *part; int err; - ddev->parent = disk->driverfs_dev; + ddev->parent = parent; dev_set_name(ddev, "%s", disk->disk_name); @@ -573,7 +573,8 @@ exit: } /** - * add_disk - add partitioning information to kernel list + * device_add_disk - add partitioning information to kernel list + * @parent: parent device for the disk * @disk: per-device partitioning information * * This function registers the partitioning information in @disk @@ -581,7 +582,7 @@ exit: * * FIXME: error handling */ -void add_disk(struct gendisk *disk) +void device_add_disk(struct device *parent, struct gendisk *disk) { struct backing_dev_info *bdi; dev_t devt; @@ -617,7 +618,7 @@ void add_disk(struct gendisk *disk) blk_register_region(disk_devt(disk), disk->minors, NULL, exact_match, exact_lock, disk); - register_disk(disk); + register_disk(parent, disk); blk_register_queue(disk); /* @@ -633,7 +634,7 @@ void add_disk(struct gendisk *disk) disk_add_events(disk); blk_integrity_add(disk); } -EXPORT_SYMBOL(add_disk); +EXPORT_SYMBOL(device_add_disk); void del_gendisk(struct gendisk *disk) { @@ -799,10 +800,9 @@ void __init printk_all_partitions(void) , disk_name(disk, part->partno, name_buf), part->info ? part->info->uuid : ""); if (is_part0) { - if (disk->driverfs_dev != NULL && - disk->driverfs_dev->driver != NULL) + if (dev->parent && dev->parent->driver) printk(" driver: %s\n", - disk->driverfs_dev->driver->name); + dev->parent->driver->name); else printk(" (driver?)\n"); } else @@ -1523,12 +1523,7 @@ static void __disk_unblock_events(struct gendisk *disk, bool check_now) if (--ev->block) goto out_unlock; - /* - * Not exactly a latency critical operation, set poll timer - * slack to 25% and kick event check. - */ intv = disk_events_poll_jiffies(disk); - set_timer_slack(&ev->dwork.timer, intv / 4); if (check_now) queue_delayed_work(system_freezable_power_efficient_wq, &ev->dwork, 0); diff --git a/block/ioprio.c b/block/ioprio.c index cc7800e9eb44..01b8116298a1 100644 --- a/block/ioprio.c +++ b/block/ioprio.c @@ -150,8 +150,10 @@ static int get_task_ioprio(struct task_struct *p) if (ret) goto out; ret = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, IOPRIO_NORM); + task_lock(p); if (p->io_context) ret = p->io_context->ioprio; + task_unlock(p); out: return ret; } diff --git a/block/partition-generic.c b/block/partition-generic.c index d7eb77e1e3a8..71d9ed9df8da 100644 --- a/block/partition-generic.c +++ b/block/partition-generic.c @@ -495,7 +495,6 @@ rescan: /* add partitions */ for (p = 1; p < state->limit; p++) { sector_t size, from; - struct partition_meta_info *info = NULL; size = state->parts[p].size; if (!size) @@ -530,8 +529,6 @@ rescan: } } - if (state->parts[p].has_info) - info = &state->parts[p].info; part = add_partition(disk, p, from, size, state->parts[p].flags, &state->parts[p].info); diff --git a/block/partitions/atari.c b/block/partitions/atari.c index 9875b05e80a2..ff1fb93712c1 100644 --- a/block/partitions/atari.c +++ b/block/partitions/atari.c @@ -42,6 +42,13 @@ int atari_partition(struct parsed_partitions *state) int part_fmt = 0; /* 0:unknown, 1:AHDI, 2:ICD/Supra */ #endif + /* + * ATARI partition scheme supports 512 lba only. If this is not + * the case, bail early to avoid miscalculating hd_size. + */ + if (bdev_logical_block_size(state->bdev) != 512) + return 0; + rs = read_part_sector(state, 0, §); if (!rs) return -1; diff --git a/crypto/Kconfig b/crypto/Kconfig index 1d33beb6a1ae..a9377bef25e3 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -93,6 +93,15 @@ config CRYPTO_AKCIPHER select CRYPTO_AKCIPHER2 select CRYPTO_ALGAPI +config CRYPTO_KPP2 + tristate + select CRYPTO_ALGAPI2 + +config CRYPTO_KPP + tristate + select CRYPTO_ALGAPI + select CRYPTO_KPP2 + config CRYPTO_RSA tristate "RSA algorithm" select CRYPTO_AKCIPHER @@ -102,6 +111,19 @@ config CRYPTO_RSA help Generic implementation of the RSA public key algorithm. +config CRYPTO_DH + tristate "Diffie-Hellman algorithm" + select CRYPTO_KPP + select MPILIB + help + Generic implementation of the Diffie-Hellman algorithm. + +config CRYPTO_ECDH + tristate "ECDH algorithm" + select CRYTPO_KPP + help + Generic implementation of the ECDH algorithm + config CRYPTO_MANAGER tristate "Cryptographic algorithm manager" select CRYPTO_MANAGER2 @@ -115,6 +137,7 @@ config CRYPTO_MANAGER2 select CRYPTO_HASH2 select CRYPTO_BLKCIPHER2 select CRYPTO_AKCIPHER2 + select CRYPTO_KPP2 config CRYPTO_USER tristate "Userspace cryptographic algorithm configuration" @@ -414,6 +437,17 @@ config CRYPTO_CRC32C_INTEL gain performance compared with software implementation. Module will be crc32c-intel. +config CRYPT_CRC32C_VPMSUM + tristate "CRC32c CRC algorithm (powerpc64)" + depends on PPC64 + select CRYPTO_HASH + select CRC32 + help + CRC32c algorithm implemented using vector polynomial multiply-sum + (vpmsum) instructions, introduced in POWER8. Enable on POWER8 + and newer processors for improved performance. + + config CRYPTO_CRC32C_SPARC64 tristate "CRC32c CRC algorithm (SPARC64)" depends on SPARC64 @@ -681,6 +715,38 @@ config CRYPTO_SHA1_MB lanes remain unfilled, a flush operation will be initiated to process the crypto jobs, adding a slight latency. +config CRYPTO_SHA256_MB + tristate "SHA256 digest algorithm (x86_64 Multi-Buffer, Experimental)" + depends on X86 && 64BIT + select CRYPTO_SHA256 + select CRYPTO_HASH + select CRYPTO_MCRYPTD + help + SHA-256 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using multi-buffer technique. This algorithm computes on + multiple data lanes concurrently with SIMD instructions for + better throughput. It should not be enabled by default but + used when there is significant amount of work to keep the keep + the data lanes filled to get performance benefit. If the data + lanes remain unfilled, a flush operation will be initiated to + process the crypto jobs, adding a slight latency. + +config CRYPTO_SHA512_MB + tristate "SHA512 digest algorithm (x86_64 Multi-Buffer, Experimental)" + depends on X86 && 64BIT + select CRYPTO_SHA512 + select CRYPTO_HASH + select CRYPTO_MCRYPTD + help + SHA-512 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using multi-buffer technique. This algorithm computes on + multiple data lanes concurrently with SIMD instructions for + better throughput. It should not be enabled by default but + used when there is significant amount of work to keep the keep + the data lanes filled to get performance benefit. If the data + lanes remain unfilled, a flush operation will be initiated to + process the crypto jobs, adding a slight latency. + config CRYPTO_SHA256 tristate "SHA224 and SHA256 digest algorithm" select CRYPTO_HASH @@ -750,6 +816,16 @@ config CRYPTO_SHA512_SPARC64 SHA-512 secure hash standard (DFIPS 180-2) implemented using sparc64 crypto instructions, when available. +config CRYPTO_SHA3 + tristate "SHA3 digest algorithm" + select CRYPTO_HASH + help + SHA-3 secure hash standard (DFIPS 202). It's based on + cryptographic sponge function family called Keccak. + + References: + http://keccak.noekeon.org/ + config CRYPTO_TGR192 tristate "Tiger digest algorithms" select CRYPTO_HASH @@ -1567,6 +1643,7 @@ config CRYPTO_DRBG_HASH config CRYPTO_DRBG_CTR bool "Enable CTR DRBG" select CRYPTO_AES + depends on CRYPTO_CTR help Enable the CTR DRBG variant as defined in NIST SP800-90A. diff --git a/crypto/Makefile b/crypto/Makefile index 4f4ef7eaae3f..99cc64ac70ef 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -20,8 +20,6 @@ crypto_blkcipher-y := ablkcipher.o crypto_blkcipher-y += blkcipher.o crypto_blkcipher-y += skcipher.o obj-$(CONFIG_CRYPTO_BLKCIPHER2) += crypto_blkcipher.o -obj-$(CONFIG_CRYPTO_BLKCIPHER2) += chainiv.o -obj-$(CONFIG_CRYPTO_BLKCIPHER2) += eseqiv.o obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o obj-$(CONFIG_CRYPTO_ECHAINIV) += echainiv.o @@ -30,6 +28,15 @@ crypto_hash-y += shash.o obj-$(CONFIG_CRYPTO_HASH2) += crypto_hash.o obj-$(CONFIG_CRYPTO_AKCIPHER2) += akcipher.o +obj-$(CONFIG_CRYPTO_KPP2) += kpp.o + +dh_generic-y := dh.o +dh_generic-y += dh_helper.o +obj-$(CONFIG_CRYPTO_DH) += dh_generic.o +ecdh_generic-y := ecc.o +ecdh_generic-y += ecdh.o +ecdh_generic-y += ecdh_helper.o +obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o $(obj)/rsapubkey-asn1.o: $(obj)/rsapubkey-asn1.c $(obj)/rsapubkey-asn1.h $(obj)/rsaprivkey-asn1.o: $(obj)/rsaprivkey-asn1.c $(obj)/rsaprivkey-asn1.h @@ -61,6 +68,7 @@ obj-$(CONFIG_CRYPTO_RMD320) += rmd320.o obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o +obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o obj-$(CONFIG_CRYPTO_TGR192) += tgr192.o obj-$(CONFIG_CRYPTO_GF128MUL) += gf128mul.o diff --git a/crypto/ablk_helper.c b/crypto/ablk_helper.c index e1fcf53bb931..1441f07d0a19 100644 --- a/crypto/ablk_helper.c +++ b/crypto/ablk_helper.c @@ -71,7 +71,8 @@ int ablk_encrypt(struct ablkcipher_request *req) struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); - if (!may_use_simd()) { + if (!may_use_simd() || + (in_atomic() && cryptd_ablkcipher_queued(ctx->cryptd_tfm))) { struct ablkcipher_request *cryptd_req = ablkcipher_request_ctx(req); @@ -90,7 +91,8 @@ int ablk_decrypt(struct ablkcipher_request *req) struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); - if (!may_use_simd()) { + if (!may_use_simd() || + (in_atomic() && cryptd_ablkcipher_queued(ctx->cryptd_tfm))) { struct ablkcipher_request *cryptd_req = ablkcipher_request_ctx(req); diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c index e5b5721809e2..d676fc59521a 100644 --- a/crypto/ablkcipher.c +++ b/crypto/ablkcipher.c @@ -14,11 +14,8 @@ */ #include -#include #include #include -#include -#include #include #include #include @@ -349,16 +346,6 @@ static unsigned int crypto_ablkcipher_ctxsize(struct crypto_alg *alg, u32 type, return alg->cra_ctxsize; } -int skcipher_null_givencrypt(struct skcipher_givcrypt_request *req) -{ - return crypto_ablkcipher_encrypt(&req->creq); -} - -int skcipher_null_givdecrypt(struct skcipher_givcrypt_request *req) -{ - return crypto_ablkcipher_decrypt(&req->creq); -} - static int crypto_init_ablkcipher_ops(struct crypto_tfm *tfm, u32 type, u32 mask) { @@ -371,10 +358,6 @@ static int crypto_init_ablkcipher_ops(struct crypto_tfm *tfm, u32 type, crt->setkey = setkey; crt->encrypt = alg->encrypt; crt->decrypt = alg->decrypt; - if (!alg->ivsize) { - crt->givencrypt = skcipher_null_givencrypt; - crt->givdecrypt = skcipher_null_givdecrypt; - } crt->base = __crypto_ablkcipher_cast(tfm); crt->ivsize = alg->ivsize; @@ -436,11 +419,6 @@ const struct crypto_type crypto_ablkcipher_type = { }; EXPORT_SYMBOL_GPL(crypto_ablkcipher_type); -static int no_givdecrypt(struct skcipher_givcrypt_request *req) -{ - return -ENOSYS; -} - static int crypto_init_givcipher_ops(struct crypto_tfm *tfm, u32 type, u32 mask) { @@ -454,8 +432,6 @@ static int crypto_init_givcipher_ops(struct crypto_tfm *tfm, u32 type, alg->setkey : setkey; crt->encrypt = alg->encrypt; crt->decrypt = alg->decrypt; - crt->givencrypt = alg->givencrypt ?: no_givdecrypt; - crt->givdecrypt = alg->givdecrypt ?: no_givdecrypt; crt->base = __crypto_ablkcipher_cast(tfm); crt->ivsize = alg->ivsize; @@ -516,202 +492,3 @@ const struct crypto_type crypto_givcipher_type = { .report = crypto_givcipher_report, }; EXPORT_SYMBOL_GPL(crypto_givcipher_type); - -const char *crypto_default_geniv(const struct crypto_alg *alg) -{ - if (((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : - alg->cra_ablkcipher.ivsize) != - alg->cra_blocksize) - return "chainiv"; - - return "eseqiv"; -} - -static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask) -{ - struct rtattr *tb[3]; - struct { - struct rtattr attr; - struct crypto_attr_type data; - } ptype; - struct { - struct rtattr attr; - struct crypto_attr_alg data; - } palg; - struct crypto_template *tmpl; - struct crypto_instance *inst; - struct crypto_alg *larval; - const char *geniv; - int err; - - larval = crypto_larval_lookup(alg->cra_driver_name, - (type & ~CRYPTO_ALG_TYPE_MASK) | - CRYPTO_ALG_TYPE_GIVCIPHER, - mask | CRYPTO_ALG_TYPE_MASK); - err = PTR_ERR(larval); - if (IS_ERR(larval)) - goto out; - - err = -EAGAIN; - if (!crypto_is_larval(larval)) - goto drop_larval; - - ptype.attr.rta_len = sizeof(ptype); - ptype.attr.rta_type = CRYPTOA_TYPE; - ptype.data.type = type | CRYPTO_ALG_GENIV; - /* GENIV tells the template that we're making a default geniv. */ - ptype.data.mask = mask | CRYPTO_ALG_GENIV; - tb[0] = &ptype.attr; - - palg.attr.rta_len = sizeof(palg); - palg.attr.rta_type = CRYPTOA_ALG; - /* Must use the exact name to locate ourselves. */ - memcpy(palg.data.name, alg->cra_driver_name, CRYPTO_MAX_ALG_NAME); - tb[1] = &palg.attr; - - tb[2] = NULL; - - if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER) - geniv = alg->cra_blkcipher.geniv; - else - geniv = alg->cra_ablkcipher.geniv; - - if (!geniv) - geniv = crypto_default_geniv(alg); - - tmpl = crypto_lookup_template(geniv); - err = -ENOENT; - if (!tmpl) - goto kill_larval; - - if (tmpl->create) { - err = tmpl->create(tmpl, tb); - if (err) - goto put_tmpl; - goto ok; - } - - inst = tmpl->alloc(tb); - err = PTR_ERR(inst); - if (IS_ERR(inst)) - goto put_tmpl; - - err = crypto_register_instance(tmpl, inst); - if (err) { - tmpl->free(inst); - goto put_tmpl; - } - -ok: - /* Redo the lookup to use the instance we just registered. */ - err = -EAGAIN; - -put_tmpl: - crypto_tmpl_put(tmpl); -kill_larval: - crypto_larval_kill(larval); -drop_larval: - crypto_mod_put(larval); -out: - crypto_mod_put(alg); - return err; -} - -struct crypto_alg *crypto_lookup_skcipher(const char *name, u32 type, u32 mask) -{ - struct crypto_alg *alg; - - alg = crypto_alg_mod_lookup(name, type, mask); - if (IS_ERR(alg)) - return alg; - - if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_GIVCIPHER) - return alg; - - if (!((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : - alg->cra_ablkcipher.ivsize)) - return alg; - - crypto_mod_put(alg); - alg = crypto_alg_mod_lookup(name, type | CRYPTO_ALG_TESTED, - mask & ~CRYPTO_ALG_TESTED); - if (IS_ERR(alg)) - return alg; - - if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_GIVCIPHER) { - if (~alg->cra_flags & (type ^ ~mask) & CRYPTO_ALG_TESTED) { - crypto_mod_put(alg); - alg = ERR_PTR(-ENOENT); - } - return alg; - } - - BUG_ON(!((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : - alg->cra_ablkcipher.ivsize)); - - return ERR_PTR(crypto_givcipher_default(alg, type, mask)); -} -EXPORT_SYMBOL_GPL(crypto_lookup_skcipher); - -int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn, const char *name, - u32 type, u32 mask) -{ - struct crypto_alg *alg; - int err; - - type = crypto_skcipher_type(type); - mask = crypto_skcipher_mask(mask); - - alg = crypto_lookup_skcipher(name, type, mask); - if (IS_ERR(alg)) - return PTR_ERR(alg); - - err = crypto_init_spawn(&spawn->base, alg, spawn->base.inst, mask); - crypto_mod_put(alg); - return err; -} -EXPORT_SYMBOL_GPL(crypto_grab_skcipher); - -struct crypto_ablkcipher *crypto_alloc_ablkcipher(const char *alg_name, - u32 type, u32 mask) -{ - struct crypto_tfm *tfm; - int err; - - type = crypto_skcipher_type(type); - mask = crypto_skcipher_mask(mask); - - for (;;) { - struct crypto_alg *alg; - - alg = crypto_lookup_skcipher(alg_name, type, mask); - if (IS_ERR(alg)) { - err = PTR_ERR(alg); - goto err; - } - - tfm = __crypto_alloc_tfm(alg, type, mask); - if (!IS_ERR(tfm)) - return __crypto_ablkcipher_cast(tfm); - - crypto_mod_put(alg); - err = PTR_ERR(tfm); - -err: - if (err != -EAGAIN) - break; - if (fatal_signal_pending(current)) { - err = -EINTR; - break; - } - } - - return ERR_PTR(err); -} -EXPORT_SYMBOL_GPL(crypto_alloc_ablkcipher); diff --git a/crypto/aead.c b/crypto/aead.c index 9b18a1e40d6a..3f5c5ff004ab 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -294,9 +294,9 @@ int aead_init_geniv(struct crypto_aead *aead) if (err) goto out; - ctx->null = crypto_get_default_null_skcipher(); - err = PTR_ERR(ctx->null); - if (IS_ERR(ctx->null)) + ctx->sknull = crypto_get_default_null_skcipher2(); + err = PTR_ERR(ctx->sknull); + if (IS_ERR(ctx->sknull)) goto out; child = crypto_spawn_aead(aead_instance_ctx(inst)); @@ -314,7 +314,7 @@ out: return err; drop_null: - crypto_put_default_null_skcipher(); + crypto_put_default_null_skcipher2(); goto out; } EXPORT_SYMBOL_GPL(aead_init_geniv); @@ -324,7 +324,7 @@ void aead_exit_geniv(struct crypto_aead *tfm) struct aead_geniv_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_aead(ctx->child); - crypto_put_default_null_skcipher(); + crypto_put_default_null_skcipher2(); } EXPORT_SYMBOL_GPL(aead_exit_geniv); @@ -346,9 +346,13 @@ static int aead_prepare_alg(struct aead_alg *alg) { struct crypto_alg *base = &alg->base; - if (max(alg->maxauthsize, alg->ivsize) > PAGE_SIZE / 8) + if (max3(alg->maxauthsize, alg->ivsize, alg->chunksize) > + PAGE_SIZE / 8) return -EINVAL; + if (!alg->chunksize) + alg->chunksize = base->cra_blocksize; + base->cra_type = &crypto_aead_type; base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; base->cra_flags |= CRYPTO_ALG_TYPE_AEAD; diff --git a/crypto/ahash.c b/crypto/ahash.c index 3887a98abcc3..2ce8bcb9049c 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -461,10 +461,10 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) static unsigned int crypto_ahash_extsize(struct crypto_alg *alg) { - if (alg->cra_type == &crypto_ahash_type) - return alg->cra_ctxsize; + if (alg->cra_type != &crypto_ahash_type) + return sizeof(struct crypto_shash *); - return sizeof(struct crypto_shash *); + return crypto_alg_extsize(alg); } #ifdef CONFIG_NET diff --git a/crypto/algapi.c b/crypto/algapi.c index 731255a6104f..df939b54b09f 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -811,6 +811,21 @@ int crypto_attr_u32(struct rtattr *rta, u32 *num) } EXPORT_SYMBOL_GPL(crypto_attr_u32); +int crypto_inst_setname(struct crypto_instance *inst, const char *name, + struct crypto_alg *alg) +{ + if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name, + alg->cra_name) >= CRYPTO_MAX_ALG_NAME) + return -ENAMETOOLONG; + + if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", + name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + return -ENAMETOOLONG; + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_inst_setname); + void *crypto_alloc_instance2(const char *name, struct crypto_alg *alg, unsigned int head) { @@ -825,13 +840,8 @@ void *crypto_alloc_instance2(const char *name, struct crypto_alg *alg, inst = (void *)(p + head); - err = -ENAMETOOLONG; - if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name, - alg->cra_name) >= CRYPTO_MAX_ALG_NAME) - goto err_free_inst; - - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", - name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + err = crypto_inst_setname(inst, name, alg); + if (err) goto err_free_inst; return p; diff --git a/crypto/asymmetric_keys/mscode_parser.c b/crypto/asymmetric_keys/mscode_parser.c index 6a76d5c70ef6..9492e1c22d38 100644 --- a/crypto/asymmetric_keys/mscode_parser.c +++ b/crypto/asymmetric_keys/mscode_parser.c @@ -124,5 +124,10 @@ int mscode_note_digest(void *context, size_t hdrlen, struct pefile_context *ctx = context; ctx->digest = kmemdup(value, vlen, GFP_KERNEL); - return ctx->digest ? 0 : -ENOMEM; + if (!ctx->digest) + return -ENOMEM; + + ctx->digest_len = vlen; + + return 0; } diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c index 44b746e9df1b..2ffd69769466 100644 --- a/crypto/asymmetric_keys/pkcs7_verify.c +++ b/crypto/asymmetric_keys/pkcs7_verify.c @@ -227,7 +227,7 @@ static int pkcs7_verify_sig_chain(struct pkcs7_message *pkcs7, if (asymmetric_key_id_same(p->id, auth)) goto found_issuer_check_skid; } - } else { + } else if (sig->auth_ids[1]) { auth = sig->auth_ids[1]; pr_debug("- want %*phN\n", auth->len, auth->data); for (p = pkcs7->certs; p; p = p->next) { diff --git a/crypto/asymmetric_keys/restrict.c b/crypto/asymmetric_keys/restrict.c index ac4bddf669de..19d1afb9890f 100644 --- a/crypto/asymmetric_keys/restrict.c +++ b/crypto/asymmetric_keys/restrict.c @@ -87,7 +87,7 @@ int restrict_link_by_signature(struct key *trust_keyring, sig = payload->data[asym_auth]; if (!sig->auth_ids[0] && !sig->auth_ids[1]) - return 0; + return -ENOKEY; if (ca_keyid && !asymmetric_key_id_partial(sig->auth_ids[1], ca_keyid)) return -EPERM; diff --git a/crypto/authenc.c b/crypto/authenc.c index 55a354d57251..a7e1ac786c5d 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -32,8 +32,8 @@ struct authenc_instance_ctx { struct crypto_authenc_ctx { struct crypto_ahash *auth; - struct crypto_ablkcipher *enc; - struct crypto_blkcipher *null; + struct crypto_skcipher *enc; + struct crypto_skcipher *null; }; struct authenc_request_ctx { @@ -83,7 +83,7 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key, { struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); struct crypto_ahash *auth = ctx->auth; - struct crypto_ablkcipher *enc = ctx->enc; + struct crypto_skcipher *enc = ctx->enc; struct crypto_authenc_keys keys; int err = -EINVAL; @@ -100,11 +100,11 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key, if (err) goto out; - crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); - crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc) & - CRYPTO_TFM_REQ_MASK); - err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen); - crypto_aead_set_flags(authenc, crypto_ablkcipher_get_flags(enc) & + crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(enc, crypto_aead_get_flags(authenc) & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen); + crypto_aead_set_flags(authenc, crypto_skcipher_get_flags(enc) & CRYPTO_TFM_RES_MASK); out: @@ -184,12 +184,15 @@ static int crypto_authenc_copy_assoc(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct blkcipher_desc desc = { - .tfm = ctx->null, - }; + SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); - return crypto_blkcipher_encrypt(&desc, req->dst, req->src, - req->assoclen); + skcipher_request_set_tfm(skreq, ctx->null); + skcipher_request_set_callback(skreq, aead_request_flags(req), + NULL, NULL); + skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, + NULL); + + return crypto_skcipher_encrypt(skreq); } static int crypto_authenc_encrypt(struct aead_request *req) @@ -199,14 +202,13 @@ static int crypto_authenc_encrypt(struct aead_request *req) struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); - struct crypto_ablkcipher *enc = ctx->enc; + struct crypto_skcipher *enc = ctx->enc; unsigned int cryptlen = req->cryptlen; - struct ablkcipher_request *abreq = (void *)(areq_ctx->tail + - ictx->reqoff); + struct skcipher_request *skreq = (void *)(areq_ctx->tail + + ictx->reqoff); struct scatterlist *src, *dst; int err; - sg_init_table(areq_ctx->src, 2); src = scatterwalk_ffwd(areq_ctx->src, req->src, req->assoclen); dst = src; @@ -215,16 +217,15 @@ static int crypto_authenc_encrypt(struct aead_request *req) if (err) return err; - sg_init_table(areq_ctx->dst, 2); dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); } - ablkcipher_request_set_tfm(abreq, enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - crypto_authenc_encrypt_done, req); - ablkcipher_request_set_crypt(abreq, src, dst, cryptlen, req->iv); + skcipher_request_set_tfm(skreq, enc); + skcipher_request_set_callback(skreq, aead_request_flags(req), + crypto_authenc_encrypt_done, req); + skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv); - err = crypto_ablkcipher_encrypt(abreq); + err = crypto_skcipher_encrypt(skreq); if (err) return err; @@ -240,8 +241,8 @@ static int crypto_authenc_decrypt_tail(struct aead_request *req, struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); - struct ablkcipher_request *abreq = (void *)(areq_ctx->tail + - ictx->reqoff); + struct skcipher_request *skreq = (void *)(areq_ctx->tail + + ictx->reqoff); unsigned int authsize = crypto_aead_authsize(authenc); u8 *ihash = ahreq->result + authsize; struct scatterlist *src, *dst; @@ -251,22 +252,19 @@ static int crypto_authenc_decrypt_tail(struct aead_request *req, if (crypto_memneq(ihash, ahreq->result, authsize)) return -EBADMSG; - sg_init_table(areq_ctx->src, 2); src = scatterwalk_ffwd(areq_ctx->src, req->src, req->assoclen); dst = src; - if (req->src != req->dst) { - sg_init_table(areq_ctx->dst, 2); + if (req->src != req->dst) dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); - } - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, src, dst, - req->cryptlen - authsize, req->iv); + skcipher_request_set_tfm(skreq, ctx->enc); + skcipher_request_set_callback(skreq, aead_request_flags(req), + req->base.complete, req->base.data); + skcipher_request_set_crypt(skreq, src, dst, + req->cryptlen - authsize, req->iv); - return crypto_ablkcipher_decrypt(abreq); + return crypto_skcipher_decrypt(skreq); } static void authenc_verify_ahash_done(struct crypto_async_request *areq, @@ -318,20 +316,20 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm) struct authenc_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_ahash *auth; - struct crypto_ablkcipher *enc; - struct crypto_blkcipher *null; + struct crypto_skcipher *enc; + struct crypto_skcipher *null; int err; auth = crypto_spawn_ahash(&ictx->auth); if (IS_ERR(auth)) return PTR_ERR(auth); - enc = crypto_spawn_skcipher(&ictx->enc); + enc = crypto_spawn_skcipher2(&ictx->enc); err = PTR_ERR(enc); if (IS_ERR(enc)) goto err_free_ahash; - null = crypto_get_default_null_skcipher(); + null = crypto_get_default_null_skcipher2(); err = PTR_ERR(null); if (IS_ERR(null)) goto err_free_skcipher; @@ -347,13 +345,13 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm) max_t(unsigned int, crypto_ahash_reqsize(auth) + sizeof(struct ahash_request), - sizeof(struct ablkcipher_request) + - crypto_ablkcipher_reqsize(enc))); + sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(enc))); return 0; err_free_skcipher: - crypto_free_ablkcipher(enc); + crypto_free_skcipher(enc); err_free_ahash: crypto_free_ahash(auth); return err; @@ -364,8 +362,8 @@ static void crypto_authenc_exit_tfm(struct crypto_aead *tfm) struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_ahash(ctx->auth); - crypto_free_ablkcipher(ctx->enc); - crypto_put_default_null_skcipher(); + crypto_free_skcipher(ctx->enc); + crypto_put_default_null_skcipher2(); } static void crypto_authenc_free(struct aead_instance *inst) @@ -384,7 +382,7 @@ static int crypto_authenc_create(struct crypto_template *tmpl, struct aead_instance *inst; struct hash_alg_common *auth; struct crypto_alg *auth_base; - struct crypto_alg *enc; + struct skcipher_alg *enc; struct authenc_instance_ctx *ctx; const char *enc_name; int err; @@ -397,7 +395,8 @@ static int crypto_authenc_create(struct crypto_template *tmpl, return -EINVAL; auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_AHASH_MASK); + CRYPTO_ALG_TYPE_AHASH_MASK | + crypto_requires_sync(algt->type, algt->mask)); if (IS_ERR(auth)) return PTR_ERR(auth); @@ -421,37 +420,40 @@ static int crypto_authenc_create(struct crypto_template *tmpl, goto err_free_inst; crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst)); - err = crypto_grab_skcipher(&ctx->enc, enc_name, 0, - crypto_requires_sync(algt->type, - algt->mask)); + err = crypto_grab_skcipher2(&ctx->enc, enc_name, 0, + crypto_requires_sync(algt->type, + algt->mask)); if (err) goto err_drop_auth; - enc = crypto_skcipher_spawn_alg(&ctx->enc); + enc = crypto_spawn_skcipher_alg(&ctx->enc); ctx->reqoff = ALIGN(2 * auth->digestsize + auth_base->cra_alignmask, auth_base->cra_alignmask + 1); err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, - "authenc(%s,%s)", auth_base->cra_name, enc->cra_name) >= + "authenc(%s,%s)", auth_base->cra_name, + enc->base.cra_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "authenc(%s,%s)", auth_base->cra_driver_name, - enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; - inst->alg.base.cra_flags = enc->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.base.cra_priority = enc->cra_priority * 10 + + inst->alg.base.cra_flags = (auth_base->cra_flags | + enc->base.cra_flags) & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = enc->base.cra_priority * 10 + auth_base->cra_priority; - inst->alg.base.cra_blocksize = enc->cra_blocksize; + inst->alg.base.cra_blocksize = enc->base.cra_blocksize; inst->alg.base.cra_alignmask = auth_base->cra_alignmask | - enc->cra_alignmask; + enc->base.cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_ctx); - inst->alg.ivsize = enc->cra_ablkcipher.ivsize; + inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc); + inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc); inst->alg.maxauthsize = auth->digestsize; inst->alg.init = crypto_authenc_init_tfm; diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 0c0468869e25..121010ac9962 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -35,8 +35,8 @@ struct authenc_esn_instance_ctx { struct crypto_authenc_esn_ctx { unsigned int reqoff; struct crypto_ahash *auth; - struct crypto_ablkcipher *enc; - struct crypto_blkcipher *null; + struct crypto_skcipher *enc; + struct crypto_skcipher *null; }; struct authenc_esn_request_ctx { @@ -65,7 +65,7 @@ static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 * { struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); struct crypto_ahash *auth = ctx->auth; - struct crypto_ablkcipher *enc = ctx->enc; + struct crypto_skcipher *enc = ctx->enc; struct crypto_authenc_keys keys; int err = -EINVAL; @@ -82,11 +82,11 @@ static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 * if (err) goto out; - crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); - crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) & + crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) & CRYPTO_TFM_REQ_MASK); - err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen); - crypto_aead_set_flags(authenc_esn, crypto_ablkcipher_get_flags(enc) & + err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen); + crypto_aead_set_flags(authenc_esn, crypto_skcipher_get_flags(enc) & CRYPTO_TFM_RES_MASK); out: @@ -182,11 +182,14 @@ static int crypto_authenc_esn_copy(struct aead_request *req, unsigned int len) { struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct blkcipher_desc desc = { - .tfm = ctx->null, - }; + SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); - return crypto_blkcipher_encrypt(&desc, req->dst, req->src, len); + skcipher_request_set_tfm(skreq, ctx->null); + skcipher_request_set_callback(skreq, aead_request_flags(req), + NULL, NULL); + skcipher_request_set_crypt(skreq, req->src, req->dst, len, NULL); + + return crypto_skcipher_encrypt(skreq); } static int crypto_authenc_esn_encrypt(struct aead_request *req) @@ -194,9 +197,9 @@ static int crypto_authenc_esn_encrypt(struct aead_request *req) struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req); struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct ablkcipher_request *abreq = (void *)(areq_ctx->tail - + ctx->reqoff); - struct crypto_ablkcipher *enc = ctx->enc; + struct skcipher_request *skreq = (void *)(areq_ctx->tail + + ctx->reqoff); + struct crypto_skcipher *enc = ctx->enc; unsigned int assoclen = req->assoclen; unsigned int cryptlen = req->cryptlen; struct scatterlist *src, *dst; @@ -215,12 +218,12 @@ static int crypto_authenc_esn_encrypt(struct aead_request *req) dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, assoclen); } - ablkcipher_request_set_tfm(abreq, enc); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - crypto_authenc_esn_encrypt_done, req); - ablkcipher_request_set_crypt(abreq, src, dst, cryptlen, req->iv); + skcipher_request_set_tfm(skreq, enc); + skcipher_request_set_callback(skreq, aead_request_flags(req), + crypto_authenc_esn_encrypt_done, req); + skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv); - err = crypto_ablkcipher_encrypt(abreq); + err = crypto_skcipher_encrypt(skreq); if (err) return err; @@ -234,8 +237,8 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, unsigned int authsize = crypto_aead_authsize(authenc_esn); struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req); struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); - struct ablkcipher_request *abreq = (void *)(areq_ctx->tail - + ctx->reqoff); + struct skcipher_request *skreq = (void *)(areq_ctx->tail + + ctx->reqoff); struct crypto_ahash *auth = ctx->auth; u8 *ohash = PTR_ALIGN((u8 *)areq_ctx->tail, crypto_ahash_alignmask(auth) + 1); @@ -256,12 +259,12 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, sg_init_table(areq_ctx->dst, 2); dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen); - ablkcipher_request_set_tfm(abreq, ctx->enc); - ablkcipher_request_set_callback(abreq, flags, - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(abreq, dst, dst, cryptlen, req->iv); + skcipher_request_set_tfm(skreq, ctx->enc); + skcipher_request_set_callback(skreq, flags, + req->base.complete, req->base.data); + skcipher_request_set_crypt(skreq, dst, dst, cryptlen, req->iv); - return crypto_ablkcipher_decrypt(abreq); + return crypto_skcipher_decrypt(skreq); } static void authenc_esn_verify_ahash_done(struct crypto_async_request *areq, @@ -331,20 +334,20 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm) struct authenc_esn_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_ahash *auth; - struct crypto_ablkcipher *enc; - struct crypto_blkcipher *null; + struct crypto_skcipher *enc; + struct crypto_skcipher *null; int err; auth = crypto_spawn_ahash(&ictx->auth); if (IS_ERR(auth)) return PTR_ERR(auth); - enc = crypto_spawn_skcipher(&ictx->enc); + enc = crypto_spawn_skcipher2(&ictx->enc); err = PTR_ERR(enc); if (IS_ERR(enc)) goto err_free_ahash; - null = crypto_get_default_null_skcipher(); + null = crypto_get_default_null_skcipher2(); err = PTR_ERR(null); if (IS_ERR(null)) goto err_free_skcipher; @@ -361,15 +364,15 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm) sizeof(struct authenc_esn_request_ctx) + ctx->reqoff + max_t(unsigned int, - crypto_ahash_reqsize(auth) + - sizeof(struct ahash_request), - sizeof(struct skcipher_givcrypt_request) + - crypto_ablkcipher_reqsize(enc))); + crypto_ahash_reqsize(auth) + + sizeof(struct ahash_request), + sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(enc))); return 0; err_free_skcipher: - crypto_free_ablkcipher(enc); + crypto_free_skcipher(enc); err_free_ahash: crypto_free_ahash(auth); return err; @@ -380,8 +383,8 @@ static void crypto_authenc_esn_exit_tfm(struct crypto_aead *tfm) struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_ahash(ctx->auth); - crypto_free_ablkcipher(ctx->enc); - crypto_put_default_null_skcipher(); + crypto_free_skcipher(ctx->enc); + crypto_put_default_null_skcipher2(); } static void crypto_authenc_esn_free(struct aead_instance *inst) @@ -400,7 +403,7 @@ static int crypto_authenc_esn_create(struct crypto_template *tmpl, struct aead_instance *inst; struct hash_alg_common *auth; struct crypto_alg *auth_base; - struct crypto_alg *enc; + struct skcipher_alg *enc; struct authenc_esn_instance_ctx *ctx; const char *enc_name; int err; @@ -413,7 +416,8 @@ static int crypto_authenc_esn_create(struct crypto_template *tmpl, return -EINVAL; auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_AHASH_MASK); + CRYPTO_ALG_TYPE_AHASH_MASK | + crypto_requires_sync(algt->type, algt->mask)); if (IS_ERR(auth)) return PTR_ERR(auth); @@ -437,34 +441,36 @@ static int crypto_authenc_esn_create(struct crypto_template *tmpl, goto err_free_inst; crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst)); - err = crypto_grab_skcipher(&ctx->enc, enc_name, 0, - crypto_requires_sync(algt->type, - algt->mask)); + err = crypto_grab_skcipher2(&ctx->enc, enc_name, 0, + crypto_requires_sync(algt->type, + algt->mask)); if (err) goto err_drop_auth; - enc = crypto_skcipher_spawn_alg(&ctx->enc); + enc = crypto_spawn_skcipher_alg(&ctx->enc); err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "authencesn(%s,%s)", auth_base->cra_name, - enc->cra_name) >= CRYPTO_MAX_ALG_NAME) + enc->base.cra_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "authencesn(%s,%s)", auth_base->cra_driver_name, - enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; - inst->alg.base.cra_flags = enc->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.base.cra_priority = enc->cra_priority * 10 + + inst->alg.base.cra_flags = (auth_base->cra_flags | + enc->base.cra_flags) & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = enc->base.cra_priority * 10 + auth_base->cra_priority; - inst->alg.base.cra_blocksize = enc->cra_blocksize; + inst->alg.base.cra_blocksize = enc->base.cra_blocksize; inst->alg.base.cra_alignmask = auth_base->cra_alignmask | - enc->cra_alignmask; + enc->base.cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_esn_ctx); - inst->alg.ivsize = enc->cra_ablkcipher.ivsize; + inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc); + inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc); inst->alg.maxauthsize = auth->digestsize; inst->alg.init = crypto_authenc_esn_init_tfm; diff --git a/crypto/blkcipher.c b/crypto/blkcipher.c index 8cc1622b2ee0..369999530108 100644 --- a/crypto/blkcipher.c +++ b/crypto/blkcipher.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -466,10 +465,6 @@ static int crypto_init_blkcipher_ops_async(struct crypto_tfm *tfm) crt->setkey = async_setkey; crt->encrypt = async_encrypt; crt->decrypt = async_decrypt; - if (!alg->ivsize) { - crt->givencrypt = skcipher_null_givencrypt; - crt->givdecrypt = skcipher_null_givdecrypt; - } crt->base = __crypto_ablkcipher_cast(tfm); crt->ivsize = alg->ivsize; @@ -560,185 +555,5 @@ const struct crypto_type crypto_blkcipher_type = { }; EXPORT_SYMBOL_GPL(crypto_blkcipher_type); -static int crypto_grab_nivcipher(struct crypto_skcipher_spawn *spawn, - const char *name, u32 type, u32 mask) -{ - struct crypto_alg *alg; - int err; - - type = crypto_skcipher_type(type); - mask = crypto_skcipher_mask(mask)| CRYPTO_ALG_GENIV; - - alg = crypto_alg_mod_lookup(name, type, mask); - if (IS_ERR(alg)) - return PTR_ERR(alg); - - err = crypto_init_spawn(&spawn->base, alg, spawn->base.inst, mask); - crypto_mod_put(alg); - return err; -} - -struct crypto_instance *skcipher_geniv_alloc(struct crypto_template *tmpl, - struct rtattr **tb, u32 type, - u32 mask) -{ - struct { - int (*setkey)(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int keylen); - int (*encrypt)(struct ablkcipher_request *req); - int (*decrypt)(struct ablkcipher_request *req); - - unsigned int min_keysize; - unsigned int max_keysize; - unsigned int ivsize; - - const char *geniv; - } balg; - const char *name; - struct crypto_skcipher_spawn *spawn; - struct crypto_attr_type *algt; - struct crypto_instance *inst; - struct crypto_alg *alg; - int err; - - algt = crypto_get_attr_type(tb); - if (IS_ERR(algt)) - return ERR_CAST(algt); - - if ((algt->type ^ (CRYPTO_ALG_TYPE_GIVCIPHER | CRYPTO_ALG_GENIV)) & - algt->mask) - return ERR_PTR(-EINVAL); - - name = crypto_attr_alg_name(tb[1]); - if (IS_ERR(name)) - return ERR_CAST(name); - - inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); - if (!inst) - return ERR_PTR(-ENOMEM); - - spawn = crypto_instance_ctx(inst); - - /* Ignore async algorithms if necessary. */ - mask |= crypto_requires_sync(algt->type, algt->mask); - - crypto_set_skcipher_spawn(spawn, inst); - err = crypto_grab_nivcipher(spawn, name, type, mask); - if (err) - goto err_free_inst; - - alg = crypto_skcipher_spawn_alg(spawn); - - if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER) { - balg.ivsize = alg->cra_blkcipher.ivsize; - balg.min_keysize = alg->cra_blkcipher.min_keysize; - balg.max_keysize = alg->cra_blkcipher.max_keysize; - - balg.setkey = async_setkey; - balg.encrypt = async_encrypt; - balg.decrypt = async_decrypt; - - balg.geniv = alg->cra_blkcipher.geniv; - } else { - balg.ivsize = alg->cra_ablkcipher.ivsize; - balg.min_keysize = alg->cra_ablkcipher.min_keysize; - balg.max_keysize = alg->cra_ablkcipher.max_keysize; - - balg.setkey = alg->cra_ablkcipher.setkey; - balg.encrypt = alg->cra_ablkcipher.encrypt; - balg.decrypt = alg->cra_ablkcipher.decrypt; - - balg.geniv = alg->cra_ablkcipher.geniv; - } - - err = -EINVAL; - if (!balg.ivsize) - goto err_drop_alg; - - /* - * This is only true if we're constructing an algorithm with its - * default IV generator. For the default generator we elide the - * template name and double-check the IV generator. - */ - if (algt->mask & CRYPTO_ALG_GENIV) { - if (!balg.geniv) - balg.geniv = crypto_default_geniv(alg); - err = -EAGAIN; - if (strcmp(tmpl->name, balg.geniv)) - goto err_drop_alg; - - memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); - memcpy(inst->alg.cra_driver_name, alg->cra_driver_name, - CRYPTO_MAX_ALG_NAME); - } else { - err = -ENAMETOOLONG; - if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, - "%s(%s)", tmpl->name, alg->cra_name) >= - CRYPTO_MAX_ALG_NAME) - goto err_drop_alg; - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "%s(%s)", tmpl->name, alg->cra_driver_name) >= - CRYPTO_MAX_ALG_NAME) - goto err_drop_alg; - } - - inst->alg.cra_flags = CRYPTO_ALG_TYPE_GIVCIPHER | CRYPTO_ALG_GENIV; - inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.cra_priority = alg->cra_priority; - inst->alg.cra_blocksize = alg->cra_blocksize; - inst->alg.cra_alignmask = alg->cra_alignmask; - inst->alg.cra_type = &crypto_givcipher_type; - - inst->alg.cra_ablkcipher.ivsize = balg.ivsize; - inst->alg.cra_ablkcipher.min_keysize = balg.min_keysize; - inst->alg.cra_ablkcipher.max_keysize = balg.max_keysize; - inst->alg.cra_ablkcipher.geniv = balg.geniv; - - inst->alg.cra_ablkcipher.setkey = balg.setkey; - inst->alg.cra_ablkcipher.encrypt = balg.encrypt; - inst->alg.cra_ablkcipher.decrypt = balg.decrypt; - -out: - return inst; - -err_drop_alg: - crypto_drop_skcipher(spawn); -err_free_inst: - kfree(inst); - inst = ERR_PTR(err); - goto out; -} -EXPORT_SYMBOL_GPL(skcipher_geniv_alloc); - -void skcipher_geniv_free(struct crypto_instance *inst) -{ - crypto_drop_skcipher(crypto_instance_ctx(inst)); - kfree(inst); -} -EXPORT_SYMBOL_GPL(skcipher_geniv_free); - -int skcipher_geniv_init(struct crypto_tfm *tfm) -{ - struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct crypto_ablkcipher *cipher; - - cipher = crypto_spawn_skcipher(crypto_instance_ctx(inst)); - if (IS_ERR(cipher)) - return PTR_ERR(cipher); - - tfm->crt_ablkcipher.base = cipher; - tfm->crt_ablkcipher.reqsize += crypto_ablkcipher_reqsize(cipher); - - return 0; -} -EXPORT_SYMBOL_GPL(skcipher_geniv_init); - -void skcipher_geniv_exit(struct crypto_tfm *tfm) -{ - crypto_free_ablkcipher(tfm->crt_ablkcipher.base); -} -EXPORT_SYMBOL_GPL(skcipher_geniv_exit); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Generic block chaining cipher type"); diff --git a/crypto/ccm.c b/crypto/ccm.c index cc31ea4335bf..006d8575ef5c 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -28,7 +28,7 @@ struct ccm_instance_ctx { struct crypto_ccm_ctx { struct crypto_cipher *cipher; - struct crypto_ablkcipher *ctr; + struct crypto_skcipher *ctr; }; struct crypto_rfc4309_ctx { @@ -50,7 +50,7 @@ struct crypto_ccm_req_priv_ctx { u32 flags; struct scatterlist src[3]; struct scatterlist dst[3]; - struct ablkcipher_request abreq; + struct skcipher_request skreq; }; static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx( @@ -83,15 +83,15 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); - struct crypto_ablkcipher *ctr = ctx->ctr; + struct crypto_skcipher *ctr = ctx->ctr; struct crypto_cipher *tfm = ctx->cipher; int err = 0; - crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); - crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) & - CRYPTO_TFM_REQ_MASK); - err = crypto_ablkcipher_setkey(ctr, key, keylen); - crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) & + crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(ctr, key, keylen); + crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) & CRYPTO_TFM_RES_MASK); if (err) goto out; @@ -347,7 +347,7 @@ static int crypto_ccm_encrypt(struct aead_request *req) struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); - struct ablkcipher_request *abreq = &pctx->abreq; + struct skcipher_request *skreq = &pctx->skreq; struct scatterlist *dst; unsigned int cryptlen = req->cryptlen; u8 *odata = pctx->odata; @@ -366,11 +366,11 @@ static int crypto_ccm_encrypt(struct aead_request *req) if (req->src != req->dst) dst = pctx->dst; - ablkcipher_request_set_tfm(abreq, ctx->ctr); - ablkcipher_request_set_callback(abreq, pctx->flags, - crypto_ccm_encrypt_done, req); - ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv); - err = crypto_ablkcipher_encrypt(abreq); + skcipher_request_set_tfm(skreq, ctx->ctr); + skcipher_request_set_callback(skreq, pctx->flags, + crypto_ccm_encrypt_done, req); + skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + 16, iv); + err = crypto_skcipher_encrypt(skreq); if (err) return err; @@ -407,7 +407,7 @@ static int crypto_ccm_decrypt(struct aead_request *req) struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); - struct ablkcipher_request *abreq = &pctx->abreq; + struct skcipher_request *skreq = &pctx->skreq; struct scatterlist *dst; unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen; @@ -429,11 +429,11 @@ static int crypto_ccm_decrypt(struct aead_request *req) if (req->src != req->dst) dst = pctx->dst; - ablkcipher_request_set_tfm(abreq, ctx->ctr); - ablkcipher_request_set_callback(abreq, pctx->flags, - crypto_ccm_decrypt_done, req); - ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv); - err = crypto_ablkcipher_decrypt(abreq); + skcipher_request_set_tfm(skreq, ctx->ctr); + skcipher_request_set_callback(skreq, pctx->flags, + crypto_ccm_decrypt_done, req); + skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + 16, iv); + err = crypto_skcipher_decrypt(skreq); if (err) return err; @@ -454,7 +454,7 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm) struct ccm_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_cipher *cipher; - struct crypto_ablkcipher *ctr; + struct crypto_skcipher *ctr; unsigned long align; int err; @@ -462,7 +462,7 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm) if (IS_ERR(cipher)) return PTR_ERR(cipher); - ctr = crypto_spawn_skcipher(&ictx->ctr); + ctr = crypto_spawn_skcipher2(&ictx->ctr); err = PTR_ERR(ctr); if (IS_ERR(ctr)) goto err_free_cipher; @@ -475,7 +475,7 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm) crypto_aead_set_reqsize( tfm, align + sizeof(struct crypto_ccm_req_priv_ctx) + - crypto_ablkcipher_reqsize(ctr)); + crypto_skcipher_reqsize(ctr)); return 0; @@ -489,7 +489,7 @@ static void crypto_ccm_exit_tfm(struct crypto_aead *tfm) struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_cipher(ctx->cipher); - crypto_free_ablkcipher(ctx->ctr); + crypto_free_skcipher(ctx->ctr); } static void crypto_ccm_free(struct aead_instance *inst) @@ -509,7 +509,7 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl, { struct crypto_attr_type *algt; struct aead_instance *inst; - struct crypto_alg *ctr; + struct skcipher_alg *ctr; struct crypto_alg *cipher; struct ccm_instance_ctx *ictx; int err; @@ -544,39 +544,40 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl, goto err_free_inst; crypto_set_skcipher_spawn(&ictx->ctr, aead_crypto_instance(inst)); - err = crypto_grab_skcipher(&ictx->ctr, ctr_name, 0, - crypto_requires_sync(algt->type, - algt->mask)); + err = crypto_grab_skcipher2(&ictx->ctr, ctr_name, 0, + crypto_requires_sync(algt->type, + algt->mask)); if (err) goto err_drop_cipher; - ctr = crypto_skcipher_spawn_alg(&ictx->ctr); + ctr = crypto_spawn_skcipher_alg(&ictx->ctr); /* Not a stream cipher? */ err = -EINVAL; - if (ctr->cra_blocksize != 1) + if (ctr->base.cra_blocksize != 1) goto err_drop_ctr; /* We want the real thing! */ - if (ctr->cra_ablkcipher.ivsize != 16) + if (crypto_skcipher_alg_ivsize(ctr) != 16) goto err_drop_ctr; err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "ccm_base(%s,%s)", ctr->cra_driver_name, + "ccm_base(%s,%s)", ctr->base.cra_driver_name, cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_ctr; memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); - inst->alg.base.cra_flags = ctr->cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = (cipher->cra_priority + - ctr->cra_priority) / 2; + ctr->base.cra_priority) / 2; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = cipher->cra_alignmask | - ctr->cra_alignmask | + ctr->base.cra_alignmask | (__alignof__(u32) - 1); inst->alg.ivsize = 16; + inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr); inst->alg.maxauthsize = 16; inst->alg.base.cra_ctxsize = sizeof(struct crypto_ccm_ctx); inst->alg.init = crypto_ccm_init_tfm; @@ -863,6 +864,7 @@ static int crypto_rfc4309_create(struct crypto_template *tmpl, inst->alg.base.cra_alignmask = alg->base.cra_alignmask; inst->alg.ivsize = 8; + inst->alg.chunksize = crypto_aead_alg_chunksize(alg); inst->alg.maxauthsize = 16; inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4309_ctx); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index 7b6b935cef23..e899ef51dc8e 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -31,7 +31,7 @@ struct chachapoly_instance_ctx { }; struct chachapoly_ctx { - struct crypto_ablkcipher *chacha; + struct crypto_skcipher *chacha; struct crypto_ahash *poly; /* key bytes we use for the ChaCha20 IV */ unsigned int saltlen; @@ -53,7 +53,7 @@ struct poly_req { struct chacha_req { u8 iv[CHACHA20_IV_SIZE]; struct scatterlist src[1]; - struct ablkcipher_request req; /* must be last member */ + struct skcipher_request req; /* must be last member */ }; struct chachapoly_req_ctx { @@ -144,12 +144,12 @@ static int chacha_decrypt(struct aead_request *req) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); } - ablkcipher_request_set_callback(&creq->req, aead_request_flags(req), - chacha_decrypt_done, req); - ablkcipher_request_set_tfm(&creq->req, ctx->chacha); - ablkcipher_request_set_crypt(&creq->req, src, dst, - rctx->cryptlen, creq->iv); - err = crypto_ablkcipher_decrypt(&creq->req); + skcipher_request_set_callback(&creq->req, aead_request_flags(req), + chacha_decrypt_done, req); + skcipher_request_set_tfm(&creq->req, ctx->chacha); + skcipher_request_set_crypt(&creq->req, src, dst, + rctx->cryptlen, creq->iv); + err = crypto_skcipher_decrypt(&creq->req); if (err) return err; @@ -393,13 +393,13 @@ static int poly_genkey(struct aead_request *req) chacha_iv(creq->iv, req, 0); - ablkcipher_request_set_callback(&creq->req, aead_request_flags(req), - poly_genkey_done, req); - ablkcipher_request_set_tfm(&creq->req, ctx->chacha); - ablkcipher_request_set_crypt(&creq->req, creq->src, creq->src, - POLY1305_KEY_SIZE, creq->iv); + skcipher_request_set_callback(&creq->req, aead_request_flags(req), + poly_genkey_done, req); + skcipher_request_set_tfm(&creq->req, ctx->chacha); + skcipher_request_set_crypt(&creq->req, creq->src, creq->src, + POLY1305_KEY_SIZE, creq->iv); - err = crypto_ablkcipher_decrypt(&creq->req); + err = crypto_skcipher_decrypt(&creq->req); if (err) return err; @@ -433,12 +433,12 @@ static int chacha_encrypt(struct aead_request *req) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); } - ablkcipher_request_set_callback(&creq->req, aead_request_flags(req), - chacha_encrypt_done, req); - ablkcipher_request_set_tfm(&creq->req, ctx->chacha); - ablkcipher_request_set_crypt(&creq->req, src, dst, - req->cryptlen, creq->iv); - err = crypto_ablkcipher_encrypt(&creq->req); + skcipher_request_set_callback(&creq->req, aead_request_flags(req), + chacha_encrypt_done, req); + skcipher_request_set_tfm(&creq->req, ctx->chacha); + skcipher_request_set_crypt(&creq->req, src, dst, + req->cryptlen, creq->iv); + err = crypto_skcipher_encrypt(&creq->req); if (err) return err; @@ -500,13 +500,13 @@ static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, keylen -= ctx->saltlen; memcpy(ctx->salt, key + keylen, ctx->saltlen); - crypto_ablkcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK); - crypto_ablkcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) & - CRYPTO_TFM_REQ_MASK); + crypto_skcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); - err = crypto_ablkcipher_setkey(ctx->chacha, key, keylen); - crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctx->chacha) & - CRYPTO_TFM_RES_MASK); + err = crypto_skcipher_setkey(ctx->chacha, key, keylen); + crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctx->chacha) & + CRYPTO_TFM_RES_MASK); return err; } @@ -524,7 +524,7 @@ static int chachapoly_init(struct crypto_aead *tfm) struct aead_instance *inst = aead_alg_instance(tfm); struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst); struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); - struct crypto_ablkcipher *chacha; + struct crypto_skcipher *chacha; struct crypto_ahash *poly; unsigned long align; @@ -532,7 +532,7 @@ static int chachapoly_init(struct crypto_aead *tfm) if (IS_ERR(poly)) return PTR_ERR(poly); - chacha = crypto_spawn_skcipher(&ictx->chacha); + chacha = crypto_spawn_skcipher2(&ictx->chacha); if (IS_ERR(chacha)) { crypto_free_ahash(poly); return PTR_ERR(chacha); @@ -548,8 +548,8 @@ static int chachapoly_init(struct crypto_aead *tfm) tfm, align + offsetof(struct chachapoly_req_ctx, u) + max(offsetof(struct chacha_req, req) + - sizeof(struct ablkcipher_request) + - crypto_ablkcipher_reqsize(chacha), + sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(chacha), offsetof(struct poly_req, req) + sizeof(struct ahash_request) + crypto_ahash_reqsize(poly))); @@ -562,7 +562,7 @@ static void chachapoly_exit(struct crypto_aead *tfm) struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_ahash(ctx->poly); - crypto_free_ablkcipher(ctx->chacha); + crypto_free_skcipher(ctx->chacha); } static void chachapoly_free(struct aead_instance *inst) @@ -579,7 +579,7 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, { struct crypto_attr_type *algt; struct aead_instance *inst; - struct crypto_alg *chacha; + struct skcipher_alg *chacha; struct crypto_alg *poly; struct hash_alg_common *poly_hash; struct chachapoly_instance_ctx *ctx; @@ -605,7 +605,9 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, poly = crypto_find_alg(poly_name, &crypto_ahash_type, CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_AHASH_MASK); + CRYPTO_ALG_TYPE_AHASH_MASK | + crypto_requires_sync(algt->type, + algt->mask)); if (IS_ERR(poly)) return PTR_ERR(poly); @@ -623,20 +625,20 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, goto err_free_inst; crypto_set_skcipher_spawn(&ctx->chacha, aead_crypto_instance(inst)); - err = crypto_grab_skcipher(&ctx->chacha, chacha_name, 0, - crypto_requires_sync(algt->type, - algt->mask)); + err = crypto_grab_skcipher2(&ctx->chacha, chacha_name, 0, + crypto_requires_sync(algt->type, + algt->mask)); if (err) goto err_drop_poly; - chacha = crypto_skcipher_spawn_alg(&ctx->chacha); + chacha = crypto_spawn_skcipher_alg(&ctx->chacha); err = -EINVAL; /* Need 16-byte IV size, including Initial Block Counter value */ - if (chacha->cra_ablkcipher.ivsize != CHACHA20_IV_SIZE) + if (crypto_skcipher_alg_ivsize(chacha) != CHACHA20_IV_SIZE) goto out_drop_chacha; /* Not a stream cipher? */ - if (chacha->cra_blocksize != 1) + if (chacha->base.cra_blocksize != 1) goto out_drop_chacha; err = -ENAMETOOLONG; @@ -645,20 +647,21 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, poly_name) >= CRYPTO_MAX_ALG_NAME) goto out_drop_chacha; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "%s(%s,%s)", name, chacha->cra_driver_name, + "%s(%s,%s)", name, chacha->base.cra_driver_name, poly->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_drop_chacha; - inst->alg.base.cra_flags = (chacha->cra_flags | poly->cra_flags) & + inst->alg.base.cra_flags = (chacha->base.cra_flags | poly->cra_flags) & CRYPTO_ALG_ASYNC; - inst->alg.base.cra_priority = (chacha->cra_priority + + inst->alg.base.cra_priority = (chacha->base.cra_priority + poly->cra_priority) / 2; inst->alg.base.cra_blocksize = 1; - inst->alg.base.cra_alignmask = chacha->cra_alignmask | + inst->alg.base.cra_alignmask = chacha->base.cra_alignmask | poly->cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) + ctx->saltlen; inst->alg.ivsize = ivsize; + inst->alg.chunksize = crypto_skcipher_alg_chunksize(chacha); inst->alg.maxauthsize = POLY1305_DIGEST_SIZE; inst->alg.init = chachapoly_init; inst->alg.exit = chachapoly_exit; diff --git a/crypto/chainiv.c b/crypto/chainiv.c deleted file mode 100644 index b4340018c8d4..000000000000 --- a/crypto/chainiv.c +++ /dev/null @@ -1,317 +0,0 @@ -/* - * chainiv: Chain IV Generator - * - * Generate IVs simply be using the last block of the previous encryption. - * This is mainly useful for CBC with a synchronous algorithm. - * - * Copyright (c) 2007 Herbert Xu - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - CHAINIV_STATE_INUSE = 0, -}; - -struct chainiv_ctx { - spinlock_t lock; - char iv[]; -}; - -struct async_chainiv_ctx { - unsigned long state; - - spinlock_t lock; - int err; - - struct crypto_queue queue; - struct work_struct postponed; - - char iv[]; -}; - -static int chainiv_givencrypt(struct skcipher_givcrypt_request *req) -{ - struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req); - struct chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv); - struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req); - unsigned int ivsize; - int err; - - ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv)); - ablkcipher_request_set_callback(subreq, req->creq.base.flags & - ~CRYPTO_TFM_REQ_MAY_SLEEP, - req->creq.base.complete, - req->creq.base.data); - ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst, - req->creq.nbytes, req->creq.info); - - spin_lock_bh(&ctx->lock); - - ivsize = crypto_ablkcipher_ivsize(geniv); - - memcpy(req->giv, ctx->iv, ivsize); - memcpy(subreq->info, ctx->iv, ivsize); - - err = crypto_ablkcipher_encrypt(subreq); - if (err) - goto unlock; - - memcpy(ctx->iv, subreq->info, ivsize); - -unlock: - spin_unlock_bh(&ctx->lock); - - return err; -} - -static int chainiv_init_common(struct crypto_tfm *tfm, char iv[]) -{ - struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm); - int err = 0; - - tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request); - - if (iv) { - err = crypto_rng_get_bytes(crypto_default_rng, iv, - crypto_ablkcipher_ivsize(geniv)); - crypto_put_default_rng(); - } - - return err ?: skcipher_geniv_init(tfm); -} - -static int chainiv_init(struct crypto_tfm *tfm) -{ - struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm); - struct chainiv_ctx *ctx = crypto_tfm_ctx(tfm); - char *iv; - - spin_lock_init(&ctx->lock); - - iv = NULL; - if (!crypto_get_default_rng()) { - crypto_ablkcipher_crt(geniv)->givencrypt = chainiv_givencrypt; - iv = ctx->iv; - } - - return chainiv_init_common(tfm, iv); -} - -static int async_chainiv_schedule_work(struct async_chainiv_ctx *ctx) -{ - int queued; - int err = ctx->err; - - if (!ctx->queue.qlen) { - smp_mb__before_atomic(); - clear_bit(CHAINIV_STATE_INUSE, &ctx->state); - - if (!ctx->queue.qlen || - test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state)) - goto out; - } - - queued = queue_work(kcrypto_wq, &ctx->postponed); - BUG_ON(!queued); - -out: - return err; -} - -static int async_chainiv_postpone_request(struct skcipher_givcrypt_request *req) -{ - struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req); - struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv); - int err; - - spin_lock_bh(&ctx->lock); - err = skcipher_enqueue_givcrypt(&ctx->queue, req); - spin_unlock_bh(&ctx->lock); - - if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state)) - return err; - - ctx->err = err; - return async_chainiv_schedule_work(ctx); -} - -static int async_chainiv_givencrypt_tail(struct skcipher_givcrypt_request *req) -{ - struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req); - struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv); - struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req); - unsigned int ivsize = crypto_ablkcipher_ivsize(geniv); - - memcpy(req->giv, ctx->iv, ivsize); - memcpy(subreq->info, ctx->iv, ivsize); - - ctx->err = crypto_ablkcipher_encrypt(subreq); - if (ctx->err) - goto out; - - memcpy(ctx->iv, subreq->info, ivsize); - -out: - return async_chainiv_schedule_work(ctx); -} - -static int async_chainiv_givencrypt(struct skcipher_givcrypt_request *req) -{ - struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req); - struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv); - struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req); - - ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv)); - ablkcipher_request_set_callback(subreq, req->creq.base.flags, - req->creq.base.complete, - req->creq.base.data); - ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst, - req->creq.nbytes, req->creq.info); - - if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state)) - goto postpone; - - if (ctx->queue.qlen) { - clear_bit(CHAINIV_STATE_INUSE, &ctx->state); - goto postpone; - } - - return async_chainiv_givencrypt_tail(req); - -postpone: - return async_chainiv_postpone_request(req); -} - -static void async_chainiv_do_postponed(struct work_struct *work) -{ - struct async_chainiv_ctx *ctx = container_of(work, - struct async_chainiv_ctx, - postponed); - struct skcipher_givcrypt_request *req; - struct ablkcipher_request *subreq; - int err; - - /* Only handle one request at a time to avoid hogging keventd. */ - spin_lock_bh(&ctx->lock); - req = skcipher_dequeue_givcrypt(&ctx->queue); - spin_unlock_bh(&ctx->lock); - - if (!req) { - async_chainiv_schedule_work(ctx); - return; - } - - subreq = skcipher_givcrypt_reqctx(req); - subreq->base.flags |= CRYPTO_TFM_REQ_MAY_SLEEP; - - err = async_chainiv_givencrypt_tail(req); - - local_bh_disable(); - skcipher_givcrypt_complete(req, err); - local_bh_enable(); -} - -static int async_chainiv_init(struct crypto_tfm *tfm) -{ - struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm); - struct async_chainiv_ctx *ctx = crypto_tfm_ctx(tfm); - char *iv; - - spin_lock_init(&ctx->lock); - - crypto_init_queue(&ctx->queue, 100); - INIT_WORK(&ctx->postponed, async_chainiv_do_postponed); - - iv = NULL; - if (!crypto_get_default_rng()) { - crypto_ablkcipher_crt(geniv)->givencrypt = - async_chainiv_givencrypt; - iv = ctx->iv; - } - - return chainiv_init_common(tfm, iv); -} - -static void async_chainiv_exit(struct crypto_tfm *tfm) -{ - struct async_chainiv_ctx *ctx = crypto_tfm_ctx(tfm); - - BUG_ON(test_bit(CHAINIV_STATE_INUSE, &ctx->state) || ctx->queue.qlen); - - skcipher_geniv_exit(tfm); -} - -static struct crypto_template chainiv_tmpl; - -static struct crypto_instance *chainiv_alloc(struct rtattr **tb) -{ - struct crypto_attr_type *algt; - struct crypto_instance *inst; - - algt = crypto_get_attr_type(tb); - if (IS_ERR(algt)) - return ERR_CAST(algt); - - inst = skcipher_geniv_alloc(&chainiv_tmpl, tb, 0, 0); - if (IS_ERR(inst)) - goto out; - - inst->alg.cra_init = chainiv_init; - inst->alg.cra_exit = skcipher_geniv_exit; - - inst->alg.cra_ctxsize = sizeof(struct chainiv_ctx); - - if (!crypto_requires_sync(algt->type, algt->mask)) { - inst->alg.cra_flags |= CRYPTO_ALG_ASYNC; - - inst->alg.cra_init = async_chainiv_init; - inst->alg.cra_exit = async_chainiv_exit; - - inst->alg.cra_ctxsize = sizeof(struct async_chainiv_ctx); - } - - inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize; - -out: - return inst; -} - -static struct crypto_template chainiv_tmpl = { - .name = "chainiv", - .alloc = chainiv_alloc, - .free = skcipher_geniv_free, - .module = THIS_MODULE, -}; - -static int __init chainiv_module_init(void) -{ - return crypto_register_template(&chainiv_tmpl); -} - -static void chainiv_module_exit(void) -{ - crypto_unregister_template(&chainiv_tmpl); -} - -module_init(chainiv_module_init); -module_exit(chainiv_module_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Chain IV Generator"); -MODULE_ALIAS_CRYPTO("chainiv"); diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 7921251cdb13..cf8037a87b2d 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ #include #include -#define CRYPTD_MAX_CPU_QLEN 100 +#define CRYPTD_MAX_CPU_QLEN 1000 struct cryptd_cpu_queue { struct crypto_queue queue; @@ -58,6 +59,7 @@ struct aead_instance_ctx { }; struct cryptd_blkcipher_ctx { + atomic_t refcnt; struct crypto_blkcipher *child; }; @@ -66,6 +68,7 @@ struct cryptd_blkcipher_request_ctx { }; struct cryptd_hash_ctx { + atomic_t refcnt; struct crypto_shash *child; }; @@ -75,6 +78,7 @@ struct cryptd_hash_request_ctx { }; struct cryptd_aead_ctx { + atomic_t refcnt; struct crypto_aead *child; }; @@ -118,11 +122,29 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue, { int cpu, err; struct cryptd_cpu_queue *cpu_queue; + struct crypto_tfm *tfm; + atomic_t *refcnt; + bool may_backlog; cpu = get_cpu(); cpu_queue = this_cpu_ptr(queue->cpu_queue); err = crypto_enqueue_request(&cpu_queue->queue, request); + + refcnt = crypto_tfm_ctx(request->tfm); + may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; + + if (err == -EBUSY && !may_backlog) + goto out_put_cpu; + queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); + + if (!atomic_read(refcnt)) + goto out_put_cpu; + + tfm = request->tfm; + atomic_inc(refcnt); + +out_put_cpu: put_cpu(); return err; @@ -206,7 +228,10 @@ static void cryptd_blkcipher_crypt(struct ablkcipher_request *req, unsigned int len)) { struct cryptd_blkcipher_request_ctx *rctx; + struct cryptd_blkcipher_ctx *ctx; + struct crypto_ablkcipher *tfm; struct blkcipher_desc desc; + int refcnt; rctx = ablkcipher_request_ctx(req); @@ -222,9 +247,16 @@ static void cryptd_blkcipher_crypt(struct ablkcipher_request *req, req->base.complete = rctx->complete; out: + tfm = crypto_ablkcipher_reqtfm(req); + ctx = crypto_ablkcipher_ctx(tfm); + refcnt = atomic_read(&ctx->refcnt); + local_bh_disable(); rctx->complete(&req->base, err); local_bh_enable(); + + if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt)) + crypto_free_ablkcipher(tfm); } static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err) @@ -456,6 +488,21 @@ static int cryptd_hash_enqueue(struct ahash_request *req, return cryptd_enqueue_request(queue, &req->base); } +static void cryptd_hash_complete(struct ahash_request *req, int err) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + int refcnt = atomic_read(&ctx->refcnt); + + local_bh_disable(); + rctx->complete(&req->base, err); + local_bh_enable(); + + if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt)) + crypto_free_ahash(tfm); +} + static void cryptd_hash_init(struct crypto_async_request *req_async, int err) { struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); @@ -475,9 +522,7 @@ static void cryptd_hash_init(struct crypto_async_request *req_async, int err) req->base.complete = rctx->complete; out: - local_bh_disable(); - rctx->complete(&req->base, err); - local_bh_enable(); + cryptd_hash_complete(req, err); } static int cryptd_hash_init_enqueue(struct ahash_request *req) @@ -500,9 +545,7 @@ static void cryptd_hash_update(struct crypto_async_request *req_async, int err) req->base.complete = rctx->complete; out: - local_bh_disable(); - rctx->complete(&req->base, err); - local_bh_enable(); + cryptd_hash_complete(req, err); } static int cryptd_hash_update_enqueue(struct ahash_request *req) @@ -523,9 +566,7 @@ static void cryptd_hash_final(struct crypto_async_request *req_async, int err) req->base.complete = rctx->complete; out: - local_bh_disable(); - rctx->complete(&req->base, err); - local_bh_enable(); + cryptd_hash_complete(req, err); } static int cryptd_hash_final_enqueue(struct ahash_request *req) @@ -546,9 +587,7 @@ static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) req->base.complete = rctx->complete; out: - local_bh_disable(); - rctx->complete(&req->base, err); - local_bh_enable(); + cryptd_hash_complete(req, err); } static int cryptd_hash_finup_enqueue(struct ahash_request *req) @@ -575,9 +614,7 @@ static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) req->base.complete = rctx->complete; out: - local_bh_disable(); - rctx->complete(&req->base, err); - local_bh_enable(); + cryptd_hash_complete(req, err); } static int cryptd_hash_digest_enqueue(struct ahash_request *req) @@ -688,7 +725,10 @@ static void cryptd_aead_crypt(struct aead_request *req, int (*crypt)(struct aead_request *req)) { struct cryptd_aead_request_ctx *rctx; + struct cryptd_aead_ctx *ctx; crypto_completion_t compl; + struct crypto_aead *tfm; + int refcnt; rctx = aead_request_ctx(req); compl = rctx->complete; @@ -697,10 +737,18 @@ static void cryptd_aead_crypt(struct aead_request *req, goto out; aead_request_set_tfm(req, child); err = crypt( req ); + out: + tfm = crypto_aead_reqtfm(req); + ctx = crypto_aead_ctx(tfm); + refcnt = atomic_read(&ctx->refcnt); + local_bh_disable(); compl(&req->base, err); local_bh_enable(); + + if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt)) + crypto_free_aead(tfm); } static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) @@ -883,6 +931,7 @@ struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, u32 type, u32 mask) { char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; + struct cryptd_blkcipher_ctx *ctx; struct crypto_tfm *tfm; if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, @@ -899,6 +948,9 @@ struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, return ERR_PTR(-EINVAL); } + ctx = crypto_tfm_ctx(tfm); + atomic_set(&ctx->refcnt, 1); + return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm)); } EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher); @@ -910,9 +962,20 @@ struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm) } EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child); +bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm) +{ + struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); + + return atomic_read(&ctx->refcnt) - 1; +} +EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued); + void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) { - crypto_free_ablkcipher(&tfm->base); + struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); + + if (atomic_dec_and_test(&ctx->refcnt)) + crypto_free_ablkcipher(&tfm->base); } EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); @@ -920,6 +983,7 @@ struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, u32 type, u32 mask) { char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; + struct cryptd_hash_ctx *ctx; struct crypto_ahash *tfm; if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, @@ -933,6 +997,9 @@ struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, return ERR_PTR(-EINVAL); } + ctx = crypto_ahash_ctx(tfm); + atomic_set(&ctx->refcnt, 1); + return __cryptd_ahash_cast(tfm); } EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); @@ -952,9 +1019,20 @@ struct shash_desc *cryptd_shash_desc(struct ahash_request *req) } EXPORT_SYMBOL_GPL(cryptd_shash_desc); +bool cryptd_ahash_queued(struct cryptd_ahash *tfm) +{ + struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); + + return atomic_read(&ctx->refcnt) - 1; +} +EXPORT_SYMBOL_GPL(cryptd_ahash_queued); + void cryptd_free_ahash(struct cryptd_ahash *tfm) { - crypto_free_ahash(&tfm->base); + struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); + + if (atomic_dec_and_test(&ctx->refcnt)) + crypto_free_ahash(&tfm->base); } EXPORT_SYMBOL_GPL(cryptd_free_ahash); @@ -962,6 +1040,7 @@ struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, u32 type, u32 mask) { char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; + struct cryptd_aead_ctx *ctx; struct crypto_aead *tfm; if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, @@ -974,6 +1053,10 @@ struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, crypto_free_aead(tfm); return ERR_PTR(-EINVAL); } + + ctx = crypto_aead_ctx(tfm); + atomic_set(&ctx->refcnt, 1); + return __cryptd_aead_cast(tfm); } EXPORT_SYMBOL_GPL(cryptd_alloc_aead); @@ -986,9 +1069,20 @@ struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) } EXPORT_SYMBOL_GPL(cryptd_aead_child); +bool cryptd_aead_queued(struct cryptd_aead *tfm) +{ + struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); + + return atomic_read(&ctx->refcnt) - 1; +} +EXPORT_SYMBOL_GPL(cryptd_aead_queued); + void cryptd_free_aead(struct cryptd_aead *tfm) { - crypto_free_aead(&tfm->base); + struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); + + if (atomic_dec_and_test(&ctx->refcnt)) + crypto_free_aead(&tfm->base); } EXPORT_SYMBOL_GPL(cryptd_free_aead); diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 941c9a434d50..20ff2c746e0b 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -26,7 +26,7 @@ #include static DEFINE_MUTEX(crypto_default_null_skcipher_lock); -static struct crypto_blkcipher *crypto_default_null_skcipher; +static struct crypto_skcipher *crypto_default_null_skcipher; static int crypto_default_null_skcipher_refcnt; static int null_compress(struct crypto_tfm *tfm, const u8 *src, @@ -153,15 +153,16 @@ MODULE_ALIAS_CRYPTO("compress_null"); MODULE_ALIAS_CRYPTO("digest_null"); MODULE_ALIAS_CRYPTO("cipher_null"); -struct crypto_blkcipher *crypto_get_default_null_skcipher(void) +struct crypto_skcipher *crypto_get_default_null_skcipher(void) { - struct crypto_blkcipher *tfm; + struct crypto_skcipher *tfm; mutex_lock(&crypto_default_null_skcipher_lock); tfm = crypto_default_null_skcipher; if (!tfm) { - tfm = crypto_alloc_blkcipher("ecb(cipher_null)", 0, 0); + tfm = crypto_alloc_skcipher("ecb(cipher_null)", + 0, CRYPTO_ALG_ASYNC); if (IS_ERR(tfm)) goto unlock; @@ -181,7 +182,7 @@ void crypto_put_default_null_skcipher(void) { mutex_lock(&crypto_default_null_skcipher_lock); if (!--crypto_default_null_skcipher_refcnt) { - crypto_free_blkcipher(crypto_default_null_skcipher); + crypto_free_skcipher(crypto_default_null_skcipher); crypto_default_null_skcipher = NULL; } mutex_unlock(&crypto_default_null_skcipher_lock); diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c index 43fe85f20d57..1c5705481c69 100644 --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "internal.h" @@ -126,6 +127,21 @@ nla_put_failure: return -EMSGSIZE; } +static int crypto_report_kpp(struct sk_buff *skb, struct crypto_alg *alg) +{ + struct crypto_report_kpp rkpp; + + strncpy(rkpp.type, "kpp", sizeof(rkpp.type)); + + if (nla_put(skb, CRYPTOCFGA_REPORT_KPP, + sizeof(struct crypto_report_kpp), &rkpp)) + goto nla_put_failure; + return 0; + +nla_put_failure: + return -EMSGSIZE; +} + static int crypto_report_one(struct crypto_alg *alg, struct crypto_user_alg *ualg, struct sk_buff *skb) { @@ -176,6 +192,10 @@ static int crypto_report_one(struct crypto_alg *alg, goto nla_put_failure; break; + case CRYPTO_ALG_TYPE_KPP: + if (crypto_report_kpp(skb, alg)) + goto nla_put_failure; + break; } out: @@ -358,32 +378,6 @@ drop_alg: return err; } -static struct crypto_alg *crypto_user_skcipher_alg(const char *name, u32 type, - u32 mask) -{ - int err; - struct crypto_alg *alg; - - type = crypto_skcipher_type(type); - mask = crypto_skcipher_mask(mask); - - for (;;) { - alg = crypto_lookup_skcipher(name, type, mask); - if (!IS_ERR(alg)) - return alg; - - err = PTR_ERR(alg); - if (err != -EAGAIN) - break; - if (fatal_signal_pending(current)) { - err = -EINTR; - break; - } - } - - return ERR_PTR(err); -} - static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh, struct nlattr **attrs) { @@ -416,16 +410,7 @@ static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh, else name = p->cru_name; - switch (p->cru_type & p->cru_mask & CRYPTO_ALG_TYPE_MASK) { - case CRYPTO_ALG_TYPE_GIVCIPHER: - case CRYPTO_ALG_TYPE_BLKCIPHER: - case CRYPTO_ALG_TYPE_ABLKCIPHER: - alg = crypto_user_skcipher_alg(name, p->cru_type, p->cru_mask); - break; - default: - alg = crypto_alg_mod_lookup(name, p->cru_type, p->cru_mask); - } - + alg = crypto_alg_mod_lookup(name, p->cru_type, p->cru_mask); if (IS_ERR(alg)) return PTR_ERR(alg); @@ -455,6 +440,7 @@ static const int crypto_msg_min[CRYPTO_NR_MSGTYPES] = { [CRYPTO_MSG_NEWALG - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg), [CRYPTO_MSG_DELALG - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg), [CRYPTO_MSG_UPDATEALG - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg), + [CRYPTO_MSG_GETALG - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg), [CRYPTO_MSG_DELRNG - CRYPTO_MSG_BASE] = 0, }; diff --git a/crypto/ctr.c b/crypto/ctr.c index 2386f7313952..ff4d21eddb83 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -26,13 +26,13 @@ struct crypto_ctr_ctx { }; struct crypto_rfc3686_ctx { - struct crypto_ablkcipher *child; + struct crypto_skcipher *child; u8 nonce[CTR_RFC3686_NONCE_SIZE]; }; struct crypto_rfc3686_req_ctx { u8 iv[CTR_RFC3686_BLOCK_SIZE]; - struct ablkcipher_request subreq CRYPTO_MINALIGN_ATTR; + struct skcipher_request subreq CRYPTO_MINALIGN_ATTR; }; static int crypto_ctr_setkey(struct crypto_tfm *parent, const u8 *key, @@ -249,11 +249,11 @@ static struct crypto_template crypto_ctr_tmpl = { .module = THIS_MODULE, }; -static int crypto_rfc3686_setkey(struct crypto_ablkcipher *parent, +static int crypto_rfc3686_setkey(struct crypto_skcipher *parent, const u8 *key, unsigned int keylen) { - struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(parent); - struct crypto_ablkcipher *child = ctx->child; + struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(parent); + struct crypto_skcipher *child = ctx->child; int err; /* the nonce is stored in bytes at end of key */ @@ -265,173 +265,178 @@ static int crypto_rfc3686_setkey(struct crypto_ablkcipher *parent, keylen -= CTR_RFC3686_NONCE_SIZE; - crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) & - CRYPTO_TFM_REQ_MASK); - err = crypto_ablkcipher_setkey(child, key, keylen); - crypto_ablkcipher_set_flags(parent, crypto_ablkcipher_get_flags(child) & - CRYPTO_TFM_RES_MASK); + crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(child, key, keylen); + crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) & + CRYPTO_TFM_RES_MASK); return err; } -static int crypto_rfc3686_crypt(struct ablkcipher_request *req) +static int crypto_rfc3686_crypt(struct skcipher_request *req) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(tfm); - struct crypto_ablkcipher *child = ctx->child; - unsigned long align = crypto_ablkcipher_alignmask(tfm); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *child = ctx->child; + unsigned long align = crypto_skcipher_alignmask(tfm); struct crypto_rfc3686_req_ctx *rctx = - (void *)PTR_ALIGN((u8 *)ablkcipher_request_ctx(req), align + 1); - struct ablkcipher_request *subreq = &rctx->subreq; + (void *)PTR_ALIGN((u8 *)skcipher_request_ctx(req), align + 1); + struct skcipher_request *subreq = &rctx->subreq; u8 *iv = rctx->iv; /* set up counter block */ memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE); - memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->info, CTR_RFC3686_IV_SIZE); + memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->iv, CTR_RFC3686_IV_SIZE); /* initialize counter portion of counter block */ *(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); - ablkcipher_request_set_tfm(subreq, child); - ablkcipher_request_set_callback(subreq, req->base.flags, - req->base.complete, req->base.data); - ablkcipher_request_set_crypt(subreq, req->src, req->dst, req->nbytes, - iv); + skcipher_request_set_tfm(subreq, child); + skcipher_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, iv); - return crypto_ablkcipher_encrypt(subreq); + return crypto_skcipher_encrypt(subreq); } -static int crypto_rfc3686_init_tfm(struct crypto_tfm *tfm) +static int crypto_rfc3686_init_tfm(struct crypto_skcipher *tfm) { - struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct crypto_skcipher_spawn *spawn = crypto_instance_ctx(inst); - struct crypto_rfc3686_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_ablkcipher *cipher; + struct skcipher_instance *inst = skcipher_alg_instance(tfm); + struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst); + struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *cipher; unsigned long align; + unsigned int reqsize; - cipher = crypto_spawn_skcipher(spawn); + cipher = crypto_spawn_skcipher2(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); ctx->child = cipher; - align = crypto_tfm_alg_alignmask(tfm); + align = crypto_skcipher_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); - tfm->crt_ablkcipher.reqsize = align + - sizeof(struct crypto_rfc3686_req_ctx) + - crypto_ablkcipher_reqsize(cipher); + reqsize = align + sizeof(struct crypto_rfc3686_req_ctx) + + crypto_skcipher_reqsize(cipher); + crypto_skcipher_set_reqsize(tfm, reqsize); return 0; } -static void crypto_rfc3686_exit_tfm(struct crypto_tfm *tfm) +static void crypto_rfc3686_exit_tfm(struct crypto_skcipher *tfm) { - struct crypto_rfc3686_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->child); +} - crypto_free_ablkcipher(ctx->child); +static void crypto_rfc3686_free(struct skcipher_instance *inst) +{ + struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst); + + crypto_drop_skcipher(spawn); + kfree(inst); } -static struct crypto_instance *crypto_rfc3686_alloc(struct rtattr **tb) +static int crypto_rfc3686_create(struct crypto_template *tmpl, + struct rtattr **tb) { struct crypto_attr_type *algt; - struct crypto_instance *inst; - struct crypto_alg *alg; + struct skcipher_instance *inst; + struct skcipher_alg *alg; struct crypto_skcipher_spawn *spawn; const char *cipher_name; int err; algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return ERR_CAST(algt); + return PTR_ERR(algt); - if ((algt->type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & algt->mask) - return ERR_PTR(-EINVAL); + if ((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask) + return -EINVAL; cipher_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(cipher_name)) - return ERR_CAST(cipher_name); + return PTR_ERR(cipher_name); inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); if (!inst) - return ERR_PTR(-ENOMEM); + return -ENOMEM; - spawn = crypto_instance_ctx(inst); + spawn = skcipher_instance_ctx(inst); - crypto_set_skcipher_spawn(spawn, inst); - err = crypto_grab_skcipher(spawn, cipher_name, 0, - crypto_requires_sync(algt->type, - algt->mask)); + crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst)); + err = crypto_grab_skcipher2(spawn, cipher_name, 0, + crypto_requires_sync(algt->type, + algt->mask)); if (err) goto err_free_inst; - alg = crypto_skcipher_spawn_alg(spawn); + alg = crypto_spawn_skcipher_alg(spawn); /* We only support 16-byte blocks. */ err = -EINVAL; - if (alg->cra_ablkcipher.ivsize != CTR_RFC3686_BLOCK_SIZE) + if (crypto_skcipher_alg_ivsize(alg) != CTR_RFC3686_BLOCK_SIZE) goto err_drop_spawn; /* Not a stream cipher? */ - if (alg->cra_blocksize != 1) + if (alg->base.cra_blocksize != 1) goto err_drop_spawn; err = -ENAMETOOLONG; - if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "rfc3686(%s)", - alg->cra_name) >= CRYPTO_MAX_ALG_NAME) + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "rfc3686(%s)", alg->base.cra_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_spawn; - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "rfc3686(%s)", alg->cra_driver_name) >= - CRYPTO_MAX_ALG_NAME) + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "rfc3686(%s)", alg->base.cra_driver_name) >= + CRYPTO_MAX_ALG_NAME) goto err_drop_spawn; - inst->alg.cra_priority = alg->cra_priority; - inst->alg.cra_blocksize = 1; - inst->alg.cra_alignmask = alg->cra_alignmask; + inst->alg.base.cra_priority = alg->base.cra_priority; + inst->alg.base.cra_blocksize = 1; + inst->alg.base.cra_alignmask = alg->base.cra_alignmask; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - (alg->cra_flags & CRYPTO_ALG_ASYNC); - inst->alg.cra_type = &crypto_ablkcipher_type; + inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.cra_ablkcipher.ivsize = CTR_RFC3686_IV_SIZE; - inst->alg.cra_ablkcipher.min_keysize = - alg->cra_ablkcipher.min_keysize + CTR_RFC3686_NONCE_SIZE; - inst->alg.cra_ablkcipher.max_keysize = - alg->cra_ablkcipher.max_keysize + CTR_RFC3686_NONCE_SIZE; + inst->alg.ivsize = CTR_RFC3686_IV_SIZE; + inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg); + inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) + + CTR_RFC3686_NONCE_SIZE; + inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg) + + CTR_RFC3686_NONCE_SIZE; - inst->alg.cra_ablkcipher.geniv = "seqiv"; + inst->alg.setkey = crypto_rfc3686_setkey; + inst->alg.encrypt = crypto_rfc3686_crypt; + inst->alg.decrypt = crypto_rfc3686_crypt; - inst->alg.cra_ablkcipher.setkey = crypto_rfc3686_setkey; - inst->alg.cra_ablkcipher.encrypt = crypto_rfc3686_crypt; - inst->alg.cra_ablkcipher.decrypt = crypto_rfc3686_crypt; + inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc3686_ctx); - inst->alg.cra_ctxsize = sizeof(struct crypto_rfc3686_ctx); + inst->alg.init = crypto_rfc3686_init_tfm; + inst->alg.exit = crypto_rfc3686_exit_tfm; - inst->alg.cra_init = crypto_rfc3686_init_tfm; - inst->alg.cra_exit = crypto_rfc3686_exit_tfm; + inst->free = crypto_rfc3686_free; - return inst; + err = skcipher_register_instance(tmpl, inst); + if (err) + goto err_drop_spawn; + +out: + return err; err_drop_spawn: crypto_drop_skcipher(spawn); err_free_inst: kfree(inst); - return ERR_PTR(err); -} - -static void crypto_rfc3686_free(struct crypto_instance *inst) -{ - struct crypto_skcipher_spawn *spawn = crypto_instance_ctx(inst); - - crypto_drop_skcipher(spawn); - kfree(inst); + goto out; } static struct crypto_template crypto_rfc3686_tmpl = { .name = "rfc3686", - .alloc = crypto_rfc3686_alloc, - .free = crypto_rfc3686_free, + .create = crypto_rfc3686_create, .module = THIS_MODULE, }; diff --git a/crypto/cts.c b/crypto/cts.c index e467ec0acf9f..51976187b2bf 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -40,7 +40,7 @@ * rfc3962 includes errata information in its Appendix A. */ -#include +#include #include #include #include @@ -51,289 +51,364 @@ #include struct crypto_cts_ctx { - struct crypto_blkcipher *child; + struct crypto_skcipher *child; }; -static int crypto_cts_setkey(struct crypto_tfm *parent, const u8 *key, - unsigned int keylen) +struct crypto_cts_reqctx { + struct scatterlist sg[2]; + unsigned offset; + struct skcipher_request subreq; +}; + +static inline u8 *crypto_cts_reqctx_space(struct skcipher_request *req) { - struct crypto_cts_ctx *ctx = crypto_tfm_ctx(parent); - struct crypto_blkcipher *child = ctx->child; - int err; + struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *child = ctx->child; - crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_blkcipher_set_flags(child, crypto_tfm_get_flags(parent) & - CRYPTO_TFM_REQ_MASK); - err = crypto_blkcipher_setkey(child, key, keylen); - crypto_tfm_set_flags(parent, crypto_blkcipher_get_flags(child) & - CRYPTO_TFM_RES_MASK); - return err; + return PTR_ALIGN((u8 *)(rctx + 1) + crypto_skcipher_reqsize(child), + crypto_skcipher_alignmask(tfm) + 1); } -static int cts_cbc_encrypt(struct crypto_cts_ctx *ctx, - struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int offset, - unsigned int nbytes) +static int crypto_cts_setkey(struct crypto_skcipher *parent, const u8 *key, + unsigned int keylen) { - int bsize = crypto_blkcipher_blocksize(desc->tfm); - u8 tmp[bsize], tmp2[bsize]; - struct blkcipher_desc lcldesc; - struct scatterlist sgsrc[1], sgdst[1]; - int lastn = nbytes - bsize; - u8 iv[bsize]; - u8 s[bsize * 2], d[bsize * 2]; + struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(parent); + struct crypto_skcipher *child = ctx->child; int err; - if (lastn < 0) - return -EINVAL; + crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(child, key, keylen); + crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) & + CRYPTO_TFM_RES_MASK); + return err; +} - sg_init_table(sgsrc, 1); - sg_init_table(sgdst, 1); +static void cts_cbc_crypt_done(struct crypto_async_request *areq, int err) +{ + struct skcipher_request *req = areq->data; - memset(s, 0, sizeof(s)); - scatterwalk_map_and_copy(s, src, offset, nbytes, 0); + if (err == -EINPROGRESS) + return; - memcpy(iv, desc->info, bsize); + skcipher_request_complete(req, err); +} - lcldesc.tfm = ctx->child; - lcldesc.info = iv; - lcldesc.flags = desc->flags; +static int cts_cbc_encrypt(struct skcipher_request *req) +{ + struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct skcipher_request *subreq = &rctx->subreq; + int bsize = crypto_skcipher_blocksize(tfm); + u8 d[bsize * 2] __attribute__ ((aligned(__alignof__(u32)))); + struct scatterlist *sg; + unsigned int offset; + int lastn; + + offset = rctx->offset; + lastn = req->cryptlen - offset; + + sg = scatterwalk_ffwd(rctx->sg, req->dst, offset - bsize); + scatterwalk_map_and_copy(d + bsize, sg, 0, bsize, 0); + + memset(d, 0, bsize); + scatterwalk_map_and_copy(d, req->src, offset, lastn, 0); + + scatterwalk_map_and_copy(d, sg, 0, bsize + lastn, 1); + memzero_explicit(d, sizeof(d)); + + skcipher_request_set_callback(subreq, req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG, + cts_cbc_crypt_done, req); + skcipher_request_set_crypt(subreq, sg, sg, bsize, req->iv); + return crypto_skcipher_encrypt(subreq); +} - sg_set_buf(&sgsrc[0], s, bsize); - sg_set_buf(&sgdst[0], tmp, bsize); - err = crypto_blkcipher_encrypt_iv(&lcldesc, sgdst, sgsrc, bsize); +static void crypto_cts_encrypt_done(struct crypto_async_request *areq, int err) +{ + struct skcipher_request *req = areq->data; - memcpy(d + bsize, tmp, lastn); + if (err) + goto out; - lcldesc.info = tmp; + err = cts_cbc_encrypt(req); + if (err == -EINPROGRESS || + (err == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return; - sg_set_buf(&sgsrc[0], s + bsize, bsize); - sg_set_buf(&sgdst[0], tmp2, bsize); - err = crypto_blkcipher_encrypt_iv(&lcldesc, sgdst, sgsrc, bsize); +out: + skcipher_request_complete(req, err); +} - memcpy(d, tmp2, bsize); +static int crypto_cts_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_request *subreq = &rctx->subreq; + int bsize = crypto_skcipher_blocksize(tfm); + unsigned int nbytes = req->cryptlen; + int cbc_blocks = (nbytes + bsize - 1) / bsize - 1; + unsigned int offset; + + skcipher_request_set_tfm(subreq, ctx->child); + + if (cbc_blocks <= 0) { + skcipher_request_set_callback(subreq, req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, nbytes, + req->iv); + return crypto_skcipher_encrypt(subreq); + } - scatterwalk_map_and_copy(d, dst, offset, nbytes, 1); + offset = cbc_blocks * bsize; + rctx->offset = offset; - memcpy(desc->info, tmp2, bsize); + skcipher_request_set_callback(subreq, req->base.flags, + crypto_cts_encrypt_done, req); + skcipher_request_set_crypt(subreq, req->src, req->dst, + offset, req->iv); - return err; + return crypto_skcipher_encrypt(subreq) ?: + cts_cbc_encrypt(req); } -static int crypto_cts_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int cts_cbc_decrypt(struct skcipher_request *req) { - struct crypto_cts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - int bsize = crypto_blkcipher_blocksize(desc->tfm); - int tot_blocks = (nbytes + bsize - 1) / bsize; - int cbc_blocks = tot_blocks > 2 ? tot_blocks - 2 : 0; - struct blkcipher_desc lcldesc; - int err; + struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct skcipher_request *subreq = &rctx->subreq; + int bsize = crypto_skcipher_blocksize(tfm); + u8 d[bsize * 2] __attribute__ ((aligned(__alignof__(u32)))); + struct scatterlist *sg; + unsigned int offset; + u8 *space; + int lastn; + + offset = rctx->offset; + lastn = req->cryptlen - offset; + + sg = scatterwalk_ffwd(rctx->sg, req->dst, offset - bsize); + + /* 1. Decrypt Cn-1 (s) to create Dn */ + scatterwalk_map_and_copy(d + bsize, sg, 0, bsize, 0); + space = crypto_cts_reqctx_space(req); + crypto_xor(d + bsize, space, bsize); + /* 2. Pad Cn with zeros at the end to create C of length BB */ + memset(d, 0, bsize); + scatterwalk_map_and_copy(d, req->src, offset, lastn, 0); + /* 3. Exclusive-or Dn with C to create Xn */ + /* 4. Select the first Ln bytes of Xn to create Pn */ + crypto_xor(d + bsize, d, lastn); + + /* 5. Append the tail (BB - Ln) bytes of Xn to Cn to create En */ + memcpy(d + lastn, d + bsize + lastn, bsize - lastn); + /* 6. Decrypt En to create Pn-1 */ - lcldesc.tfm = ctx->child; - lcldesc.info = desc->info; - lcldesc.flags = desc->flags; - - if (tot_blocks == 1) { - err = crypto_blkcipher_encrypt_iv(&lcldesc, dst, src, bsize); - } else if (nbytes <= bsize * 2) { - err = cts_cbc_encrypt(ctx, desc, dst, src, 0, nbytes); - } else { - /* do normal function for tot_blocks - 2 */ - err = crypto_blkcipher_encrypt_iv(&lcldesc, dst, src, - cbc_blocks * bsize); - if (err == 0) { - /* do cts for final two blocks */ - err = cts_cbc_encrypt(ctx, desc, dst, src, - cbc_blocks * bsize, - nbytes - (cbc_blocks * bsize)); - } - } + scatterwalk_map_and_copy(d, sg, 0, bsize + lastn, 1); + memzero_explicit(d, sizeof(d)); - return err; + skcipher_request_set_callback(subreq, req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG, + cts_cbc_crypt_done, req); + + skcipher_request_set_crypt(subreq, sg, sg, bsize, space); + return crypto_skcipher_decrypt(subreq); } -static int cts_cbc_decrypt(struct crypto_cts_ctx *ctx, - struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int offset, - unsigned int nbytes) +static void crypto_cts_decrypt_done(struct crypto_async_request *areq, int err) { - int bsize = crypto_blkcipher_blocksize(desc->tfm); - u8 tmp[bsize]; - struct blkcipher_desc lcldesc; - struct scatterlist sgsrc[1], sgdst[1]; - int lastn = nbytes - bsize; - u8 iv[bsize]; - u8 s[bsize * 2], d[bsize * 2]; - int err; - - if (lastn < 0) - return -EINVAL; + struct skcipher_request *req = areq->data; - sg_init_table(sgsrc, 1); - sg_init_table(sgdst, 1); + if (err) + goto out; - scatterwalk_map_and_copy(s, src, offset, nbytes, 0); + err = cts_cbc_decrypt(req); + if (err == -EINPROGRESS || + (err == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return; - lcldesc.tfm = ctx->child; - lcldesc.info = iv; - lcldesc.flags = desc->flags; +out: + skcipher_request_complete(req, err); +} - /* 1. Decrypt Cn-1 (s) to create Dn (tmp)*/ - memset(iv, 0, sizeof(iv)); - sg_set_buf(&sgsrc[0], s, bsize); - sg_set_buf(&sgdst[0], tmp, bsize); - err = crypto_blkcipher_decrypt_iv(&lcldesc, sgdst, sgsrc, bsize); - if (err) - return err; - /* 2. Pad Cn with zeros at the end to create C of length BB */ - memset(iv, 0, sizeof(iv)); - memcpy(iv, s + bsize, lastn); - /* 3. Exclusive-or Dn (tmp) with C (iv) to create Xn (tmp) */ - crypto_xor(tmp, iv, bsize); - /* 4. Select the first Ln bytes of Xn (tmp) to create Pn */ - memcpy(d + bsize, tmp, lastn); - - /* 5. Append the tail (BB - Ln) bytes of Xn (tmp) to Cn to create En */ - memcpy(s + bsize + lastn, tmp + lastn, bsize - lastn); - /* 6. Decrypt En to create Pn-1 */ - memzero_explicit(iv, sizeof(iv)); +static int crypto_cts_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_cts_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_request *subreq = &rctx->subreq; + int bsize = crypto_skcipher_blocksize(tfm); + unsigned int nbytes = req->cryptlen; + int cbc_blocks = (nbytes + bsize - 1) / bsize - 1; + unsigned int offset; + u8 *space; + + skcipher_request_set_tfm(subreq, ctx->child); + + if (cbc_blocks <= 0) { + skcipher_request_set_callback(subreq, req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, nbytes, + req->iv); + return crypto_skcipher_decrypt(subreq); + } - sg_set_buf(&sgsrc[0], s + bsize, bsize); - sg_set_buf(&sgdst[0], d, bsize); - err = crypto_blkcipher_decrypt_iv(&lcldesc, sgdst, sgsrc, bsize); + skcipher_request_set_callback(subreq, req->base.flags, + crypto_cts_decrypt_done, req); - /* XOR with previous block */ - crypto_xor(d, desc->info, bsize); + space = crypto_cts_reqctx_space(req); - scatterwalk_map_and_copy(d, dst, offset, nbytes, 1); + offset = cbc_blocks * bsize; + rctx->offset = offset; - memcpy(desc->info, s, bsize); - return err; -} + if (cbc_blocks <= 1) + memcpy(space, req->iv, bsize); + else + scatterwalk_map_and_copy(space, req->src, offset - 2 * bsize, + bsize, 0); -static int crypto_cts_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - struct crypto_cts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - int bsize = crypto_blkcipher_blocksize(desc->tfm); - int tot_blocks = (nbytes + bsize - 1) / bsize; - int cbc_blocks = tot_blocks > 2 ? tot_blocks - 2 : 0; - struct blkcipher_desc lcldesc; - int err; + skcipher_request_set_crypt(subreq, req->src, req->dst, + offset, req->iv); - lcldesc.tfm = ctx->child; - lcldesc.info = desc->info; - lcldesc.flags = desc->flags; - - if (tot_blocks == 1) { - err = crypto_blkcipher_decrypt_iv(&lcldesc, dst, src, bsize); - } else if (nbytes <= bsize * 2) { - err = cts_cbc_decrypt(ctx, desc, dst, src, 0, nbytes); - } else { - /* do normal function for tot_blocks - 2 */ - err = crypto_blkcipher_decrypt_iv(&lcldesc, dst, src, - cbc_blocks * bsize); - if (err == 0) { - /* do cts for final two blocks */ - err = cts_cbc_decrypt(ctx, desc, dst, src, - cbc_blocks * bsize, - nbytes - (cbc_blocks * bsize)); - } - } - return err; + return crypto_skcipher_decrypt(subreq) ?: + cts_cbc_decrypt(req); } -static int crypto_cts_init_tfm(struct crypto_tfm *tfm) +static int crypto_cts_init_tfm(struct crypto_skcipher *tfm) { - struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct crypto_spawn *spawn = crypto_instance_ctx(inst); - struct crypto_cts_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_blkcipher *cipher; - - cipher = crypto_spawn_blkcipher(spawn); + struct skcipher_instance *inst = skcipher_alg_instance(tfm); + struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst); + struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *cipher; + unsigned reqsize; + unsigned bsize; + unsigned align; + + cipher = crypto_spawn_skcipher2(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); ctx->child = cipher; + + align = crypto_skcipher_alignmask(tfm); + bsize = crypto_skcipher_blocksize(cipher); + reqsize = ALIGN(sizeof(struct crypto_cts_reqctx) + + crypto_skcipher_reqsize(cipher), + crypto_tfm_ctx_alignment()) + + (align & ~(crypto_tfm_ctx_alignment() - 1)) + bsize; + + crypto_skcipher_set_reqsize(tfm, reqsize); + return 0; } -static void crypto_cts_exit_tfm(struct crypto_tfm *tfm) +static void crypto_cts_exit_tfm(struct crypto_skcipher *tfm) { - struct crypto_cts_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_blkcipher(ctx->child); + struct crypto_cts_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->child); } -static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb) +static void crypto_cts_free(struct skcipher_instance *inst) { - struct crypto_instance *inst; - struct crypto_alg *alg; + crypto_drop_skcipher(skcipher_instance_ctx(inst)); + kfree(inst); +} + +static int crypto_cts_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct crypto_skcipher_spawn *spawn; + struct skcipher_instance *inst; + struct crypto_attr_type *algt; + struct skcipher_alg *alg; + const char *cipher_name; int err; - err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER); + algt = crypto_get_attr_type(tb); + if (IS_ERR(algt)) + return PTR_ERR(algt); + + if ((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask) + return -EINVAL; + + cipher_name = crypto_attr_alg_name(tb[1]); + if (IS_ERR(cipher_name)) + return PTR_ERR(cipher_name); + + inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); + if (!inst) + return -ENOMEM; + + spawn = skcipher_instance_ctx(inst); + + crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst)); + err = crypto_grab_skcipher2(spawn, cipher_name, 0, + crypto_requires_sync(algt->type, + algt->mask)); if (err) - return ERR_PTR(err); + goto err_free_inst; - alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_BLKCIPHER, - CRYPTO_ALG_TYPE_MASK); - if (IS_ERR(alg)) - return ERR_CAST(alg); + alg = crypto_spawn_skcipher_alg(spawn); - inst = ERR_PTR(-EINVAL); - if (!is_power_of_2(alg->cra_blocksize)) - goto out_put_alg; + err = -EINVAL; + if (crypto_skcipher_alg_ivsize(alg) != alg->base.cra_blocksize) + goto err_drop_spawn; - if (strncmp(alg->cra_name, "cbc(", 4)) - goto out_put_alg; + if (strncmp(alg->base.cra_name, "cbc(", 4)) + goto err_drop_spawn; - inst = crypto_alloc_instance("cts", alg); - if (IS_ERR(inst)) - goto out_put_alg; + err = crypto_inst_setname(skcipher_crypto_instance(inst), "cts", + &alg->base); + if (err) + goto err_drop_spawn; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER; - inst->alg.cra_priority = alg->cra_priority; - inst->alg.cra_blocksize = alg->cra_blocksize; - inst->alg.cra_alignmask = alg->cra_alignmask; - inst->alg.cra_type = &crypto_blkcipher_type; + inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = alg->base.cra_priority; + inst->alg.base.cra_blocksize = alg->base.cra_blocksize; + inst->alg.base.cra_alignmask = alg->base.cra_alignmask; /* We access the data as u32s when xoring. */ - inst->alg.cra_alignmask |= __alignof__(u32) - 1; + inst->alg.base.cra_alignmask |= __alignof__(u32) - 1; - inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize; - inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize; - inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize; + inst->alg.ivsize = alg->base.cra_blocksize; + inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg); + inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg); + inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg); - inst->alg.cra_ctxsize = sizeof(struct crypto_cts_ctx); + inst->alg.base.cra_ctxsize = sizeof(struct crypto_cts_ctx); - inst->alg.cra_init = crypto_cts_init_tfm; - inst->alg.cra_exit = crypto_cts_exit_tfm; + inst->alg.init = crypto_cts_init_tfm; + inst->alg.exit = crypto_cts_exit_tfm; - inst->alg.cra_blkcipher.setkey = crypto_cts_setkey; - inst->alg.cra_blkcipher.encrypt = crypto_cts_encrypt; - inst->alg.cra_blkcipher.decrypt = crypto_cts_decrypt; + inst->alg.setkey = crypto_cts_setkey; + inst->alg.encrypt = crypto_cts_encrypt; + inst->alg.decrypt = crypto_cts_decrypt; -out_put_alg: - crypto_mod_put(alg); - return inst; -} + inst->free = crypto_cts_free; -static void crypto_cts_free(struct crypto_instance *inst) -{ - crypto_drop_spawn(crypto_instance_ctx(inst)); + err = skcipher_register_instance(tmpl, inst); + if (err) + goto err_drop_spawn; + +out: + return err; + +err_drop_spawn: + crypto_drop_skcipher(spawn); +err_free_inst: kfree(inst); + goto out; } static struct crypto_template crypto_cts_tmpl = { .name = "cts", - .alloc = crypto_cts_alloc, - .free = crypto_cts_free, + .create = crypto_cts_create, .module = THIS_MODULE, }; diff --git a/crypto/dh.c b/crypto/dh.c new file mode 100644 index 000000000000..9d19360e7189 --- /dev/null +++ b/crypto/dh.c @@ -0,0 +1,189 @@ +/* Diffie-Hellman Key Agreement Method [RFC2631] + * + * Copyright (c) 2016, Intel Corporation + * Authors: Salvatore Benedetto + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public Licence + * as published by the Free Software Foundation; either version + * 2 of the Licence, or (at your option) any later version. + */ + +#include +#include +#include +#include +#include + +struct dh_ctx { + MPI p; + MPI g; + MPI xa; +}; + +static inline void dh_clear_params(struct dh_ctx *ctx) +{ + mpi_free(ctx->p); + mpi_free(ctx->g); + ctx->p = NULL; + ctx->g = NULL; +} + +static void dh_free_ctx(struct dh_ctx *ctx) +{ + dh_clear_params(ctx); + mpi_free(ctx->xa); + ctx->xa = NULL; +} + +/* + * If base is g we compute the public key + * ya = g^xa mod p; [RFC2631 sec 2.1.1] + * else if base if the counterpart public key we compute the shared secret + * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1] + */ +static int _compute_val(const struct dh_ctx *ctx, MPI base, MPI val) +{ + /* val = base^xa mod p */ + return mpi_powm(val, base, ctx->xa, ctx->p); +} + +static inline struct dh_ctx *dh_get_ctx(struct crypto_kpp *tfm) +{ + return kpp_tfm_ctx(tfm); +} + +static int dh_check_params_length(unsigned int p_len) +{ + return (p_len < 1536) ? -EINVAL : 0; +} + +static int dh_set_params(struct dh_ctx *ctx, struct dh *params) +{ + if (unlikely(!params->p || !params->g)) + return -EINVAL; + + if (dh_check_params_length(params->p_size << 3)) + return -EINVAL; + + ctx->p = mpi_read_raw_data(params->p, params->p_size); + if (!ctx->p) + return -EINVAL; + + ctx->g = mpi_read_raw_data(params->g, params->g_size); + if (!ctx->g) { + mpi_free(ctx->p); + return -EINVAL; + } + + return 0; +} + +static int dh_set_secret(struct crypto_kpp *tfm, void *buf, unsigned int len) +{ + struct dh_ctx *ctx = dh_get_ctx(tfm); + struct dh params; + + if (crypto_dh_decode_key(buf, len, ¶ms) < 0) + return -EINVAL; + + if (dh_set_params(ctx, ¶ms) < 0) + return -EINVAL; + + ctx->xa = mpi_read_raw_data(params.key, params.key_size); + if (!ctx->xa) { + dh_clear_params(ctx); + return -EINVAL; + } + + return 0; +} + +static int dh_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct dh_ctx *ctx = dh_get_ctx(tfm); + MPI base, val = mpi_alloc(0); + int ret = 0; + int sign; + + if (!val) + return -ENOMEM; + + if (unlikely(!ctx->xa)) { + ret = -EINVAL; + goto err_free_val; + } + + if (req->src) { + base = mpi_read_raw_from_sgl(req->src, req->src_len); + if (!base) { + ret = EINVAL; + goto err_free_val; + } + } else { + base = ctx->g; + } + + ret = _compute_val(ctx, base, val); + if (ret) + goto err_free_base; + + ret = mpi_write_to_sgl(val, req->dst, req->dst_len, &sign); + if (ret) + goto err_free_base; + + if (sign < 0) + ret = -EBADMSG; +err_free_base: + if (req->src) + mpi_free(base); +err_free_val: + mpi_free(val); + return ret; +} + +static int dh_max_size(struct crypto_kpp *tfm) +{ + struct dh_ctx *ctx = dh_get_ctx(tfm); + + return mpi_get_size(ctx->p); +} + +static void dh_exit_tfm(struct crypto_kpp *tfm) +{ + struct dh_ctx *ctx = dh_get_ctx(tfm); + + dh_free_ctx(ctx); +} + +static struct kpp_alg dh = { + .set_secret = dh_set_secret, + .generate_public_key = dh_compute_value, + .compute_shared_secret = dh_compute_value, + .max_size = dh_max_size, + .exit = dh_exit_tfm, + .base = { + .cra_name = "dh", + .cra_driver_name = "dh-generic", + .cra_priority = 100, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct dh_ctx), + }, +}; + +static int dh_init(void) +{ + return crypto_register_kpp(&dh); +} + +static void dh_exit(void) +{ + crypto_unregister_kpp(&dh); +} + +module_init(dh_init); +module_exit(dh_exit); +MODULE_ALIAS_CRYPTO("dh"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("DH generic algorithm"); diff --git a/crypto/dh_helper.c b/crypto/dh_helper.c new file mode 100644 index 000000000000..02db76b20d00 --- /dev/null +++ b/crypto/dh_helper.c @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2016, Intel Corporation + * Authors: Salvatore Benedetto + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public Licence + * as published by the Free Software Foundation; either version + * 2 of the Licence, or (at your option) any later version. + */ +#include +#include +#include +#include +#include +#include + +#define DH_KPP_SECRET_MIN_SIZE (sizeof(struct kpp_secret) + 3 * sizeof(int)) + +static inline u8 *dh_pack_data(void *dst, const void *src, size_t size) +{ + memcpy(dst, src, size); + return dst + size; +} + +static inline const u8 *dh_unpack_data(void *dst, const void *src, size_t size) +{ + memcpy(dst, src, size); + return src + size; +} + +static inline int dh_data_size(const struct dh *p) +{ + return p->key_size + p->p_size + p->g_size; +} + +int crypto_dh_key_len(const struct dh *p) +{ + return DH_KPP_SECRET_MIN_SIZE + dh_data_size(p); +} +EXPORT_SYMBOL_GPL(crypto_dh_key_len); + +int crypto_dh_encode_key(char *buf, unsigned int len, const struct dh *params) +{ + u8 *ptr = buf; + struct kpp_secret secret = { + .type = CRYPTO_KPP_SECRET_TYPE_DH, + .len = len + }; + + if (unlikely(!buf)) + return -EINVAL; + + if (len != crypto_dh_key_len(params)) + return -EINVAL; + + ptr = dh_pack_data(ptr, &secret, sizeof(secret)); + ptr = dh_pack_data(ptr, ¶ms->key_size, sizeof(params->key_size)); + ptr = dh_pack_data(ptr, ¶ms->p_size, sizeof(params->p_size)); + ptr = dh_pack_data(ptr, ¶ms->g_size, sizeof(params->g_size)); + ptr = dh_pack_data(ptr, params->key, params->key_size); + ptr = dh_pack_data(ptr, params->p, params->p_size); + dh_pack_data(ptr, params->g, params->g_size); + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_dh_encode_key); + +int crypto_dh_decode_key(const char *buf, unsigned int len, struct dh *params) +{ + const u8 *ptr = buf; + struct kpp_secret secret; + + if (unlikely(!buf || len < DH_KPP_SECRET_MIN_SIZE)) + return -EINVAL; + + ptr = dh_unpack_data(&secret, ptr, sizeof(secret)); + if (secret.type != CRYPTO_KPP_SECRET_TYPE_DH) + return -EINVAL; + + ptr = dh_unpack_data(¶ms->key_size, ptr, sizeof(params->key_size)); + ptr = dh_unpack_data(¶ms->p_size, ptr, sizeof(params->p_size)); + ptr = dh_unpack_data(¶ms->g_size, ptr, sizeof(params->g_size)); + if (secret.len != crypto_dh_key_len(params)) + return -EINVAL; + + /* Don't allocate memory. Set pointers to data within + * the given buffer + */ + params->key = (void *)ptr; + params->p = (void *)(ptr + params->key_size); + params->g = (void *)(ptr + params->key_size + params->p_size); + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_dh_decode_key); diff --git a/crypto/drbg.c b/crypto/drbg.c index 0a3538f6cf22..f752da3a7c75 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -252,10 +252,16 @@ MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes192"); MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes128"); MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes128"); -static int drbg_kcapi_sym(struct drbg_state *drbg, const unsigned char *key, - unsigned char *outval, const struct drbg_string *in); +static void drbg_kcapi_symsetkey(struct drbg_state *drbg, + const unsigned char *key); +static int drbg_kcapi_sym(struct drbg_state *drbg, unsigned char *outval, + const struct drbg_string *in); static int drbg_init_sym_kernel(struct drbg_state *drbg); static int drbg_fini_sym_kernel(struct drbg_state *drbg); +static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, + u8 *inbuf, u32 inbuflen, + u8 *outbuf, u32 outlen); +#define DRBG_CTR_NULL_LEN 128 /* BCC function for CTR DRBG as defined in 10.4.3 */ static int drbg_ctr_bcc(struct drbg_state *drbg, @@ -270,6 +276,7 @@ static int drbg_ctr_bcc(struct drbg_state *drbg, drbg_string_fill(&data, out, drbg_blocklen(drbg)); /* 10.4.3 step 2 / 4 */ + drbg_kcapi_symsetkey(drbg, key); list_for_each_entry(curr, in, list) { const unsigned char *pos = curr->buf; size_t len = curr->len; @@ -278,7 +285,7 @@ static int drbg_ctr_bcc(struct drbg_state *drbg, /* 10.4.3 step 4.2 */ if (drbg_blocklen(drbg) == cnt) { cnt = 0; - ret = drbg_kcapi_sym(drbg, key, out, &data); + ret = drbg_kcapi_sym(drbg, out, &data); if (ret) return ret; } @@ -290,7 +297,7 @@ static int drbg_ctr_bcc(struct drbg_state *drbg, } /* 10.4.3 step 4.2 for last block */ if (cnt) - ret = drbg_kcapi_sym(drbg, key, out, &data); + ret = drbg_kcapi_sym(drbg, out, &data); return ret; } @@ -425,6 +432,7 @@ static int drbg_ctr_df(struct drbg_state *drbg, /* 10.4.2 step 12: overwriting of outval is implemented in next step */ /* 10.4.2 step 13 */ + drbg_kcapi_symsetkey(drbg, temp); while (generated_len < bytes_to_return) { short blocklen = 0; /* @@ -432,7 +440,7 @@ static int drbg_ctr_df(struct drbg_state *drbg, * implicit as the key is only drbg_blocklen in size based on * the implementation of the cipher function callback */ - ret = drbg_kcapi_sym(drbg, temp, X, &cipherin); + ret = drbg_kcapi_sym(drbg, X, &cipherin); if (ret) goto out; blocklen = (drbg_blocklen(drbg) < @@ -476,49 +484,47 @@ static int drbg_ctr_update(struct drbg_state *drbg, struct list_head *seed, unsigned char *temp = drbg->scratchpad; unsigned char *df_data = drbg->scratchpad + drbg_statelen(drbg) + drbg_blocklen(drbg); - unsigned char *temp_p, *df_data_p; /* pointer to iterate over buffers */ - unsigned int len = 0; - struct drbg_string cipherin; if (3 > reseed) memset(df_data, 0, drbg_statelen(drbg)); - /* 10.2.1.3.2 step 2 and 10.2.1.4.2 step 2 */ - if (seed) { - ret = drbg_ctr_df(drbg, df_data, drbg_statelen(drbg), seed); + if (!reseed) { + /* + * The DRBG uses the CTR mode of the underlying AES cipher. The + * CTR mode increments the counter value after the AES operation + * but SP800-90A requires that the counter is incremented before + * the AES operation. Hence, we increment it at the time we set + * it by one. + */ + crypto_inc(drbg->V, drbg_blocklen(drbg)); + + ret = crypto_skcipher_setkey(drbg->ctr_handle, drbg->C, + drbg_keylen(drbg)); if (ret) goto out; } - drbg_string_fill(&cipherin, drbg->V, drbg_blocklen(drbg)); - /* - * 10.2.1.3.2 steps 2 and 3 are already covered as the allocation - * zeroizes all memory during initialization - */ - while (len < (drbg_statelen(drbg))) { - /* 10.2.1.2 step 2.1 */ - crypto_inc(drbg->V, drbg_blocklen(drbg)); - /* - * 10.2.1.2 step 2.2 */ - ret = drbg_kcapi_sym(drbg, drbg->C, temp + len, &cipherin); + /* 10.2.1.3.2 step 2 and 10.2.1.4.2 step 2 */ + if (seed) { + ret = drbg_ctr_df(drbg, df_data, drbg_statelen(drbg), seed); if (ret) goto out; - /* 10.2.1.2 step 2.3 and 3 */ - len += drbg_blocklen(drbg); } - /* 10.2.1.2 step 4 */ - temp_p = temp; - df_data_p = df_data; - for (len = 0; len < drbg_statelen(drbg); len++) { - *temp_p ^= *df_data_p; - df_data_p++; temp_p++; - } + ret = drbg_kcapi_sym_ctr(drbg, df_data, drbg_statelen(drbg), + temp, drbg_statelen(drbg)); + if (ret) + return ret; /* 10.2.1.2 step 5 */ - memcpy(drbg->C, temp, drbg_keylen(drbg)); + ret = crypto_skcipher_setkey(drbg->ctr_handle, temp, + drbg_keylen(drbg)); + if (ret) + goto out; /* 10.2.1.2 step 6 */ memcpy(drbg->V, temp + drbg_keylen(drbg), drbg_blocklen(drbg)); + /* See above: increment counter by one to compensate timing of CTR op */ + crypto_inc(drbg->V, drbg_blocklen(drbg)); ret = 0; out: @@ -537,9 +543,8 @@ static int drbg_ctr_generate(struct drbg_state *drbg, unsigned char *buf, unsigned int buflen, struct list_head *addtl) { - int len = 0; - int ret = 0; - struct drbg_string data; + int ret; + int len = min_t(int, buflen, INT_MAX); /* 10.2.1.5.2 step 2 */ if (addtl && !list_empty(addtl)) { @@ -549,33 +554,16 @@ static int drbg_ctr_generate(struct drbg_state *drbg, } /* 10.2.1.5.2 step 4.1 */ - crypto_inc(drbg->V, drbg_blocklen(drbg)); - drbg_string_fill(&data, drbg->V, drbg_blocklen(drbg)); - while (len < buflen) { - int outlen = 0; - /* 10.2.1.5.2 step 4.2 */ - ret = drbg_kcapi_sym(drbg, drbg->C, drbg->scratchpad, &data); - if (ret) { - len = ret; - goto out; - } - outlen = (drbg_blocklen(drbg) < (buflen - len)) ? - drbg_blocklen(drbg) : (buflen - len); - /* 10.2.1.5.2 step 4.3 */ - memcpy(buf + len, drbg->scratchpad, outlen); - len += outlen; - /* 10.2.1.5.2 step 6 */ - if (len < buflen) - crypto_inc(drbg->V, drbg_blocklen(drbg)); - } + ret = drbg_kcapi_sym_ctr(drbg, drbg->ctr_null_value, DRBG_CTR_NULL_LEN, + buf, len); + if (ret) + return ret; /* 10.2.1.5.2 step 6 */ ret = drbg_ctr_update(drbg, NULL, 3); if (ret) len = ret; -out: - memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); return len; } @@ -1145,11 +1133,11 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) if (!drbg) return; kzfree(drbg->V); - drbg->V = NULL; + drbg->Vbuf = NULL; kzfree(drbg->C); - drbg->C = NULL; - kzfree(drbg->scratchpad); - drbg->scratchpad = NULL; + drbg->Cbuf = NULL; + kzfree(drbg->scratchpadbuf); + drbg->scratchpadbuf = NULL; drbg->reseed_ctr = 0; drbg->d_ops = NULL; drbg->core = NULL; @@ -1185,12 +1173,18 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) goto err; } - drbg->V = kmalloc(drbg_statelen(drbg), GFP_KERNEL); - if (!drbg->V) - goto err; - drbg->C = kmalloc(drbg_statelen(drbg), GFP_KERNEL); - if (!drbg->C) + ret = drbg->d_ops->crypto_init(drbg); + if (ret < 0) goto err; + + drbg->Vbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL); + if (!drbg->Vbuf) + goto fini; + drbg->V = PTR_ALIGN(drbg->Vbuf, ret + 1); + drbg->Cbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL); + if (!drbg->Cbuf) + goto fini; + drbg->C = PTR_ALIGN(drbg->Cbuf, ret + 1); /* scratchpad is only generated for CTR and Hash */ if (drbg->core->flags & DRBG_HMAC) sb_size = 0; @@ -1204,13 +1198,16 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) sb_size = drbg_statelen(drbg) + drbg_blocklen(drbg); if (0 < sb_size) { - drbg->scratchpad = kzalloc(sb_size, GFP_KERNEL); - if (!drbg->scratchpad) - goto err; + drbg->scratchpadbuf = kzalloc(sb_size + ret, GFP_KERNEL); + if (!drbg->scratchpadbuf) + goto fini; + drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1); } return 0; +fini: + drbg->d_ops->crypto_fini(drbg); err: drbg_dealloc_state(drbg); return ret; @@ -1478,10 +1475,6 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers, if (ret) goto unlock; - ret = -EFAULT; - if (drbg->d_ops->crypto_init(drbg)) - goto err; - ret = drbg_prepare_hrng(drbg); if (ret) goto free_everything; @@ -1505,8 +1498,6 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers, mutex_unlock(&drbg->drbg_mutex); return ret; -err: - drbg_dealloc_state(drbg); unlock: mutex_unlock(&drbg->drbg_mutex); return ret; @@ -1591,7 +1582,8 @@ static int drbg_init_hash_kernel(struct drbg_state *drbg) sdesc->shash.tfm = tfm; sdesc->shash.flags = 0; drbg->priv_data = sdesc; - return 0; + + return crypto_shash_alignmask(tfm); } static int drbg_fini_hash_kernel(struct drbg_state *drbg) @@ -1627,10 +1619,45 @@ static int drbg_kcapi_hash(struct drbg_state *drbg, unsigned char *outval, #endif /* (CONFIG_CRYPTO_DRBG_HASH || CONFIG_CRYPTO_DRBG_HMAC) */ #ifdef CONFIG_CRYPTO_DRBG_CTR +static int drbg_fini_sym_kernel(struct drbg_state *drbg) +{ + struct crypto_cipher *tfm = + (struct crypto_cipher *)drbg->priv_data; + if (tfm) + crypto_free_cipher(tfm); + drbg->priv_data = NULL; + + if (drbg->ctr_handle) + crypto_free_skcipher(drbg->ctr_handle); + drbg->ctr_handle = NULL; + + if (drbg->ctr_req) + skcipher_request_free(drbg->ctr_req); + drbg->ctr_req = NULL; + + kfree(drbg->ctr_null_value_buf); + drbg->ctr_null_value = NULL; + + return 0; +} + +static void drbg_skcipher_cb(struct crypto_async_request *req, int error) +{ + struct drbg_state *drbg = req->data; + + if (error == -EINPROGRESS) + return; + drbg->ctr_async_err = error; + complete(&drbg->ctr_completion); +} + static int drbg_init_sym_kernel(struct drbg_state *drbg) { - int ret = 0; struct crypto_cipher *tfm; + struct crypto_skcipher *sk_tfm; + struct skcipher_request *req; + unsigned int alignmask; + char ctr_name[CRYPTO_MAX_ALG_NAME]; tfm = crypto_alloc_cipher(drbg->core->backend_cra_name, 0, 0); if (IS_ERR(tfm)) { @@ -1640,31 +1667,103 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg) } BUG_ON(drbg_blocklen(drbg) != crypto_cipher_blocksize(tfm)); drbg->priv_data = tfm; - return ret; + + if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", + drbg->core->backend_cra_name) >= CRYPTO_MAX_ALG_NAME) { + drbg_fini_sym_kernel(drbg); + return -EINVAL; + } + sk_tfm = crypto_alloc_skcipher(ctr_name, 0, 0); + if (IS_ERR(sk_tfm)) { + pr_info("DRBG: could not allocate CTR cipher TFM handle: %s\n", + ctr_name); + drbg_fini_sym_kernel(drbg); + return PTR_ERR(sk_tfm); + } + drbg->ctr_handle = sk_tfm; + + req = skcipher_request_alloc(sk_tfm, GFP_KERNEL); + if (!req) { + pr_info("DRBG: could not allocate request queue\n"); + drbg_fini_sym_kernel(drbg); + return -ENOMEM; + } + drbg->ctr_req = req; + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + drbg_skcipher_cb, drbg); + + alignmask = crypto_skcipher_alignmask(sk_tfm); + drbg->ctr_null_value_buf = kzalloc(DRBG_CTR_NULL_LEN + alignmask, + GFP_KERNEL); + if (!drbg->ctr_null_value_buf) { + drbg_fini_sym_kernel(drbg); + return -ENOMEM; + } + drbg->ctr_null_value = (u8 *)PTR_ALIGN(drbg->ctr_null_value_buf, + alignmask + 1); + + return alignmask; } -static int drbg_fini_sym_kernel(struct drbg_state *drbg) +static void drbg_kcapi_symsetkey(struct drbg_state *drbg, + const unsigned char *key) { struct crypto_cipher *tfm = (struct crypto_cipher *)drbg->priv_data; - if (tfm) - crypto_free_cipher(tfm); - drbg->priv_data = NULL; - return 0; + + crypto_cipher_setkey(tfm, key, (drbg_keylen(drbg))); } -static int drbg_kcapi_sym(struct drbg_state *drbg, const unsigned char *key, - unsigned char *outval, const struct drbg_string *in) +static int drbg_kcapi_sym(struct drbg_state *drbg, unsigned char *outval, + const struct drbg_string *in) { struct crypto_cipher *tfm = (struct crypto_cipher *)drbg->priv_data; - crypto_cipher_setkey(tfm, key, (drbg_keylen(drbg))); /* there is only component in *in */ BUG_ON(in->len < drbg_blocklen(drbg)); crypto_cipher_encrypt_one(tfm, outval, in->buf); return 0; } + +static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, + u8 *inbuf, u32 inlen, + u8 *outbuf, u32 outlen) +{ + struct scatterlist sg_in; + + sg_init_one(&sg_in, inbuf, inlen); + + while (outlen) { + u32 cryptlen = min_t(u32, inlen, outlen); + struct scatterlist sg_out; + int ret; + + sg_init_one(&sg_out, outbuf, cryptlen); + skcipher_request_set_crypt(drbg->ctr_req, &sg_in, &sg_out, + cryptlen, drbg->V); + ret = crypto_skcipher_encrypt(drbg->ctr_req); + switch (ret) { + case 0: + break; + case -EINPROGRESS: + case -EBUSY: + ret = wait_for_completion_interruptible( + &drbg->ctr_completion); + if (!ret && !drbg->ctr_async_err) { + reinit_completion(&drbg->ctr_completion); + break; + } + default: + return ret; + } + init_completion(&drbg->ctr_completion); + + outlen -= cryptlen; + } + + return 0; +} #endif /* CONFIG_CRYPTO_DRBG_CTR */ /*************************************************************** diff --git a/crypto/ecc.c b/crypto/ecc.c new file mode 100644 index 000000000000..414c78a9c214 --- /dev/null +++ b/crypto/ecc.c @@ -0,0 +1,1018 @@ +/* + * Copyright (c) 2013, Kenneth MacKay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include "ecc.h" +#include "ecc_curve_defs.h" + +typedef struct { + u64 m_low; + u64 m_high; +} uint128_t; + +static inline const struct ecc_curve *ecc_get_curve(unsigned int curve_id) +{ + switch (curve_id) { + /* In FIPS mode only allow P256 and higher */ + case ECC_CURVE_NIST_P192: + return fips_enabled ? NULL : &nist_p192; + case ECC_CURVE_NIST_P256: + return &nist_p256; + default: + return NULL; + } +} + +static u64 *ecc_alloc_digits_space(unsigned int ndigits) +{ + size_t len = ndigits * sizeof(u64); + + if (!len) + return NULL; + + return kmalloc(len, GFP_KERNEL); +} + +static void ecc_free_digits_space(u64 *space) +{ + kzfree(space); +} + +static struct ecc_point *ecc_alloc_point(unsigned int ndigits) +{ + struct ecc_point *p = kmalloc(sizeof(*p), GFP_KERNEL); + + if (!p) + return NULL; + + p->x = ecc_alloc_digits_space(ndigits); + if (!p->x) + goto err_alloc_x; + + p->y = ecc_alloc_digits_space(ndigits); + if (!p->y) + goto err_alloc_y; + + p->ndigits = ndigits; + + return p; + +err_alloc_y: + ecc_free_digits_space(p->x); +err_alloc_x: + kfree(p); + return NULL; +} + +static void ecc_free_point(struct ecc_point *p) +{ + if (!p) + return; + + kzfree(p->x); + kzfree(p->y); + kzfree(p); +} + +static void vli_clear(u64 *vli, unsigned int ndigits) +{ + int i; + + for (i = 0; i < ndigits; i++) + vli[i] = 0; +} + +/* Returns true if vli == 0, false otherwise. */ +static bool vli_is_zero(const u64 *vli, unsigned int ndigits) +{ + int i; + + for (i = 0; i < ndigits; i++) { + if (vli[i]) + return false; + } + + return true; +} + +/* Returns nonzero if bit bit of vli is set. */ +static u64 vli_test_bit(const u64 *vli, unsigned int bit) +{ + return (vli[bit / 64] & ((u64)1 << (bit % 64))); +} + +/* Counts the number of 64-bit "digits" in vli. */ +static unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits) +{ + int i; + + /* Search from the end until we find a non-zero digit. + * We do it in reverse because we expect that most digits will + * be nonzero. + */ + for (i = ndigits - 1; i >= 0 && vli[i] == 0; i--); + + return (i + 1); +} + +/* Counts the number of bits required for vli. */ +static unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits) +{ + unsigned int i, num_digits; + u64 digit; + + num_digits = vli_num_digits(vli, ndigits); + if (num_digits == 0) + return 0; + + digit = vli[num_digits - 1]; + for (i = 0; digit; i++) + digit >>= 1; + + return ((num_digits - 1) * 64 + i); +} + +/* Sets dest = src. */ +static void vli_set(u64 *dest, const u64 *src, unsigned int ndigits) +{ + int i; + + for (i = 0; i < ndigits; i++) + dest[i] = src[i]; +} + +/* Returns sign of left - right. */ +static int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits) +{ + int i; + + for (i = ndigits - 1; i >= 0; i--) { + if (left[i] > right[i]) + return 1; + else if (left[i] < right[i]) + return -1; + } + + return 0; +} + +/* Computes result = in << c, returning carry. Can modify in place + * (if result == in). 0 < shift < 64. + */ +static u64 vli_lshift(u64 *result, const u64 *in, unsigned int shift, + unsigned int ndigits) +{ + u64 carry = 0; + int i; + + for (i = 0; i < ndigits; i++) { + u64 temp = in[i]; + + result[i] = (temp << shift) | carry; + carry = temp >> (64 - shift); + } + + return carry; +} + +/* Computes vli = vli >> 1. */ +static void vli_rshift1(u64 *vli, unsigned int ndigits) +{ + u64 *end = vli; + u64 carry = 0; + + vli += ndigits; + + while (vli-- > end) { + u64 temp = *vli; + *vli = (temp >> 1) | carry; + carry = temp << 63; + } +} + +/* Computes result = left + right, returning carry. Can modify in place. */ +static u64 vli_add(u64 *result, const u64 *left, const u64 *right, + unsigned int ndigits) +{ + u64 carry = 0; + int i; + + for (i = 0; i < ndigits; i++) { + u64 sum; + + sum = left[i] + right[i] + carry; + if (sum != left[i]) + carry = (sum < left[i]); + + result[i] = sum; + } + + return carry; +} + +/* Computes result = left - right, returning borrow. Can modify in place. */ +static u64 vli_sub(u64 *result, const u64 *left, const u64 *right, + unsigned int ndigits) +{ + u64 borrow = 0; + int i; + + for (i = 0; i < ndigits; i++) { + u64 diff; + + diff = left[i] - right[i] - borrow; + if (diff != left[i]) + borrow = (diff > left[i]); + + result[i] = diff; + } + + return borrow; +} + +static uint128_t mul_64_64(u64 left, u64 right) +{ + u64 a0 = left & 0xffffffffull; + u64 a1 = left >> 32; + u64 b0 = right & 0xffffffffull; + u64 b1 = right >> 32; + u64 m0 = a0 * b0; + u64 m1 = a0 * b1; + u64 m2 = a1 * b0; + u64 m3 = a1 * b1; + uint128_t result; + + m2 += (m0 >> 32); + m2 += m1; + + /* Overflow */ + if (m2 < m1) + m3 += 0x100000000ull; + + result.m_low = (m0 & 0xffffffffull) | (m2 << 32); + result.m_high = m3 + (m2 >> 32); + + return result; +} + +static uint128_t add_128_128(uint128_t a, uint128_t b) +{ + uint128_t result; + + result.m_low = a.m_low + b.m_low; + result.m_high = a.m_high + b.m_high + (result.m_low < a.m_low); + + return result; +} + +static void vli_mult(u64 *result, const u64 *left, const u64 *right, + unsigned int ndigits) +{ + uint128_t r01 = { 0, 0 }; + u64 r2 = 0; + unsigned int i, k; + + /* Compute each digit of result in sequence, maintaining the + * carries. + */ + for (k = 0; k < ndigits * 2 - 1; k++) { + unsigned int min; + + if (k < ndigits) + min = 0; + else + min = (k + 1) - ndigits; + + for (i = min; i <= k && i < ndigits; i++) { + uint128_t product; + + product = mul_64_64(left[i], right[k - i]); + + r01 = add_128_128(r01, product); + r2 += (r01.m_high < product.m_high); + } + + result[k] = r01.m_low; + r01.m_low = r01.m_high; + r01.m_high = r2; + r2 = 0; + } + + result[ndigits * 2 - 1] = r01.m_low; +} + +static void vli_square(u64 *result, const u64 *left, unsigned int ndigits) +{ + uint128_t r01 = { 0, 0 }; + u64 r2 = 0; + int i, k; + + for (k = 0; k < ndigits * 2 - 1; k++) { + unsigned int min; + + if (k < ndigits) + min = 0; + else + min = (k + 1) - ndigits; + + for (i = min; i <= k && i <= k - i; i++) { + uint128_t product; + + product = mul_64_64(left[i], left[k - i]); + + if (i < k - i) { + r2 += product.m_high >> 63; + product.m_high = (product.m_high << 1) | + (product.m_low >> 63); + product.m_low <<= 1; + } + + r01 = add_128_128(r01, product); + r2 += (r01.m_high < product.m_high); + } + + result[k] = r01.m_low; + r01.m_low = r01.m_high; + r01.m_high = r2; + r2 = 0; + } + + result[ndigits * 2 - 1] = r01.m_low; +} + +/* Computes result = (left + right) % mod. + * Assumes that left < mod and right < mod, result != mod. + */ +static void vli_mod_add(u64 *result, const u64 *left, const u64 *right, + const u64 *mod, unsigned int ndigits) +{ + u64 carry; + + carry = vli_add(result, left, right, ndigits); + + /* result > mod (result = mod + remainder), so subtract mod to + * get remainder. + */ + if (carry || vli_cmp(result, mod, ndigits) >= 0) + vli_sub(result, result, mod, ndigits); +} + +/* Computes result = (left - right) % mod. + * Assumes that left < mod and right < mod, result != mod. + */ +static void vli_mod_sub(u64 *result, const u64 *left, const u64 *right, + const u64 *mod, unsigned int ndigits) +{ + u64 borrow = vli_sub(result, left, right, ndigits); + + /* In this case, p_result == -diff == (max int) - diff. + * Since -x % d == d - x, we can get the correct result from + * result + mod (with overflow). + */ + if (borrow) + vli_add(result, result, mod, ndigits); +} + +/* Computes p_result = p_product % curve_p. + * See algorithm 5 and 6 from + * http://www.isys.uni-klu.ac.at/PDF/2001-0126-MT.pdf + */ +static void vli_mmod_fast_192(u64 *result, const u64 *product, + const u64 *curve_prime, u64 *tmp) +{ + const unsigned int ndigits = 3; + int carry; + + vli_set(result, product, ndigits); + + vli_set(tmp, &product[3], ndigits); + carry = vli_add(result, result, tmp, ndigits); + + tmp[0] = 0; + tmp[1] = product[3]; + tmp[2] = product[4]; + carry += vli_add(result, result, tmp, ndigits); + + tmp[0] = tmp[1] = product[5]; + tmp[2] = 0; + carry += vli_add(result, result, tmp, ndigits); + + while (carry || vli_cmp(curve_prime, result, ndigits) != 1) + carry -= vli_sub(result, result, curve_prime, ndigits); +} + +/* Computes result = product % curve_prime + * from http://www.nsa.gov/ia/_files/nist-routines.pdf + */ +static void vli_mmod_fast_256(u64 *result, const u64 *product, + const u64 *curve_prime, u64 *tmp) +{ + int carry; + const unsigned int ndigits = 4; + + /* t */ + vli_set(result, product, ndigits); + + /* s1 */ + tmp[0] = 0; + tmp[1] = product[5] & 0xffffffff00000000ull; + tmp[2] = product[6]; + tmp[3] = product[7]; + carry = vli_lshift(tmp, tmp, 1, ndigits); + carry += vli_add(result, result, tmp, ndigits); + + /* s2 */ + tmp[1] = product[6] << 32; + tmp[2] = (product[6] >> 32) | (product[7] << 32); + tmp[3] = product[7] >> 32; + carry += vli_lshift(tmp, tmp, 1, ndigits); + carry += vli_add(result, result, tmp, ndigits); + + /* s3 */ + tmp[0] = product[4]; + tmp[1] = product[5] & 0xffffffff; + tmp[2] = 0; + tmp[3] = product[7]; + carry += vli_add(result, result, tmp, ndigits); + + /* s4 */ + tmp[0] = (product[4] >> 32) | (product[5] << 32); + tmp[1] = (product[5] >> 32) | (product[6] & 0xffffffff00000000ull); + tmp[2] = product[7]; + tmp[3] = (product[6] >> 32) | (product[4] << 32); + carry += vli_add(result, result, tmp, ndigits); + + /* d1 */ + tmp[0] = (product[5] >> 32) | (product[6] << 32); + tmp[1] = (product[6] >> 32); + tmp[2] = 0; + tmp[3] = (product[4] & 0xffffffff) | (product[5] << 32); + carry -= vli_sub(result, result, tmp, ndigits); + + /* d2 */ + tmp[0] = product[6]; + tmp[1] = product[7]; + tmp[2] = 0; + tmp[3] = (product[4] >> 32) | (product[5] & 0xffffffff00000000ull); + carry -= vli_sub(result, result, tmp, ndigits); + + /* d3 */ + tmp[0] = (product[6] >> 32) | (product[7] << 32); + tmp[1] = (product[7] >> 32) | (product[4] << 32); + tmp[2] = (product[4] >> 32) | (product[5] << 32); + tmp[3] = (product[6] << 32); + carry -= vli_sub(result, result, tmp, ndigits); + + /* d4 */ + tmp[0] = product[7]; + tmp[1] = product[4] & 0xffffffff00000000ull; + tmp[2] = product[5]; + tmp[3] = product[6] & 0xffffffff00000000ull; + carry -= vli_sub(result, result, tmp, ndigits); + + if (carry < 0) { + do { + carry += vli_add(result, result, curve_prime, ndigits); + } while (carry < 0); + } else { + while (carry || vli_cmp(curve_prime, result, ndigits) != 1) + carry -= vli_sub(result, result, curve_prime, ndigits); + } +} + +/* Computes result = product % curve_prime + * from http://www.nsa.gov/ia/_files/nist-routines.pdf +*/ +static bool vli_mmod_fast(u64 *result, u64 *product, + const u64 *curve_prime, unsigned int ndigits) +{ + u64 tmp[2 * ndigits]; + + switch (ndigits) { + case 3: + vli_mmod_fast_192(result, product, curve_prime, tmp); + break; + case 4: + vli_mmod_fast_256(result, product, curve_prime, tmp); + break; + default: + pr_err("unsupports digits size!\n"); + return false; + } + + return true; +} + +/* Computes result = (left * right) % curve_prime. */ +static void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right, + const u64 *curve_prime, unsigned int ndigits) +{ + u64 product[2 * ndigits]; + + vli_mult(product, left, right, ndigits); + vli_mmod_fast(result, product, curve_prime, ndigits); +} + +/* Computes result = left^2 % curve_prime. */ +static void vli_mod_square_fast(u64 *result, const u64 *left, + const u64 *curve_prime, unsigned int ndigits) +{ + u64 product[2 * ndigits]; + + vli_square(product, left, ndigits); + vli_mmod_fast(result, product, curve_prime, ndigits); +} + +#define EVEN(vli) (!(vli[0] & 1)) +/* Computes result = (1 / p_input) % mod. All VLIs are the same size. + * See "From Euclid's GCD to Montgomery Multiplication to the Great Divide" + * https://labs.oracle.com/techrep/2001/smli_tr-2001-95.pdf + */ +static void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, + unsigned int ndigits) +{ + u64 a[ndigits], b[ndigits]; + u64 u[ndigits], v[ndigits]; + u64 carry; + int cmp_result; + + if (vli_is_zero(input, ndigits)) { + vli_clear(result, ndigits); + return; + } + + vli_set(a, input, ndigits); + vli_set(b, mod, ndigits); + vli_clear(u, ndigits); + u[0] = 1; + vli_clear(v, ndigits); + + while ((cmp_result = vli_cmp(a, b, ndigits)) != 0) { + carry = 0; + + if (EVEN(a)) { + vli_rshift1(a, ndigits); + + if (!EVEN(u)) + carry = vli_add(u, u, mod, ndigits); + + vli_rshift1(u, ndigits); + if (carry) + u[ndigits - 1] |= 0x8000000000000000ull; + } else if (EVEN(b)) { + vli_rshift1(b, ndigits); + + if (!EVEN(v)) + carry = vli_add(v, v, mod, ndigits); + + vli_rshift1(v, ndigits); + if (carry) + v[ndigits - 1] |= 0x8000000000000000ull; + } else if (cmp_result > 0) { + vli_sub(a, a, b, ndigits); + vli_rshift1(a, ndigits); + + if (vli_cmp(u, v, ndigits) < 0) + vli_add(u, u, mod, ndigits); + + vli_sub(u, u, v, ndigits); + if (!EVEN(u)) + carry = vli_add(u, u, mod, ndigits); + + vli_rshift1(u, ndigits); + if (carry) + u[ndigits - 1] |= 0x8000000000000000ull; + } else { + vli_sub(b, b, a, ndigits); + vli_rshift1(b, ndigits); + + if (vli_cmp(v, u, ndigits) < 0) + vli_add(v, v, mod, ndigits); + + vli_sub(v, v, u, ndigits); + if (!EVEN(v)) + carry = vli_add(v, v, mod, ndigits); + + vli_rshift1(v, ndigits); + if (carry) + v[ndigits - 1] |= 0x8000000000000000ull; + } + } + + vli_set(result, u, ndigits); +} + +/* ------ Point operations ------ */ + +/* Returns true if p_point is the point at infinity, false otherwise. */ +static bool ecc_point_is_zero(const struct ecc_point *point) +{ + return (vli_is_zero(point->x, point->ndigits) && + vli_is_zero(point->y, point->ndigits)); +} + +/* Point multiplication algorithm using Montgomery's ladder with co-Z + * coordinates. From http://eprint.iacr.org/2011/338.pdf + */ + +/* Double in place */ +static void ecc_point_double_jacobian(u64 *x1, u64 *y1, u64 *z1, + u64 *curve_prime, unsigned int ndigits) +{ + /* t1 = x, t2 = y, t3 = z */ + u64 t4[ndigits]; + u64 t5[ndigits]; + + if (vli_is_zero(z1, ndigits)) + return; + + /* t4 = y1^2 */ + vli_mod_square_fast(t4, y1, curve_prime, ndigits); + /* t5 = x1*y1^2 = A */ + vli_mod_mult_fast(t5, x1, t4, curve_prime, ndigits); + /* t4 = y1^4 */ + vli_mod_square_fast(t4, t4, curve_prime, ndigits); + /* t2 = y1*z1 = z3 */ + vli_mod_mult_fast(y1, y1, z1, curve_prime, ndigits); + /* t3 = z1^2 */ + vli_mod_square_fast(z1, z1, curve_prime, ndigits); + + /* t1 = x1 + z1^2 */ + vli_mod_add(x1, x1, z1, curve_prime, ndigits); + /* t3 = 2*z1^2 */ + vli_mod_add(z1, z1, z1, curve_prime, ndigits); + /* t3 = x1 - z1^2 */ + vli_mod_sub(z1, x1, z1, curve_prime, ndigits); + /* t1 = x1^2 - z1^4 */ + vli_mod_mult_fast(x1, x1, z1, curve_prime, ndigits); + + /* t3 = 2*(x1^2 - z1^4) */ + vli_mod_add(z1, x1, x1, curve_prime, ndigits); + /* t1 = 3*(x1^2 - z1^4) */ + vli_mod_add(x1, x1, z1, curve_prime, ndigits); + if (vli_test_bit(x1, 0)) { + u64 carry = vli_add(x1, x1, curve_prime, ndigits); + + vli_rshift1(x1, ndigits); + x1[ndigits - 1] |= carry << 63; + } else { + vli_rshift1(x1, ndigits); + } + /* t1 = 3/2*(x1^2 - z1^4) = B */ + + /* t3 = B^2 */ + vli_mod_square_fast(z1, x1, curve_prime, ndigits); + /* t3 = B^2 - A */ + vli_mod_sub(z1, z1, t5, curve_prime, ndigits); + /* t3 = B^2 - 2A = x3 */ + vli_mod_sub(z1, z1, t5, curve_prime, ndigits); + /* t5 = A - x3 */ + vli_mod_sub(t5, t5, z1, curve_prime, ndigits); + /* t1 = B * (A - x3) */ + vli_mod_mult_fast(x1, x1, t5, curve_prime, ndigits); + /* t4 = B * (A - x3) - y1^4 = y3 */ + vli_mod_sub(t4, x1, t4, curve_prime, ndigits); + + vli_set(x1, z1, ndigits); + vli_set(z1, y1, ndigits); + vli_set(y1, t4, ndigits); +} + +/* Modify (x1, y1) => (x1 * z^2, y1 * z^3) */ +static void apply_z(u64 *x1, u64 *y1, u64 *z, u64 *curve_prime, + unsigned int ndigits) +{ + u64 t1[ndigits]; + + vli_mod_square_fast(t1, z, curve_prime, ndigits); /* z^2 */ + vli_mod_mult_fast(x1, x1, t1, curve_prime, ndigits); /* x1 * z^2 */ + vli_mod_mult_fast(t1, t1, z, curve_prime, ndigits); /* z^3 */ + vli_mod_mult_fast(y1, y1, t1, curve_prime, ndigits); /* y1 * z^3 */ +} + +/* P = (x1, y1) => 2P, (x2, y2) => P' */ +static void xycz_initial_double(u64 *x1, u64 *y1, u64 *x2, u64 *y2, + u64 *p_initial_z, u64 *curve_prime, + unsigned int ndigits) +{ + u64 z[ndigits]; + + vli_set(x2, x1, ndigits); + vli_set(y2, y1, ndigits); + + vli_clear(z, ndigits); + z[0] = 1; + + if (p_initial_z) + vli_set(z, p_initial_z, ndigits); + + apply_z(x1, y1, z, curve_prime, ndigits); + + ecc_point_double_jacobian(x1, y1, z, curve_prime, ndigits); + + apply_z(x2, y2, z, curve_prime, ndigits); +} + +/* Input P = (x1, y1, Z), Q = (x2, y2, Z) + * Output P' = (x1', y1', Z3), P + Q = (x3, y3, Z3) + * or P => P', Q => P + Q + */ +static void xycz_add(u64 *x1, u64 *y1, u64 *x2, u64 *y2, u64 *curve_prime, + unsigned int ndigits) +{ + /* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */ + u64 t5[ndigits]; + + /* t5 = x2 - x1 */ + vli_mod_sub(t5, x2, x1, curve_prime, ndigits); + /* t5 = (x2 - x1)^2 = A */ + vli_mod_square_fast(t5, t5, curve_prime, ndigits); + /* t1 = x1*A = B */ + vli_mod_mult_fast(x1, x1, t5, curve_prime, ndigits); + /* t3 = x2*A = C */ + vli_mod_mult_fast(x2, x2, t5, curve_prime, ndigits); + /* t4 = y2 - y1 */ + vli_mod_sub(y2, y2, y1, curve_prime, ndigits); + /* t5 = (y2 - y1)^2 = D */ + vli_mod_square_fast(t5, y2, curve_prime, ndigits); + + /* t5 = D - B */ + vli_mod_sub(t5, t5, x1, curve_prime, ndigits); + /* t5 = D - B - C = x3 */ + vli_mod_sub(t5, t5, x2, curve_prime, ndigits); + /* t3 = C - B */ + vli_mod_sub(x2, x2, x1, curve_prime, ndigits); + /* t2 = y1*(C - B) */ + vli_mod_mult_fast(y1, y1, x2, curve_prime, ndigits); + /* t3 = B - x3 */ + vli_mod_sub(x2, x1, t5, curve_prime, ndigits); + /* t4 = (y2 - y1)*(B - x3) */ + vli_mod_mult_fast(y2, y2, x2, curve_prime, ndigits); + /* t4 = y3 */ + vli_mod_sub(y2, y2, y1, curve_prime, ndigits); + + vli_set(x2, t5, ndigits); +} + +/* Input P = (x1, y1, Z), Q = (x2, y2, Z) + * Output P + Q = (x3, y3, Z3), P - Q = (x3', y3', Z3) + * or P => P - Q, Q => P + Q + */ +static void xycz_add_c(u64 *x1, u64 *y1, u64 *x2, u64 *y2, u64 *curve_prime, + unsigned int ndigits) +{ + /* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */ + u64 t5[ndigits]; + u64 t6[ndigits]; + u64 t7[ndigits]; + + /* t5 = x2 - x1 */ + vli_mod_sub(t5, x2, x1, curve_prime, ndigits); + /* t5 = (x2 - x1)^2 = A */ + vli_mod_square_fast(t5, t5, curve_prime, ndigits); + /* t1 = x1*A = B */ + vli_mod_mult_fast(x1, x1, t5, curve_prime, ndigits); + /* t3 = x2*A = C */ + vli_mod_mult_fast(x2, x2, t5, curve_prime, ndigits); + /* t4 = y2 + y1 */ + vli_mod_add(t5, y2, y1, curve_prime, ndigits); + /* t4 = y2 - y1 */ + vli_mod_sub(y2, y2, y1, curve_prime, ndigits); + + /* t6 = C - B */ + vli_mod_sub(t6, x2, x1, curve_prime, ndigits); + /* t2 = y1 * (C - B) */ + vli_mod_mult_fast(y1, y1, t6, curve_prime, ndigits); + /* t6 = B + C */ + vli_mod_add(t6, x1, x2, curve_prime, ndigits); + /* t3 = (y2 - y1)^2 */ + vli_mod_square_fast(x2, y2, curve_prime, ndigits); + /* t3 = x3 */ + vli_mod_sub(x2, x2, t6, curve_prime, ndigits); + + /* t7 = B - x3 */ + vli_mod_sub(t7, x1, x2, curve_prime, ndigits); + /* t4 = (y2 - y1)*(B - x3) */ + vli_mod_mult_fast(y2, y2, t7, curve_prime, ndigits); + /* t4 = y3 */ + vli_mod_sub(y2, y2, y1, curve_prime, ndigits); + + /* t7 = (y2 + y1)^2 = F */ + vli_mod_square_fast(t7, t5, curve_prime, ndigits); + /* t7 = x3' */ + vli_mod_sub(t7, t7, t6, curve_prime, ndigits); + /* t6 = x3' - B */ + vli_mod_sub(t6, t7, x1, curve_prime, ndigits); + /* t6 = (y2 + y1)*(x3' - B) */ + vli_mod_mult_fast(t6, t6, t5, curve_prime, ndigits); + /* t2 = y3' */ + vli_mod_sub(y1, t6, y1, curve_prime, ndigits); + + vli_set(x1, t7, ndigits); +} + +static void ecc_point_mult(struct ecc_point *result, + const struct ecc_point *point, const u64 *scalar, + u64 *initial_z, u64 *curve_prime, + unsigned int ndigits) +{ + /* R0 and R1 */ + u64 rx[2][ndigits]; + u64 ry[2][ndigits]; + u64 z[ndigits]; + int i, nb; + int num_bits = vli_num_bits(scalar, ndigits); + + vli_set(rx[1], point->x, ndigits); + vli_set(ry[1], point->y, ndigits); + + xycz_initial_double(rx[1], ry[1], rx[0], ry[0], initial_z, curve_prime, + ndigits); + + for (i = num_bits - 2; i > 0; i--) { + nb = !vli_test_bit(scalar, i); + xycz_add_c(rx[1 - nb], ry[1 - nb], rx[nb], ry[nb], curve_prime, + ndigits); + xycz_add(rx[nb], ry[nb], rx[1 - nb], ry[1 - nb], curve_prime, + ndigits); + } + + nb = !vli_test_bit(scalar, 0); + xycz_add_c(rx[1 - nb], ry[1 - nb], rx[nb], ry[nb], curve_prime, + ndigits); + + /* Find final 1/Z value. */ + /* X1 - X0 */ + vli_mod_sub(z, rx[1], rx[0], curve_prime, ndigits); + /* Yb * (X1 - X0) */ + vli_mod_mult_fast(z, z, ry[1 - nb], curve_prime, ndigits); + /* xP * Yb * (X1 - X0) */ + vli_mod_mult_fast(z, z, point->x, curve_prime, ndigits); + + /* 1 / (xP * Yb * (X1 - X0)) */ + vli_mod_inv(z, z, curve_prime, point->ndigits); + + /* yP / (xP * Yb * (X1 - X0)) */ + vli_mod_mult_fast(z, z, point->y, curve_prime, ndigits); + /* Xb * yP / (xP * Yb * (X1 - X0)) */ + vli_mod_mult_fast(z, z, rx[1 - nb], curve_prime, ndigits); + /* End 1/Z calculation */ + + xycz_add(rx[nb], ry[nb], rx[1 - nb], ry[1 - nb], curve_prime, ndigits); + + apply_z(rx[0], ry[0], z, curve_prime, ndigits); + + vli_set(result->x, rx[0], ndigits); + vli_set(result->y, ry[0], ndigits); +} + +static inline void ecc_swap_digits(const u64 *in, u64 *out, + unsigned int ndigits) +{ + int i; + + for (i = 0; i < ndigits; i++) + out[i] = __swab64(in[ndigits - 1 - i]); +} + +int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, + const u8 *private_key, unsigned int private_key_len) +{ + int nbytes; + const struct ecc_curve *curve = ecc_get_curve(curve_id); + + if (!private_key) + return -EINVAL; + + nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT; + + if (private_key_len != nbytes) + return -EINVAL; + + if (vli_is_zero((const u64 *)&private_key[0], ndigits)) + return -EINVAL; + + /* Make sure the private key is in the range [1, n-1]. */ + if (vli_cmp(curve->n, (const u64 *)&private_key[0], ndigits) != 1) + return -EINVAL; + + return 0; +} + +int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits, + const u8 *private_key, unsigned int private_key_len, + u8 *public_key, unsigned int public_key_len) +{ + int ret = 0; + struct ecc_point *pk; + u64 priv[ndigits]; + unsigned int nbytes; + const struct ecc_curve *curve = ecc_get_curve(curve_id); + + if (!private_key || !curve) { + ret = -EINVAL; + goto out; + } + + ecc_swap_digits((const u64 *)private_key, priv, ndigits); + + pk = ecc_alloc_point(ndigits); + if (!pk) { + ret = -ENOMEM; + goto out; + } + + ecc_point_mult(pk, &curve->g, priv, NULL, curve->p, ndigits); + if (ecc_point_is_zero(pk)) { + ret = -EAGAIN; + goto err_free_point; + } + + nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT; + ecc_swap_digits(pk->x, (u64 *)public_key, ndigits); + ecc_swap_digits(pk->y, (u64 *)&public_key[nbytes], ndigits); + +err_free_point: + ecc_free_point(pk); +out: + return ret; +} + +int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, + const u8 *private_key, unsigned int private_key_len, + const u8 *public_key, unsigned int public_key_len, + u8 *secret, unsigned int secret_len) +{ + int ret = 0; + struct ecc_point *product, *pk; + u64 priv[ndigits]; + u64 rand_z[ndigits]; + unsigned int nbytes; + const struct ecc_curve *curve = ecc_get_curve(curve_id); + + if (!private_key || !public_key || !curve) { + ret = -EINVAL; + goto out; + } + + nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT; + + get_random_bytes(rand_z, nbytes); + + pk = ecc_alloc_point(ndigits); + if (!pk) { + ret = -ENOMEM; + goto out; + } + + product = ecc_alloc_point(ndigits); + if (!product) { + ret = -ENOMEM; + goto err_alloc_product; + } + + ecc_swap_digits((const u64 *)public_key, pk->x, ndigits); + ecc_swap_digits((const u64 *)&public_key[nbytes], pk->y, ndigits); + ecc_swap_digits((const u64 *)private_key, priv, ndigits); + + ecc_point_mult(product, pk, priv, rand_z, curve->p, ndigits); + + ecc_swap_digits(product->x, (u64 *)secret, ndigits); + + if (ecc_point_is_zero(product)) + ret = -EFAULT; + + ecc_free_point(product); +err_alloc_product: + ecc_free_point(pk); +out: + return ret; +} diff --git a/crypto/ecc.h b/crypto/ecc.h new file mode 100644 index 000000000000..663d598c7406 --- /dev/null +++ b/crypto/ecc.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2013, Kenneth MacKay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _CRYPTO_ECC_H +#define _CRYPTO_ECC_H + +#define ECC_MAX_DIGITS 4 /* 256 */ + +#define ECC_DIGITS_TO_BYTES_SHIFT 3 + +/** + * ecc_is_key_valid() - Validate a given ECDH private key + * + * @curve_id: id representing the curve to use + * @ndigits: curve number of digits + * @private_key: private key to be used for the given curve + * @private_key_len: private key len + * + * Returns 0 if the key is acceptable, a negative value otherwise + */ +int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, + const u8 *private_key, unsigned int private_key_len); + +/** + * ecdh_make_pub_key() - Compute an ECC public key + * + * @curve_id: id representing the curve to use + * @private_key: pregenerated private key for the given curve + * @private_key_len: length of private_key + * @public_key: buffer for storing the public key generated + * @public_key_len: length of the public_key buffer + * + * Returns 0 if the public key was generated successfully, a negative value + * if an error occurred. + */ +int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits, + const u8 *private_key, unsigned int private_key_len, + u8 *public_key, unsigned int public_key_len); + +/** + * crypto_ecdh_shared_secret() - Compute a shared secret + * + * @curve_id: id representing the curve to use + * @private_key: private key of part A + * @private_key_len: length of private_key + * @public_key: public key of counterpart B + * @public_key_len: length of public_key + * @secret: buffer for storing the calculated shared secret + * @secret_len: length of the secret buffer + * + * Note: It is recommended that you hash the result of crypto_ecdh_shared_secret + * before using it for symmetric encryption or HMAC. + * + * Returns 0 if the shared secret was generated successfully, a negative value + * if an error occurred. + */ +int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, + const u8 *private_key, unsigned int private_key_len, + const u8 *public_key, unsigned int public_key_len, + u8 *secret, unsigned int secret_len); +#endif diff --git a/crypto/ecc_curve_defs.h b/crypto/ecc_curve_defs.h new file mode 100644 index 000000000000..03ae5f714028 --- /dev/null +++ b/crypto/ecc_curve_defs.h @@ -0,0 +1,57 @@ +#ifndef _CRYTO_ECC_CURVE_DEFS_H +#define _CRYTO_ECC_CURVE_DEFS_H + +struct ecc_point { + u64 *x; + u64 *y; + u8 ndigits; +}; + +struct ecc_curve { + char *name; + struct ecc_point g; + u64 *p; + u64 *n; +}; + +/* NIST P-192 */ +static u64 nist_p192_g_x[] = { 0xF4FF0AFD82FF1012ull, 0x7CBF20EB43A18800ull, + 0x188DA80EB03090F6ull }; +static u64 nist_p192_g_y[] = { 0x73F977A11E794811ull, 0x631011ED6B24CDD5ull, + 0x07192B95FFC8DA78ull }; +static u64 nist_p192_p[] = { 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFEull, + 0xFFFFFFFFFFFFFFFFull }; +static u64 nist_p192_n[] = { 0x146BC9B1B4D22831ull, 0xFFFFFFFF99DEF836ull, + 0xFFFFFFFFFFFFFFFFull }; +static struct ecc_curve nist_p192 = { + .name = "nist_192", + .g = { + .x = nist_p192_g_x, + .y = nist_p192_g_y, + .ndigits = 3, + }, + .p = nist_p192_p, + .n = nist_p192_n +}; + +/* NIST P-256 */ +static u64 nist_p256_g_x[] = { 0xF4A13945D898C296ull, 0x77037D812DEB33A0ull, + 0xF8BCE6E563A440F2ull, 0x6B17D1F2E12C4247ull }; +static u64 nist_p256_g_y[] = { 0xCBB6406837BF51F5ull, 0x2BCE33576B315ECEull, + 0x8EE7EB4A7C0F9E16ull, 0x4FE342E2FE1A7F9Bull }; +static u64 nist_p256_p[] = { 0xFFFFFFFFFFFFFFFFull, 0x00000000FFFFFFFFull, + 0x0000000000000000ull, 0xFFFFFFFF00000001ull }; +static u64 nist_p256_n[] = { 0xF3B9CAC2FC632551ull, 0xBCE6FAADA7179E84ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFF00000000ull }; +static struct ecc_curve nist_p256 = { + .name = "nist_256", + .g = { + .x = nist_p256_g_x, + .y = nist_p256_g_y, + .ndigits = 4, + }, + .p = nist_p256_p, + .n = nist_p256_n +}; + +#endif diff --git a/crypto/ecdh.c b/crypto/ecdh.c new file mode 100644 index 000000000000..3de289806d67 --- /dev/null +++ b/crypto/ecdh.c @@ -0,0 +1,151 @@ +/* ECDH key-agreement protocol + * + * Copyright (c) 2016, Intel Corporation + * Authors: Salvator Benedetto + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public Licence + * as published by the Free Software Foundation; either version + * 2 of the Licence, or (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include "ecc.h" + +struct ecdh_ctx { + unsigned int curve_id; + unsigned int ndigits; + u64 private_key[ECC_MAX_DIGITS]; + u64 public_key[2 * ECC_MAX_DIGITS]; + u64 shared_secret[ECC_MAX_DIGITS]; +}; + +static inline struct ecdh_ctx *ecdh_get_ctx(struct crypto_kpp *tfm) +{ + return kpp_tfm_ctx(tfm); +} + +static unsigned int ecdh_supported_curve(unsigned int curve_id) +{ + switch (curve_id) { + case ECC_CURVE_NIST_P192: return 3; + case ECC_CURVE_NIST_P256: return 4; + default: return 0; + } +} + +static int ecdh_set_secret(struct crypto_kpp *tfm, void *buf, unsigned int len) +{ + struct ecdh_ctx *ctx = ecdh_get_ctx(tfm); + struct ecdh params; + unsigned int ndigits; + + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) + return -EINVAL; + + ndigits = ecdh_supported_curve(params.curve_id); + if (!ndigits) + return -EINVAL; + + ctx->curve_id = params.curve_id; + ctx->ndigits = ndigits; + + if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits, + (const u8 *)params.key, params.key_size) < 0) + return -EINVAL; + + memcpy(ctx->private_key, params.key, params.key_size); + + return 0; +} + +static int ecdh_compute_value(struct kpp_request *req) +{ + int ret = 0; + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct ecdh_ctx *ctx = ecdh_get_ctx(tfm); + size_t copied, nbytes; + void *buf; + + nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT; + + if (req->src) { + copied = sg_copy_to_buffer(req->src, 1, ctx->public_key, + 2 * nbytes); + if (copied != 2 * nbytes) + return -EINVAL; + + ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits, + (const u8 *)ctx->private_key, nbytes, + (const u8 *)ctx->public_key, 2 * nbytes, + (u8 *)ctx->shared_secret, nbytes); + + buf = ctx->shared_secret; + } else { + ret = ecdh_make_pub_key(ctx->curve_id, ctx->ndigits, + (const u8 *)ctx->private_key, nbytes, + (u8 *)ctx->public_key, + sizeof(ctx->public_key)); + buf = ctx->public_key; + /* Public part is a point thus it has both coordinates */ + nbytes *= 2; + } + + if (ret < 0) + return ret; + + copied = sg_copy_from_buffer(req->dst, 1, buf, nbytes); + if (copied != nbytes) + return -EINVAL; + + return ret; +} + +static int ecdh_max_size(struct crypto_kpp *tfm) +{ + struct ecdh_ctx *ctx = ecdh_get_ctx(tfm); + int nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT; + + /* Public key is made of two coordinates */ + return 2 * nbytes; +} + +static void no_exit_tfm(struct crypto_kpp *tfm) +{ + return; +} + +static struct kpp_alg ecdh = { + .set_secret = ecdh_set_secret, + .generate_public_key = ecdh_compute_value, + .compute_shared_secret = ecdh_compute_value, + .max_size = ecdh_max_size, + .exit = no_exit_tfm, + .base = { + .cra_name = "ecdh", + .cra_driver_name = "ecdh-generic", + .cra_priority = 100, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ecdh_ctx), + }, +}; + +static int ecdh_init(void) +{ + return crypto_register_kpp(&ecdh); +} + +static void ecdh_exit(void) +{ + crypto_unregister_kpp(&ecdh); +} + +module_init(ecdh_init); +module_exit(ecdh_exit); +MODULE_ALIAS_CRYPTO("ecdh"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ECDH generic algorithm"); diff --git a/crypto/ecdh_helper.c b/crypto/ecdh_helper.c new file mode 100644 index 000000000000..3cd8a2414e60 --- /dev/null +++ b/crypto/ecdh_helper.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016, Intel Corporation + * Authors: Salvatore Benedetto + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public Licence + * as published by the Free Software Foundation; either version + * 2 of the Licence, or (at your option) any later version. + */ +#include +#include +#include +#include +#include +#include + +#define ECDH_KPP_SECRET_MIN_SIZE (sizeof(struct kpp_secret) + 2 * sizeof(short)) + +static inline u8 *ecdh_pack_data(void *dst, const void *src, size_t sz) +{ + memcpy(dst, src, sz); + return dst + sz; +} + +static inline const u8 *ecdh_unpack_data(void *dst, const void *src, size_t sz) +{ + memcpy(dst, src, sz); + return src + sz; +} + +int crypto_ecdh_key_len(const struct ecdh *params) +{ + return ECDH_KPP_SECRET_MIN_SIZE + params->key_size; +} +EXPORT_SYMBOL_GPL(crypto_ecdh_key_len); + +int crypto_ecdh_encode_key(char *buf, unsigned int len, + const struct ecdh *params) +{ + u8 *ptr = buf; + struct kpp_secret secret = { + .type = CRYPTO_KPP_SECRET_TYPE_ECDH, + .len = len + }; + + if (unlikely(!buf)) + return -EINVAL; + + if (len != crypto_ecdh_key_len(params)) + return -EINVAL; + + ptr = ecdh_pack_data(ptr, &secret, sizeof(secret)); + ptr = ecdh_pack_data(ptr, ¶ms->curve_id, sizeof(params->curve_id)); + ptr = ecdh_pack_data(ptr, ¶ms->key_size, sizeof(params->key_size)); + ecdh_pack_data(ptr, params->key, params->key_size); + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_ecdh_encode_key); + +int crypto_ecdh_decode_key(const char *buf, unsigned int len, + struct ecdh *params) +{ + const u8 *ptr = buf; + struct kpp_secret secret; + + if (unlikely(!buf || len < ECDH_KPP_SECRET_MIN_SIZE)) + return -EINVAL; + + ptr = ecdh_unpack_data(&secret, ptr, sizeof(secret)); + if (secret.type != CRYPTO_KPP_SECRET_TYPE_ECDH) + return -EINVAL; + + ptr = ecdh_unpack_data(¶ms->curve_id, ptr, sizeof(params->curve_id)); + ptr = ecdh_unpack_data(¶ms->key_size, ptr, sizeof(params->key_size)); + if (secret.len != crypto_ecdh_key_len(params)) + return -EINVAL; + + /* Don't allocate memory. Set pointer to data + * within the given buffer + */ + params->key = (void *)ptr; + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_ecdh_decode_key); diff --git a/crypto/echainiv.c b/crypto/echainiv.c index b96a84560b67..1b01fe98e91f 100644 --- a/crypto/echainiv.c +++ b/crypto/echainiv.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -112,13 +113,16 @@ static int echainiv_encrypt(struct aead_request *req) info = req->iv; if (req->src != req->dst) { - struct blkcipher_desc desc = { - .tfm = ctx->null, - }; + SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull); - err = crypto_blkcipher_encrypt( - &desc, req->dst, req->src, - req->assoclen + req->cryptlen); + skcipher_request_set_tfm(nreq, ctx->sknull); + skcipher_request_set_callback(nreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(nreq, req->src, req->dst, + req->assoclen + req->cryptlen, + NULL); + + err = crypto_skcipher_encrypt(nreq); if (err) return err; } diff --git a/crypto/eseqiv.c b/crypto/eseqiv.c deleted file mode 100644 index 16dda72fc4f8..000000000000 --- a/crypto/eseqiv.c +++ /dev/null @@ -1,242 +0,0 @@ -/* - * eseqiv: Encrypted Sequence Number IV Generator - * - * This generator generates an IV based on a sequence number by xoring it - * with a salt and then encrypting it with the same key as used to encrypt - * the plain text. This algorithm requires that the block size be equal - * to the IV size. It is mainly useful for CBC. - * - * Copyright (c) 2007 Herbert Xu - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct eseqiv_request_ctx { - struct scatterlist src[2]; - struct scatterlist dst[2]; - char tail[]; -}; - -struct eseqiv_ctx { - spinlock_t lock; - unsigned int reqoff; - char salt[]; -}; - -static void eseqiv_complete2(struct skcipher_givcrypt_request *req) -{ - struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req); - struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req); - - memcpy(req->giv, PTR_ALIGN((u8 *)reqctx->tail, - crypto_ablkcipher_alignmask(geniv) + 1), - crypto_ablkcipher_ivsize(geniv)); -} - -static void eseqiv_complete(struct crypto_async_request *base, int err) -{ - struct skcipher_givcrypt_request *req = base->data; - - if (err) - goto out; - - eseqiv_complete2(req); - -out: - skcipher_givcrypt_complete(req, err); -} - -static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req) -{ - struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req); - struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv); - struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req); - struct ablkcipher_request *subreq; - crypto_completion_t compl; - void *data; - struct scatterlist *osrc, *odst; - struct scatterlist *dst; - struct page *srcp; - struct page *dstp; - u8 *giv; - u8 *vsrc; - u8 *vdst; - __be64 seq; - unsigned int ivsize; - unsigned int len; - int err; - - subreq = (void *)(reqctx->tail + ctx->reqoff); - ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv)); - - giv = req->giv; - compl = req->creq.base.complete; - data = req->creq.base.data; - - osrc = req->creq.src; - odst = req->creq.dst; - srcp = sg_page(osrc); - dstp = sg_page(odst); - vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + osrc->offset; - vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + odst->offset; - - ivsize = crypto_ablkcipher_ivsize(geniv); - - if (vsrc != giv + ivsize && vdst != giv + ivsize) { - giv = PTR_ALIGN((u8 *)reqctx->tail, - crypto_ablkcipher_alignmask(geniv) + 1); - compl = eseqiv_complete; - data = req; - } - - ablkcipher_request_set_callback(subreq, req->creq.base.flags, compl, - data); - - sg_init_table(reqctx->src, 2); - sg_set_buf(reqctx->src, giv, ivsize); - scatterwalk_crypto_chain(reqctx->src, osrc, vsrc == giv + ivsize, 2); - - dst = reqctx->src; - if (osrc != odst) { - sg_init_table(reqctx->dst, 2); - sg_set_buf(reqctx->dst, giv, ivsize); - scatterwalk_crypto_chain(reqctx->dst, odst, vdst == giv + ivsize, 2); - - dst = reqctx->dst; - } - - ablkcipher_request_set_crypt(subreq, reqctx->src, dst, - req->creq.nbytes + ivsize, - req->creq.info); - - memcpy(req->creq.info, ctx->salt, ivsize); - - len = ivsize; - if (ivsize > sizeof(u64)) { - memset(req->giv, 0, ivsize - sizeof(u64)); - len = sizeof(u64); - } - seq = cpu_to_be64(req->seq); - memcpy(req->giv + ivsize - len, &seq, len); - - err = crypto_ablkcipher_encrypt(subreq); - if (err) - goto out; - - if (giv != req->giv) - eseqiv_complete2(req); - -out: - return err; -} - -static int eseqiv_init(struct crypto_tfm *tfm) -{ - struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm); - struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv); - unsigned long alignmask; - unsigned int reqsize; - int err; - - spin_lock_init(&ctx->lock); - - alignmask = crypto_tfm_ctx_alignment() - 1; - reqsize = sizeof(struct eseqiv_request_ctx); - - if (alignmask & reqsize) { - alignmask &= reqsize; - alignmask--; - } - - alignmask = ~alignmask; - alignmask &= crypto_ablkcipher_alignmask(geniv); - - reqsize += alignmask; - reqsize += crypto_ablkcipher_ivsize(geniv); - reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment()); - - ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx); - - tfm->crt_ablkcipher.reqsize = reqsize + - sizeof(struct ablkcipher_request); - - err = 0; - if (!crypto_get_default_rng()) { - crypto_ablkcipher_crt(geniv)->givencrypt = eseqiv_givencrypt; - err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt, - crypto_ablkcipher_ivsize(geniv)); - crypto_put_default_rng(); - } - - return err ?: skcipher_geniv_init(tfm); -} - -static struct crypto_template eseqiv_tmpl; - -static struct crypto_instance *eseqiv_alloc(struct rtattr **tb) -{ - struct crypto_instance *inst; - int err; - - inst = skcipher_geniv_alloc(&eseqiv_tmpl, tb, 0, 0); - if (IS_ERR(inst)) - goto out; - - err = -EINVAL; - if (inst->alg.cra_ablkcipher.ivsize != inst->alg.cra_blocksize) - goto free_inst; - - inst->alg.cra_init = eseqiv_init; - inst->alg.cra_exit = skcipher_geniv_exit; - - inst->alg.cra_ctxsize = sizeof(struct eseqiv_ctx); - inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize; - -out: - return inst; - -free_inst: - skcipher_geniv_free(inst); - inst = ERR_PTR(err); - goto out; -} - -static struct crypto_template eseqiv_tmpl = { - .name = "eseqiv", - .alloc = eseqiv_alloc, - .free = skcipher_geniv_free, - .module = THIS_MODULE, -}; - -static int __init eseqiv_module_init(void) -{ - return crypto_register_template(&eseqiv_tmpl); -} - -static void __exit eseqiv_module_exit(void) -{ - crypto_unregister_template(&eseqiv_tmpl); -} - -module_init(eseqiv_module_init); -module_exit(eseqiv_module_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Encrypted Sequence Number IV Generator"); -MODULE_ALIAS_CRYPTO("eseqiv"); diff --git a/crypto/gcm.c b/crypto/gcm.c index bec329b3de8d..70a892e87ccb 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -29,7 +29,7 @@ struct gcm_instance_ctx { }; struct crypto_gcm_ctx { - struct crypto_ablkcipher *ctr; + struct crypto_skcipher *ctr; struct crypto_ahash *ghash; }; @@ -50,7 +50,7 @@ struct crypto_rfc4543_instance_ctx { struct crypto_rfc4543_ctx { struct crypto_aead *child; - struct crypto_blkcipher *null; + struct crypto_skcipher *null; u8 nonce[4]; }; @@ -74,7 +74,7 @@ struct crypto_gcm_req_priv_ctx { struct crypto_gcm_ghash_ctx ghash_ctx; union { struct ahash_request ahreq; - struct ablkcipher_request abreq; + struct skcipher_request skreq; } u; }; @@ -114,7 +114,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_ahash *ghash = ctx->ghash; - struct crypto_ablkcipher *ctr = ctx->ctr; + struct crypto_skcipher *ctr = ctx->ctr; struct { be128 hash; u8 iv[8]; @@ -122,35 +122,35 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, struct crypto_gcm_setkey_result result; struct scatterlist sg[1]; - struct ablkcipher_request req; + struct skcipher_request req; } *data; int err; - crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); - crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) & - CRYPTO_TFM_REQ_MASK); - err = crypto_ablkcipher_setkey(ctr, key, keylen); - crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) & + crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(ctr, key, keylen); + crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) & CRYPTO_TFM_RES_MASK); if (err) return err; - data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr), + data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr), GFP_KERNEL); if (!data) return -ENOMEM; init_completion(&data->result.completion); sg_init_one(data->sg, &data->hash, sizeof(data->hash)); - ablkcipher_request_set_tfm(&data->req, ctr); - ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | - CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_gcm_setkey_done, - &data->result); - ablkcipher_request_set_crypt(&data->req, data->sg, data->sg, - sizeof(data->hash), data->iv); - - err = crypto_ablkcipher_encrypt(&data->req); + skcipher_request_set_tfm(&data->req, ctr); + skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | + CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_gcm_setkey_done, + &data->result); + skcipher_request_set_crypt(&data->req, data->sg, data->sg, + sizeof(data->hash), data->iv); + + err = crypto_skcipher_encrypt(&data->req); if (err == -EINPROGRESS || err == -EBUSY) { err = wait_for_completion_interruptible( &data->result.completion); @@ -223,13 +223,13 @@ static void crypto_gcm_init_crypt(struct aead_request *req, struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ablkcipher_request *ablk_req = &pctx->u.abreq; + struct skcipher_request *skreq = &pctx->u.skreq; struct scatterlist *dst; dst = req->src == req->dst ? pctx->src : pctx->dst; - ablkcipher_request_set_tfm(ablk_req, ctx->ctr); - ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, + skcipher_request_set_tfm(skreq, ctx->ctr); + skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + sizeof(pctx->auth_tag), pctx->iv); } @@ -494,14 +494,14 @@ out: static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ablkcipher_request *abreq = &pctx->u.abreq; + struct skcipher_request *skreq = &pctx->u.skreq; u32 flags = aead_request_flags(req); crypto_gcm_init_common(req); crypto_gcm_init_crypt(req, req->cryptlen); - ablkcipher_request_set_callback(abreq, flags, gcm_encrypt_done, req); + skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); - return crypto_ablkcipher_encrypt(abreq) ?: + return crypto_skcipher_encrypt(skreq) ?: gcm_encrypt_continue(req, flags); } @@ -533,12 +533,12 @@ static void gcm_decrypt_done(struct crypto_async_request *areq, int err) static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ablkcipher_request *abreq = &pctx->u.abreq; + struct skcipher_request *skreq = &pctx->u.skreq; struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; crypto_gcm_init_crypt(req, gctx->cryptlen); - ablkcipher_request_set_callback(abreq, flags, gcm_decrypt_done, req); - return crypto_ablkcipher_decrypt(abreq) ?: crypto_gcm_verify(req); + skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); + return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); } static int crypto_gcm_decrypt(struct aead_request *req) @@ -566,7 +566,7 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm) struct aead_instance *inst = aead_alg_instance(tfm); struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); - struct crypto_ablkcipher *ctr; + struct crypto_skcipher *ctr; struct crypto_ahash *ghash; unsigned long align; int err; @@ -575,7 +575,7 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm) if (IS_ERR(ghash)) return PTR_ERR(ghash); - ctr = crypto_spawn_skcipher(&ictx->ctr); + ctr = crypto_spawn_skcipher2(&ictx->ctr); err = PTR_ERR(ctr); if (IS_ERR(ctr)) goto err_free_hash; @@ -587,8 +587,8 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm) align &= ~(crypto_tfm_ctx_alignment() - 1); crypto_aead_set_reqsize(tfm, align + offsetof(struct crypto_gcm_req_priv_ctx, u) + - max(sizeof(struct ablkcipher_request) + - crypto_ablkcipher_reqsize(ctr), + max(sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(ctr), sizeof(struct ahash_request) + crypto_ahash_reqsize(ghash))); @@ -604,7 +604,7 @@ static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_ahash(ctx->ghash); - crypto_free_ablkcipher(ctx->ctr); + crypto_free_skcipher(ctx->ctr); } static void crypto_gcm_free(struct aead_instance *inst) @@ -624,7 +624,7 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, { struct crypto_attr_type *algt; struct aead_instance *inst; - struct crypto_alg *ctr; + struct skcipher_alg *ctr; struct crypto_alg *ghash_alg; struct hash_alg_common *ghash; struct gcm_instance_ctx *ctx; @@ -639,7 +639,9 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_AHASH_MASK); + CRYPTO_ALG_TYPE_AHASH_MASK | + crypto_requires_sync(algt->type, + algt->mask)); if (IS_ERR(ghash_alg)) return PTR_ERR(ghash_alg); @@ -661,41 +663,42 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, goto err_drop_ghash; crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst)); - err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, - crypto_requires_sync(algt->type, - algt->mask)); + err = crypto_grab_skcipher2(&ctx->ctr, ctr_name, 0, + crypto_requires_sync(algt->type, + algt->mask)); if (err) goto err_drop_ghash; - ctr = crypto_skcipher_spawn_alg(&ctx->ctr); + ctr = crypto_spawn_skcipher_alg(&ctx->ctr); /* We only support 16-byte blocks. */ - if (ctr->cra_ablkcipher.ivsize != 16) + if (crypto_skcipher_alg_ivsize(ctr) != 16) goto out_put_ctr; /* Not a stream cipher? */ err = -EINVAL; - if (ctr->cra_blocksize != 1) + if (ctr->base.cra_blocksize != 1) goto out_put_ctr; err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "gcm_base(%s,%s)", ctr->cra_driver_name, + "gcm_base(%s,%s)", ctr->base.cra_driver_name, ghash_alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_put_ctr; memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); - inst->alg.base.cra_flags = (ghash->base.cra_flags | ctr->cra_flags) & - CRYPTO_ALG_ASYNC; + inst->alg.base.cra_flags = (ghash->base.cra_flags | + ctr->base.cra_flags) & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = (ghash->base.cra_priority + - ctr->cra_priority) / 2; + ctr->base.cra_priority) / 2; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = ghash->base.cra_alignmask | - ctr->cra_alignmask; + ctr->base.cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); inst->alg.ivsize = 12; + inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr); inst->alg.maxauthsize = 16; inst->alg.init = crypto_gcm_init_tfm; inst->alg.exit = crypto_gcm_exit_tfm; @@ -980,6 +983,7 @@ static int crypto_rfc4106_create(struct crypto_template *tmpl, inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); inst->alg.ivsize = 8; + inst->alg.chunksize = crypto_aead_alg_chunksize(alg); inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); inst->alg.init = crypto_rfc4106_init_tfm; @@ -1084,11 +1088,13 @@ static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) unsigned int authsize = crypto_aead_authsize(aead); unsigned int nbytes = req->assoclen + req->cryptlen - (enc ? 0 : authsize); - struct blkcipher_desc desc = { - .tfm = ctx->null, - }; + SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null); - return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes); + skcipher_request_set_tfm(nreq, ctx->null); + skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL); + skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL); + + return crypto_skcipher_encrypt(nreq); } static int crypto_rfc4543_encrypt(struct aead_request *req) @@ -1108,7 +1114,7 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) struct crypto_aead_spawn *spawn = &ictx->aead; struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_aead *aead; - struct crypto_blkcipher *null; + struct crypto_skcipher *null; unsigned long align; int err = 0; @@ -1116,7 +1122,7 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) if (IS_ERR(aead)) return PTR_ERR(aead); - null = crypto_get_default_null_skcipher(); + null = crypto_get_default_null_skcipher2(); err = PTR_ERR(null); if (IS_ERR(null)) goto err_free_aead; @@ -1144,7 +1150,7 @@ static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_aead(ctx->child); - crypto_put_default_null_skcipher(); + crypto_put_default_null_skcipher2(); } static void crypto_rfc4543_free(struct aead_instance *inst) @@ -1219,6 +1225,7 @@ static int crypto_rfc4543_create(struct crypto_template *tmpl, inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); inst->alg.ivsize = 8; + inst->alg.chunksize = crypto_aead_alg_chunksize(alg); inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); inst->alg.init = crypto_rfc4543_init_tfm; diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 597cedd3531c..c4938497eedb 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -87,24 +87,28 @@ void jent_memcpy(void *dest, const void *src, unsigned int n) memcpy(dest, src, n); } +/* + * Obtain a high-resolution time stamp value. The time stamp is used to measure + * the execution time of a given code path and its variations. Hence, the time + * stamp must have a sufficiently high resolution. + * + * Note, if the function returns zero because a given architecture does not + * implement a high-resolution time stamp, the RNG code's runtime test + * will detect it and will not produce output. + */ void jent_get_nstime(__u64 *out) { - struct timespec ts; __u64 tmp = 0; tmp = random_get_entropy(); /* - * If random_get_entropy does not return a value (which is possible on, - * for example, MIPS), invoke __getnstimeofday + * If random_get_entropy does not return a value, i.e. it is not + * implemented for a given architecture, use a clock source. * hoping that there are timers we can work with. */ - if ((0 == tmp) && - (0 == __getnstimeofday(&ts))) { - tmp = ts.tv_sec; - tmp = tmp << 32; - tmp = tmp | ts.tv_nsec; - } + if (tmp == 0) + tmp = ktime_get_ns(); *out = tmp; } diff --git a/crypto/kpp.c b/crypto/kpp.c new file mode 100644 index 000000000000..d36ce05eee43 --- /dev/null +++ b/crypto/kpp.c @@ -0,0 +1,123 @@ +/* + * Key-agreement Protocol Primitives (KPP) + * + * Copyright (c) 2016, Intel Corporation + * Authors: Salvatore Benedetto + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#ifdef CONFIG_NET +static int crypto_kpp_report(struct sk_buff *skb, struct crypto_alg *alg) +{ + struct crypto_report_kpp rkpp; + + strncpy(rkpp.type, "kpp", sizeof(rkpp.type)); + + if (nla_put(skb, CRYPTOCFGA_REPORT_KPP, + sizeof(struct crypto_report_kpp), &rkpp)) + goto nla_put_failure; + return 0; + +nla_put_failure: + return -EMSGSIZE; +} +#else +static int crypto_kpp_report(struct sk_buff *skb, struct crypto_alg *alg) +{ + return -ENOSYS; +} +#endif + +static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg) + __attribute__ ((unused)); + +static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg) +{ + seq_puts(m, "type : kpp\n"); +} + +static void crypto_kpp_exit_tfm(struct crypto_tfm *tfm) +{ + struct crypto_kpp *kpp = __crypto_kpp_tfm(tfm); + struct kpp_alg *alg = crypto_kpp_alg(kpp); + + alg->exit(kpp); +} + +static int crypto_kpp_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_kpp *kpp = __crypto_kpp_tfm(tfm); + struct kpp_alg *alg = crypto_kpp_alg(kpp); + + if (alg->exit) + kpp->base.exit = crypto_kpp_exit_tfm; + + if (alg->init) + return alg->init(kpp); + + return 0; +} + +static const struct crypto_type crypto_kpp_type = { + .extsize = crypto_alg_extsize, + .init_tfm = crypto_kpp_init_tfm, +#ifdef CONFIG_PROC_FS + .show = crypto_kpp_show, +#endif + .report = crypto_kpp_report, + .maskclear = ~CRYPTO_ALG_TYPE_MASK, + .maskset = CRYPTO_ALG_TYPE_MASK, + .type = CRYPTO_ALG_TYPE_KPP, + .tfmsize = offsetof(struct crypto_kpp, base), +}; + +struct crypto_kpp *crypto_alloc_kpp(const char *alg_name, u32 type, u32 mask) +{ + return crypto_alloc_tfm(alg_name, &crypto_kpp_type, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_alloc_kpp); + +static void kpp_prepare_alg(struct kpp_alg *alg) +{ + struct crypto_alg *base = &alg->base; + + base->cra_type = &crypto_kpp_type; + base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; + base->cra_flags |= CRYPTO_ALG_TYPE_KPP; +} + +int crypto_register_kpp(struct kpp_alg *alg) +{ + struct crypto_alg *base = &alg->base; + + kpp_prepare_alg(alg); + return crypto_register_alg(base); +} +EXPORT_SYMBOL_GPL(crypto_register_kpp); + +void crypto_unregister_kpp(struct kpp_alg *alg) +{ + crypto_unregister_alg(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_unregister_kpp); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Key-agreement Protocol Primitives"); diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c index c4eb9da49d4f..86fb59b109a9 100644 --- a/crypto/mcryptd.c +++ b/crypto/mcryptd.c @@ -41,7 +41,7 @@ struct mcryptd_flush_list { static struct mcryptd_flush_list __percpu *mcryptd_flist; struct hashd_instance_ctx { - struct crypto_shash_spawn spawn; + struct crypto_ahash_spawn spawn; struct mcryptd_queue *queue; }; @@ -272,18 +272,18 @@ static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm) { struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); - struct crypto_shash_spawn *spawn = &ictx->spawn; + struct crypto_ahash_spawn *spawn = &ictx->spawn; struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_shash *hash; + struct crypto_ahash *hash; - hash = crypto_spawn_shash(spawn); + hash = crypto_spawn_ahash(spawn); if (IS_ERR(hash)) return PTR_ERR(hash); ctx->child = hash; crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct mcryptd_hash_request_ctx) + - crypto_shash_descsize(hash)); + crypto_ahash_reqsize(hash)); return 0; } @@ -291,21 +291,21 @@ static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm) { struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_shash(ctx->child); + crypto_free_ahash(ctx->child); } static int mcryptd_hash_setkey(struct crypto_ahash *parent, const u8 *key, unsigned int keylen) { struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); - struct crypto_shash *child = ctx->child; + struct crypto_ahash *child = ctx->child; int err; - crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & + crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) & CRYPTO_TFM_REQ_MASK); - err = crypto_shash_setkey(child, key, keylen); - crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) & + err = crypto_ahash_setkey(child, key, keylen); + crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) & CRYPTO_TFM_RES_MASK); return err; } @@ -331,20 +331,20 @@ static int mcryptd_hash_enqueue(struct ahash_request *req, static void mcryptd_hash_init(struct crypto_async_request *req_async, int err) { struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_shash *child = ctx->child; + struct crypto_ahash *child = ctx->child; struct ahash_request *req = ahash_request_cast(req_async); struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); - struct shash_desc *desc = &rctx->desc; + struct ahash_request *desc = &rctx->areq; if (unlikely(err == -EINPROGRESS)) goto out; - desc->tfm = child; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; + ahash_request_set_tfm(desc, child); + ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP, + rctx->complete, req_async); - err = crypto_shash_init(desc); - - req->base.complete = rctx->complete; + rctx->out = req->result; + err = crypto_ahash_init(desc); out: local_bh_disable(); @@ -365,7 +365,8 @@ static void mcryptd_hash_update(struct crypto_async_request *req_async, int err) if (unlikely(err == -EINPROGRESS)) goto out; - err = shash_ahash_mcryptd_update(req, &rctx->desc); + rctx->out = req->result; + err = ahash_mcryptd_update(&rctx->areq); if (err) { req->base.complete = rctx->complete; goto out; @@ -391,7 +392,8 @@ static void mcryptd_hash_final(struct crypto_async_request *req_async, int err) if (unlikely(err == -EINPROGRESS)) goto out; - err = shash_ahash_mcryptd_final(req, &rctx->desc); + rctx->out = req->result; + err = ahash_mcryptd_final(&rctx->areq); if (err) { req->base.complete = rctx->complete; goto out; @@ -416,8 +418,8 @@ static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err) if (unlikely(err == -EINPROGRESS)) goto out; - - err = shash_ahash_mcryptd_finup(req, &rctx->desc); + rctx->out = req->result; + err = ahash_mcryptd_finup(&rctx->areq); if (err) { req->base.complete = rctx->complete; @@ -439,25 +441,21 @@ static int mcryptd_hash_finup_enqueue(struct ahash_request *req) static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err) { struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_shash *child = ctx->child; + struct crypto_ahash *child = ctx->child; struct ahash_request *req = ahash_request_cast(req_async); struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); - struct shash_desc *desc = &rctx->desc; + struct ahash_request *desc = &rctx->areq; if (unlikely(err == -EINPROGRESS)) goto out; - desc->tfm = child; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* check this again */ - - err = shash_ahash_mcryptd_digest(req, desc); + ahash_request_set_tfm(desc, child); + ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP, + rctx->complete, req_async); - if (err) { - req->base.complete = rctx->complete; - goto out; - } + rctx->out = req->result; + err = ahash_mcryptd_digest(desc); - return; out: local_bh_disable(); rctx->complete(&req->base, err); @@ -473,14 +471,14 @@ static int mcryptd_hash_export(struct ahash_request *req, void *out) { struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); - return crypto_shash_export(&rctx->desc, out); + return crypto_ahash_export(&rctx->areq, out); } static int mcryptd_hash_import(struct ahash_request *req, const void *in) { struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); - return crypto_shash_import(&rctx->desc, in); + return crypto_ahash_import(&rctx->areq, in); } static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, @@ -488,7 +486,7 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, { struct hashd_instance_ctx *ctx; struct ahash_instance *inst; - struct shash_alg *salg; + struct hash_alg_common *halg; struct crypto_alg *alg; u32 type = 0; u32 mask = 0; @@ -496,11 +494,11 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, mcryptd_check_internal(tb, &type, &mask); - salg = shash_attr_alg(tb[1], type, mask); - if (IS_ERR(salg)) - return PTR_ERR(salg); + halg = ahash_attr_alg(tb[1], type, mask); + if (IS_ERR(halg)) + return PTR_ERR(halg); - alg = &salg->base; + alg = &halg->base; pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name); inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(), sizeof(*ctx)); @@ -511,7 +509,7 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, ctx = ahash_instance_ctx(inst); ctx->queue = queue; - err = crypto_init_shash_spawn(&ctx->spawn, salg, + err = crypto_init_ahash_spawn(&ctx->spawn, halg, ahash_crypto_instance(inst)); if (err) goto out_free_inst; @@ -521,8 +519,8 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, type |= CRYPTO_ALG_INTERNAL; inst->alg.halg.base.cra_flags = type; - inst->alg.halg.digestsize = salg->digestsize; - inst->alg.halg.statesize = salg->statesize; + inst->alg.halg.digestsize = halg->digestsize; + inst->alg.halg.statesize = halg->statesize; inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx); inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm; @@ -539,7 +537,7 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, err = ahash_register_instance(tmpl, inst); if (err) { - crypto_drop_shash(&ctx->spawn); + crypto_drop_ahash(&ctx->spawn); out_free_inst: kfree(inst); } @@ -575,7 +573,7 @@ static void mcryptd_free(struct crypto_instance *inst) switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_AHASH: - crypto_drop_shash(&hctx->spawn); + crypto_drop_ahash(&hctx->spawn); kfree(ahash_instance(inst)); return; default: @@ -612,55 +610,38 @@ struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name, } EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash); -int shash_ahash_mcryptd_digest(struct ahash_request *req, - struct shash_desc *desc) +int ahash_mcryptd_digest(struct ahash_request *desc) { int err; - err = crypto_shash_init(desc) ?: - shash_ahash_mcryptd_finup(req, desc); + err = crypto_ahash_init(desc) ?: + ahash_mcryptd_finup(desc); return err; } -EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest); -int shash_ahash_mcryptd_update(struct ahash_request *req, - struct shash_desc *desc) +int ahash_mcryptd_update(struct ahash_request *desc) { - struct crypto_shash *tfm = desc->tfm; - struct shash_alg *shash = crypto_shash_alg(tfm); - /* alignment is to be done by multi-buffer crypto algorithm if needed */ - return shash->update(desc, NULL, 0); + return crypto_ahash_update(desc); } -EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update); -int shash_ahash_mcryptd_finup(struct ahash_request *req, - struct shash_desc *desc) +int ahash_mcryptd_finup(struct ahash_request *desc) { - struct crypto_shash *tfm = desc->tfm; - struct shash_alg *shash = crypto_shash_alg(tfm); - /* alignment is to be done by multi-buffer crypto algorithm if needed */ - return shash->finup(desc, NULL, 0, req->result); + return crypto_ahash_finup(desc); } -EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup); -int shash_ahash_mcryptd_final(struct ahash_request *req, - struct shash_desc *desc) +int ahash_mcryptd_final(struct ahash_request *desc) { - struct crypto_shash *tfm = desc->tfm; - struct shash_alg *shash = crypto_shash_alg(tfm); - /* alignment is to be done by multi-buffer crypto algorithm if needed */ - return shash->final(desc, req->result); + return crypto_ahash_final(desc); } -EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final); -struct crypto_shash *mcryptd_ahash_child(struct mcryptd_ahash *tfm) +struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm) { struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); @@ -668,12 +649,12 @@ struct crypto_shash *mcryptd_ahash_child(struct mcryptd_ahash *tfm) } EXPORT_SYMBOL_GPL(mcryptd_ahash_child); -struct shash_desc *mcryptd_shash_desc(struct ahash_request *req) +struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req) { struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); - return &rctx->desc; + return &rctx->areq; } -EXPORT_SYMBOL_GPL(mcryptd_shash_desc); +EXPORT_SYMBOL_GPL(mcryptd_ahash_desc); void mcryptd_free_ahash(struct mcryptd_ahash *tfm) { @@ -681,7 +662,6 @@ void mcryptd_free_ahash(struct mcryptd_ahash *tfm) } EXPORT_SYMBOL_GPL(mcryptd_free_ahash); - static int __init mcryptd_init(void) { int err, cpu; diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c index ead8dc0d084e..877019a6d3ea 100644 --- a/crypto/rsa-pkcs1pad.c +++ b/crypto/rsa-pkcs1pad.c @@ -92,60 +92,66 @@ static const struct rsa_asn1_template *rsa_lookup_asn1(const char *name) struct pkcs1pad_ctx { struct crypto_akcipher *child; - const char *hash_name; unsigned int key_size; }; struct pkcs1pad_inst_ctx { struct crypto_akcipher_spawn spawn; - const char *hash_name; + const struct rsa_asn1_template *digest_info; }; struct pkcs1pad_request { - struct akcipher_request child_req; - - struct scatterlist in_sg[3], out_sg[2]; + struct scatterlist in_sg[2], out_sg[1]; uint8_t *in_buf, *out_buf; + struct akcipher_request child_req; }; static int pkcs1pad_set_pub_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); - int err, size; + int err; + + ctx->key_size = 0; err = crypto_akcipher_set_pub_key(ctx->child, key, keylen); + if (err) + return err; - if (!err) { - /* Find out new modulus size from rsa implementation */ - size = crypto_akcipher_maxsize(ctx->child); + /* Find out new modulus size from rsa implementation */ + err = crypto_akcipher_maxsize(ctx->child); + if (err < 0) + return err; - ctx->key_size = size > 0 ? size : 0; - if (size <= 0) - err = size; - } + if (err > PAGE_SIZE) + return -ENOTSUPP; - return err; + ctx->key_size = err; + return 0; } static int pkcs1pad_set_priv_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); - int err, size; + int err; + + ctx->key_size = 0; err = crypto_akcipher_set_priv_key(ctx->child, key, keylen); + if (err) + return err; - if (!err) { - /* Find out new modulus size from rsa implementation */ - size = crypto_akcipher_maxsize(ctx->child); + /* Find out new modulus size from rsa implementation */ + err = crypto_akcipher_maxsize(ctx->child); + if (err < 0) + return err; - ctx->key_size = size > 0 ? size : 0; - if (size <= 0) - err = size; - } + if (err > PAGE_SIZE) + return -ENOTSUPP; - return err; + ctx->key_size = err; + return 0; } static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm) @@ -164,19 +170,10 @@ static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm) static void pkcs1pad_sg_set_buf(struct scatterlist *sg, void *buf, size_t len, struct scatterlist *next) { - int nsegs = next ? 1 : 0; - - if (offset_in_page(buf) + len <= PAGE_SIZE) { - nsegs += 1; - sg_init_table(sg, nsegs); - sg_set_buf(sg, buf, len); - } else { - nsegs += 2; - sg_init_table(sg, nsegs); - sg_set_buf(sg + 0, buf, PAGE_SIZE - offset_in_page(buf)); - sg_set_buf(sg + 1, buf + PAGE_SIZE - offset_in_page(buf), - offset_in_page(buf) + len - PAGE_SIZE); - } + int nsegs = next ? 2 : 1; + + sg_init_table(sg, nsegs); + sg_set_buf(sg, buf, len); if (next) sg_chain(sg, nsegs, next); @@ -187,37 +184,36 @@ static int pkcs1pad_encrypt_sign_complete(struct akcipher_request *req, int err) struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req); - size_t pad_len = ctx->key_size - req_ctx->child_req.dst_len; - size_t chunk_len, pad_left; - struct sg_mapping_iter miter; - - if (!err) { - if (pad_len) { - sg_miter_start(&miter, req->dst, - sg_nents_for_len(req->dst, pad_len), - SG_MITER_ATOMIC | SG_MITER_TO_SG); - - pad_left = pad_len; - while (pad_left) { - sg_miter_next(&miter); - - chunk_len = min(miter.length, pad_left); - memset(miter.addr, 0, chunk_len); - pad_left -= chunk_len; - } - - sg_miter_stop(&miter); - } - - sg_pcopy_from_buffer(req->dst, - sg_nents_for_len(req->dst, ctx->key_size), - req_ctx->out_buf, req_ctx->child_req.dst_len, - pad_len); - } + unsigned int pad_len; + unsigned int len; + u8 *out_buf; + + if (err) + goto out; + + len = req_ctx->child_req.dst_len; + pad_len = ctx->key_size - len; + + /* Four billion to one */ + if (likely(!pad_len)) + goto out; + + out_buf = kzalloc(ctx->key_size, GFP_ATOMIC); + err = -ENOMEM; + if (!out_buf) + goto out; + + sg_copy_to_buffer(req->dst, sg_nents_for_len(req->dst, len), + out_buf + pad_len, len); + sg_copy_from_buffer(req->dst, + sg_nents_for_len(req->dst, ctx->key_size), + out_buf, ctx->key_size); + kzfree(out_buf); + +out: req->dst_len = ctx->key_size; kfree(req_ctx->in_buf); - kzfree(req_ctx->out_buf); return err; } @@ -257,21 +253,8 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) return -EOVERFLOW; } - if (ctx->key_size > PAGE_SIZE) - return -ENOTSUPP; - - /* - * Replace both input and output to add the padding in the input and - * the potential missing leading zeros in the output. - */ - req_ctx->child_req.src = req_ctx->in_sg; - req_ctx->child_req.src_len = ctx->key_size - 1; - req_ctx->child_req.dst = req_ctx->out_sg; - req_ctx->child_req.dst_len = ctx->key_size; - req_ctx->in_buf = kmalloc(ctx->key_size - 1 - req->src_len, - (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC); + GFP_KERNEL); if (!req_ctx->in_buf) return -ENOMEM; @@ -284,9 +267,7 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) pkcs1pad_sg_set_buf(req_ctx->in_sg, req_ctx->in_buf, ctx->key_size - 1 - req->src_len, req->src); - req_ctx->out_buf = kmalloc(ctx->key_size, - (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC); + req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL); if (!req_ctx->out_buf) { kfree(req_ctx->in_buf); return -ENOMEM; @@ -299,6 +280,10 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) akcipher_request_set_callback(&req_ctx->child_req, req->base.flags, pkcs1pad_encrypt_sign_complete_cb, req); + /* Reuse output buffer */ + akcipher_request_set_crypt(&req_ctx->child_req, req_ctx->in_sg, + req->dst, ctx->key_size - 1, req->dst_len); + err = crypto_akcipher_encrypt(&req_ctx->child_req); if (err != -EINPROGRESS && (err != -EBUSY || @@ -380,18 +365,7 @@ static int pkcs1pad_decrypt(struct akcipher_request *req) if (!ctx->key_size || req->src_len != ctx->key_size) return -EINVAL; - if (ctx->key_size > PAGE_SIZE) - return -ENOTSUPP; - - /* Reuse input buffer, output to a new buffer */ - req_ctx->child_req.src = req->src; - req_ctx->child_req.src_len = req->src_len; - req_ctx->child_req.dst = req_ctx->out_sg; - req_ctx->child_req.dst_len = ctx->key_size ; - - req_ctx->out_buf = kmalloc(ctx->key_size, - (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC); + req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL); if (!req_ctx->out_buf) return -ENOMEM; @@ -402,6 +376,11 @@ static int pkcs1pad_decrypt(struct akcipher_request *req) akcipher_request_set_callback(&req_ctx->child_req, req->base.flags, pkcs1pad_decrypt_complete_cb, req); + /* Reuse input buffer, output to a new buffer */ + akcipher_request_set_crypt(&req_ctx->child_req, req->src, + req_ctx->out_sg, req->src_len, + ctx->key_size); + err = crypto_akcipher_decrypt(&req_ctx->child_req); if (err != -EINPROGRESS && (err != -EBUSY || @@ -416,20 +395,16 @@ static int pkcs1pad_sign(struct akcipher_request *req) struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req); - const struct rsa_asn1_template *digest_info = NULL; + struct akcipher_instance *inst = akcipher_alg_instance(tfm); + struct pkcs1pad_inst_ctx *ictx = akcipher_instance_ctx(inst); + const struct rsa_asn1_template *digest_info = ictx->digest_info; int err; unsigned int ps_end, digest_size = 0; if (!ctx->key_size) return -EINVAL; - if (ctx->hash_name) { - digest_info = rsa_lookup_asn1(ctx->hash_name); - if (!digest_info) - return -EINVAL; - - digest_size = digest_info->size; - } + digest_size = digest_info->size; if (req->src_len + digest_size > ctx->key_size - 11) return -EOVERFLOW; @@ -439,21 +414,8 @@ static int pkcs1pad_sign(struct akcipher_request *req) return -EOVERFLOW; } - if (ctx->key_size > PAGE_SIZE) - return -ENOTSUPP; - - /* - * Replace both input and output to add the padding in the input and - * the potential missing leading zeros in the output. - */ - req_ctx->child_req.src = req_ctx->in_sg; - req_ctx->child_req.src_len = ctx->key_size - 1; - req_ctx->child_req.dst = req_ctx->out_sg; - req_ctx->child_req.dst_len = ctx->key_size; - req_ctx->in_buf = kmalloc(ctx->key_size - 1 - req->src_len, - (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC); + GFP_KERNEL); if (!req_ctx->in_buf) return -ENOMEM; @@ -462,29 +424,20 @@ static int pkcs1pad_sign(struct akcipher_request *req) memset(req_ctx->in_buf + 1, 0xff, ps_end - 1); req_ctx->in_buf[ps_end] = 0x00; - if (digest_info) { - memcpy(req_ctx->in_buf + ps_end + 1, digest_info->data, - digest_info->size); - } + memcpy(req_ctx->in_buf + ps_end + 1, digest_info->data, + digest_info->size); pkcs1pad_sg_set_buf(req_ctx->in_sg, req_ctx->in_buf, ctx->key_size - 1 - req->src_len, req->src); - req_ctx->out_buf = kmalloc(ctx->key_size, - (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC); - if (!req_ctx->out_buf) { - kfree(req_ctx->in_buf); - return -ENOMEM; - } - - pkcs1pad_sg_set_buf(req_ctx->out_sg, req_ctx->out_buf, - ctx->key_size, NULL); - akcipher_request_set_tfm(&req_ctx->child_req, ctx->child); akcipher_request_set_callback(&req_ctx->child_req, req->base.flags, pkcs1pad_encrypt_sign_complete_cb, req); + /* Reuse output buffer */ + akcipher_request_set_crypt(&req_ctx->child_req, req_ctx->in_sg, + req->dst, ctx->key_size - 1, req->dst_len); + err = crypto_akcipher_sign(&req_ctx->child_req); if (err != -EINPROGRESS && (err != -EBUSY || @@ -499,56 +452,58 @@ static int pkcs1pad_verify_complete(struct akcipher_request *req, int err) struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req); - const struct rsa_asn1_template *digest_info; + struct akcipher_instance *inst = akcipher_alg_instance(tfm); + struct pkcs1pad_inst_ctx *ictx = akcipher_instance_ctx(inst); + const struct rsa_asn1_template *digest_info = ictx->digest_info; + unsigned int dst_len; unsigned int pos; - - if (err == -EOVERFLOW) - /* Decrypted value had no leading 0 byte */ - err = -EINVAL; + u8 *out_buf; if (err) goto done; - if (req_ctx->child_req.dst_len != ctx->key_size - 1) { - err = -EINVAL; + err = -EINVAL; + dst_len = req_ctx->child_req.dst_len; + if (dst_len < ctx->key_size - 1) goto done; + + out_buf = req_ctx->out_buf; + if (dst_len == ctx->key_size) { + if (out_buf[0] != 0x00) + /* Decrypted value had no leading 0 byte */ + goto done; + + dst_len--; + out_buf++; } err = -EBADMSG; - if (req_ctx->out_buf[0] != 0x01) + if (out_buf[0] != 0x01) goto done; - for (pos = 1; pos < req_ctx->child_req.dst_len; pos++) - if (req_ctx->out_buf[pos] != 0xff) + for (pos = 1; pos < dst_len; pos++) + if (out_buf[pos] != 0xff) break; - if (pos < 9 || pos == req_ctx->child_req.dst_len || - req_ctx->out_buf[pos] != 0x00) + if (pos < 9 || pos == dst_len || out_buf[pos] != 0x00) goto done; pos++; - if (ctx->hash_name) { - digest_info = rsa_lookup_asn1(ctx->hash_name); - if (!digest_info) - goto done; - - if (memcmp(req_ctx->out_buf + pos, digest_info->data, - digest_info->size)) - goto done; + if (memcmp(out_buf + pos, digest_info->data, digest_info->size)) + goto done; - pos += digest_info->size; - } + pos += digest_info->size; err = 0; - if (req->dst_len < req_ctx->child_req.dst_len - pos) + if (req->dst_len < dst_len - pos) err = -EOVERFLOW; - req->dst_len = req_ctx->child_req.dst_len - pos; + req->dst_len = dst_len - pos; if (!err) sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst, req->dst_len), - req_ctx->out_buf + pos, req->dst_len); + out_buf + pos, req->dst_len); done: kzfree(req_ctx->out_buf); @@ -588,18 +543,7 @@ static int pkcs1pad_verify(struct akcipher_request *req) if (!ctx->key_size || req->src_len < ctx->key_size) return -EINVAL; - if (ctx->key_size > PAGE_SIZE) - return -ENOTSUPP; - - /* Reuse input buffer, output to a new buffer */ - req_ctx->child_req.src = req->src; - req_ctx->child_req.src_len = req->src_len; - req_ctx->child_req.dst = req_ctx->out_sg; - req_ctx->child_req.dst_len = ctx->key_size; - - req_ctx->out_buf = kmalloc(ctx->key_size, - (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC); + req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL); if (!req_ctx->out_buf) return -ENOMEM; @@ -610,6 +554,11 @@ static int pkcs1pad_verify(struct akcipher_request *req) akcipher_request_set_callback(&req_ctx->child_req, req->base.flags, pkcs1pad_verify_complete_cb, req); + /* Reuse input buffer, output to a new buffer */ + akcipher_request_set_crypt(&req_ctx->child_req, req->src, + req_ctx->out_sg, req->src_len, + ctx->key_size); + err = crypto_akcipher_verify(&req_ctx->child_req); if (err != -EINPROGRESS && (err != -EBUSY || @@ -626,12 +575,11 @@ static int pkcs1pad_init_tfm(struct crypto_akcipher *tfm) struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); struct crypto_akcipher *child_tfm; - child_tfm = crypto_spawn_akcipher(akcipher_instance_ctx(inst)); + child_tfm = crypto_spawn_akcipher(&ictx->spawn); if (IS_ERR(child_tfm)) return PTR_ERR(child_tfm); ctx->child = child_tfm; - ctx->hash_name = ictx->hash_name; return 0; } @@ -648,12 +596,12 @@ static void pkcs1pad_free(struct akcipher_instance *inst) struct crypto_akcipher_spawn *spawn = &ctx->spawn; crypto_drop_akcipher(spawn); - kfree(ctx->hash_name); kfree(inst); } static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb) { + const struct rsa_asn1_template *digest_info; struct crypto_attr_type *algt; struct akcipher_instance *inst; struct pkcs1pad_inst_ctx *ctx; @@ -676,7 +624,11 @@ static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb) hash_name = crypto_attr_alg_name(tb[2]); if (IS_ERR(hash_name)) - hash_name = NULL; + return PTR_ERR(hash_name); + + digest_info = rsa_lookup_asn1(hash_name); + if (!digest_info) + return -EINVAL; inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); if (!inst) @@ -684,7 +636,7 @@ static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb) ctx = akcipher_instance_ctx(inst); spawn = &ctx->spawn; - ctx->hash_name = hash_name ? kstrdup(hash_name, GFP_KERNEL) : NULL; + ctx->digest_info = digest_info; crypto_set_spawn(&spawn->base, akcipher_crypto_instance(inst)); err = crypto_grab_akcipher(spawn, rsa_alg_name, 0, @@ -696,27 +648,14 @@ static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb) err = -ENAMETOOLONG; - if (!hash_name) { - if (snprintf(inst->alg.base.cra_name, - CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)", - rsa_alg->base.cra_name) >= - CRYPTO_MAX_ALG_NAME || - snprintf(inst->alg.base.cra_driver_name, - CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)", - rsa_alg->base.cra_driver_name) >= - CRYPTO_MAX_ALG_NAME) + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "pkcs1pad(%s,%s)", rsa_alg->base.cra_name, hash_name) >= + CRYPTO_MAX_ALG_NAME || + snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "pkcs1pad(%s,%s)", + rsa_alg->base.cra_driver_name, hash_name) >= + CRYPTO_MAX_ALG_NAME) goto out_drop_alg; - } else { - if (snprintf(inst->alg.base.cra_name, - CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s,%s)", - rsa_alg->base.cra_name, hash_name) >= - CRYPTO_MAX_ALG_NAME || - snprintf(inst->alg.base.cra_driver_name, - CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s,%s)", - rsa_alg->base.cra_driver_name, hash_name) >= - CRYPTO_MAX_ALG_NAME) - goto out_free_hash; - } inst->alg.base.cra_flags = rsa_alg->base.cra_flags & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = rsa_alg->base.cra_priority; @@ -738,12 +677,10 @@ static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb) err = akcipher_register_instance(tmpl, inst); if (err) - goto out_free_hash; + goto out_drop_alg; return 0; -out_free_hash: - kfree(ctx->hash_name); out_drop_alg: crypto_drop_akcipher(spawn); out_free_inst: diff --git a/crypto/rsa.c b/crypto/rsa.c index 77d737f52147..4c280b6a3ea9 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -10,16 +10,23 @@ */ #include +#include #include #include #include #include +struct rsa_mpi_key { + MPI n; + MPI e; + MPI d; +}; + /* * RSAEP function [RFC3447 sec 5.1.1] * c = m^e mod n; */ -static int _rsa_enc(const struct rsa_key *key, MPI c, MPI m) +static int _rsa_enc(const struct rsa_mpi_key *key, MPI c, MPI m) { /* (1) Validate 0 <= m < n */ if (mpi_cmp_ui(m, 0) < 0 || mpi_cmp(m, key->n) >= 0) @@ -33,7 +40,7 @@ static int _rsa_enc(const struct rsa_key *key, MPI c, MPI m) * RSADP function [RFC3447 sec 5.1.2] * m = c^d mod n; */ -static int _rsa_dec(const struct rsa_key *key, MPI m, MPI c) +static int _rsa_dec(const struct rsa_mpi_key *key, MPI m, MPI c) { /* (1) Validate 0 <= c < n */ if (mpi_cmp_ui(c, 0) < 0 || mpi_cmp(c, key->n) >= 0) @@ -47,7 +54,7 @@ static int _rsa_dec(const struct rsa_key *key, MPI m, MPI c) * RSASP1 function [RFC3447 sec 5.2.1] * s = m^d mod n */ -static int _rsa_sign(const struct rsa_key *key, MPI s, MPI m) +static int _rsa_sign(const struct rsa_mpi_key *key, MPI s, MPI m) { /* (1) Validate 0 <= m < n */ if (mpi_cmp_ui(m, 0) < 0 || mpi_cmp(m, key->n) >= 0) @@ -61,7 +68,7 @@ static int _rsa_sign(const struct rsa_key *key, MPI s, MPI m) * RSAVP1 function [RFC3447 sec 5.2.2] * m = s^e mod n; */ -static int _rsa_verify(const struct rsa_key *key, MPI m, MPI s) +static int _rsa_verify(const struct rsa_mpi_key *key, MPI m, MPI s) { /* (1) Validate 0 <= s < n */ if (mpi_cmp_ui(s, 0) < 0 || mpi_cmp(s, key->n) >= 0) @@ -71,7 +78,7 @@ static int _rsa_verify(const struct rsa_key *key, MPI m, MPI s) return mpi_powm(m, s, key->e, key->n); } -static inline struct rsa_key *rsa_get_key(struct crypto_akcipher *tfm) +static inline struct rsa_mpi_key *rsa_get_key(struct crypto_akcipher *tfm) { return akcipher_tfm_ctx(tfm); } @@ -79,7 +86,7 @@ static inline struct rsa_key *rsa_get_key(struct crypto_akcipher *tfm) static int rsa_enc(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - const struct rsa_key *pkey = rsa_get_key(tfm); + const struct rsa_mpi_key *pkey = rsa_get_key(tfm); MPI m, c = mpi_alloc(0); int ret = 0; int sign; @@ -101,7 +108,7 @@ static int rsa_enc(struct akcipher_request *req) if (ret) goto err_free_m; - ret = mpi_write_to_sgl(c, req->dst, &req->dst_len, &sign); + ret = mpi_write_to_sgl(c, req->dst, req->dst_len, &sign); if (ret) goto err_free_m; @@ -118,7 +125,7 @@ err_free_c: static int rsa_dec(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - const struct rsa_key *pkey = rsa_get_key(tfm); + const struct rsa_mpi_key *pkey = rsa_get_key(tfm); MPI c, m = mpi_alloc(0); int ret = 0; int sign; @@ -140,7 +147,7 @@ static int rsa_dec(struct akcipher_request *req) if (ret) goto err_free_c; - ret = mpi_write_to_sgl(m, req->dst, &req->dst_len, &sign); + ret = mpi_write_to_sgl(m, req->dst, req->dst_len, &sign); if (ret) goto err_free_c; @@ -156,7 +163,7 @@ err_free_m: static int rsa_sign(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - const struct rsa_key *pkey = rsa_get_key(tfm); + const struct rsa_mpi_key *pkey = rsa_get_key(tfm); MPI m, s = mpi_alloc(0); int ret = 0; int sign; @@ -178,7 +185,7 @@ static int rsa_sign(struct akcipher_request *req) if (ret) goto err_free_m; - ret = mpi_write_to_sgl(s, req->dst, &req->dst_len, &sign); + ret = mpi_write_to_sgl(s, req->dst, req->dst_len, &sign); if (ret) goto err_free_m; @@ -195,7 +202,7 @@ err_free_s: static int rsa_verify(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - const struct rsa_key *pkey = rsa_get_key(tfm); + const struct rsa_mpi_key *pkey = rsa_get_key(tfm); MPI s, m = mpi_alloc(0); int ret = 0; int sign; @@ -219,7 +226,7 @@ static int rsa_verify(struct akcipher_request *req) if (ret) goto err_free_s; - ret = mpi_write_to_sgl(m, req->dst, &req->dst_len, &sign); + ret = mpi_write_to_sgl(m, req->dst, req->dst_len, &sign); if (ret) goto err_free_s; @@ -233,6 +240,16 @@ err_free_m: return ret; } +static void rsa_free_mpi_key(struct rsa_mpi_key *key) +{ + mpi_free(key->d); + mpi_free(key->e); + mpi_free(key->n); + key->d = NULL; + key->e = NULL; + key->n = NULL; +} + static int rsa_check_key_length(unsigned int len) { switch (len) { @@ -251,49 +268,87 @@ static int rsa_check_key_length(unsigned int len) static int rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { - struct rsa_key *pkey = akcipher_tfm_ctx(tfm); + struct rsa_mpi_key *mpi_key = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {0}; int ret; - ret = rsa_parse_pub_key(pkey, key, keylen); + /* Free the old MPI key if any */ + rsa_free_mpi_key(mpi_key); + + ret = rsa_parse_pub_key(&raw_key, key, keylen); if (ret) return ret; - if (rsa_check_key_length(mpi_get_size(pkey->n) << 3)) { - rsa_free_key(pkey); - ret = -EINVAL; + mpi_key->e = mpi_read_raw_data(raw_key.e, raw_key.e_sz); + if (!mpi_key->e) + goto err; + + mpi_key->n = mpi_read_raw_data(raw_key.n, raw_key.n_sz); + if (!mpi_key->n) + goto err; + + if (rsa_check_key_length(mpi_get_size(mpi_key->n) << 3)) { + rsa_free_mpi_key(mpi_key); + return -EINVAL; } - return ret; + + return 0; + +err: + rsa_free_mpi_key(mpi_key); + return -ENOMEM; } static int rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { - struct rsa_key *pkey = akcipher_tfm_ctx(tfm); + struct rsa_mpi_key *mpi_key = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {0}; int ret; - ret = rsa_parse_priv_key(pkey, key, keylen); + /* Free the old MPI key if any */ + rsa_free_mpi_key(mpi_key); + + ret = rsa_parse_priv_key(&raw_key, key, keylen); if (ret) return ret; - if (rsa_check_key_length(mpi_get_size(pkey->n) << 3)) { - rsa_free_key(pkey); - ret = -EINVAL; + mpi_key->d = mpi_read_raw_data(raw_key.d, raw_key.d_sz); + if (!mpi_key->d) + goto err; + + mpi_key->e = mpi_read_raw_data(raw_key.e, raw_key.e_sz); + if (!mpi_key->e) + goto err; + + mpi_key->n = mpi_read_raw_data(raw_key.n, raw_key.n_sz); + if (!mpi_key->n) + goto err; + + if (rsa_check_key_length(mpi_get_size(mpi_key->n) << 3)) { + rsa_free_mpi_key(mpi_key); + return -EINVAL; } - return ret; + + return 0; + +err: + rsa_free_mpi_key(mpi_key); + return -ENOMEM; } static int rsa_max_size(struct crypto_akcipher *tfm) { - struct rsa_key *pkey = akcipher_tfm_ctx(tfm); + struct rsa_mpi_key *pkey = akcipher_tfm_ctx(tfm); return pkey->n ? mpi_get_size(pkey->n) : -EINVAL; } static void rsa_exit_tfm(struct crypto_akcipher *tfm) { - struct rsa_key *pkey = akcipher_tfm_ctx(tfm); + struct rsa_mpi_key *pkey = akcipher_tfm_ctx(tfm); - rsa_free_key(pkey); + rsa_free_mpi_key(pkey); } static struct akcipher_alg rsa = { @@ -310,7 +365,7 @@ static struct akcipher_alg rsa = { .cra_driver_name = "rsa-generic", .cra_priority = 100, .cra_module = THIS_MODULE, - .cra_ctxsize = sizeof(struct rsa_key), + .cra_ctxsize = sizeof(struct rsa_mpi_key), }, }; diff --git a/crypto/rsa_helper.c b/crypto/rsa_helper.c index d226f48d0907..4df6451e7543 100644 --- a/crypto/rsa_helper.c +++ b/crypto/rsa_helper.c @@ -22,20 +22,29 @@ int rsa_get_n(void *context, size_t hdrlen, unsigned char tag, const void *value, size_t vlen) { struct rsa_key *key = context; + const u8 *ptr = value; + size_t n_sz = vlen; - key->n = mpi_read_raw_data(value, vlen); - - if (!key->n) - return -ENOMEM; - - /* In FIPS mode only allow key size 2K & 3K */ - if (fips_enabled && (mpi_get_size(key->n) != 256 && - mpi_get_size(key->n) != 384)) { - pr_err("RSA: key size not allowed in FIPS mode\n"); - mpi_free(key->n); - key->n = NULL; + /* invalid key provided */ + if (!value || !vlen) return -EINVAL; + + if (fips_enabled) { + while (!*ptr && n_sz) { + ptr++; + n_sz--; + } + + /* In FIPS mode only allow key size 2K & 3K */ + if (n_sz != 256 && n_sz != 384) { + pr_err("RSA: key size not allowed in FIPS mode\n"); + return -EINVAL; + } } + + key->n = value; + key->n_sz = vlen; + return 0; } @@ -44,10 +53,12 @@ int rsa_get_e(void *context, size_t hdrlen, unsigned char tag, { struct rsa_key *key = context; - key->e = mpi_read_raw_data(value, vlen); + /* invalid key provided */ + if (!value || !key->n_sz || !vlen || vlen > key->n_sz) + return -EINVAL; - if (!key->e) - return -ENOMEM; + key->e = value; + key->e_sz = vlen; return 0; } @@ -57,46 +68,95 @@ int rsa_get_d(void *context, size_t hdrlen, unsigned char tag, { struct rsa_key *key = context; - key->d = mpi_read_raw_data(value, vlen); + /* invalid key provided */ + if (!value || !key->n_sz || !vlen || vlen > key->n_sz) + return -EINVAL; - if (!key->d) - return -ENOMEM; + key->d = value; + key->d_sz = vlen; - /* In FIPS mode only allow key size 2K & 3K */ - if (fips_enabled && (mpi_get_size(key->d) != 256 && - mpi_get_size(key->d) != 384)) { - pr_err("RSA: key size not allowed in FIPS mode\n"); - mpi_free(key->d); - key->d = NULL; + return 0; +} + +int rsa_get_p(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct rsa_key *key = context; + + /* invalid key provided */ + if (!value || !vlen || vlen > key->n_sz) return -EINVAL; - } + + key->p = value; + key->p_sz = vlen; + return 0; } -static void free_mpis(struct rsa_key *key) +int rsa_get_q(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) { - mpi_free(key->n); - mpi_free(key->e); - mpi_free(key->d); - key->n = NULL; - key->e = NULL; - key->d = NULL; + struct rsa_key *key = context; + + /* invalid key provided */ + if (!value || !vlen || vlen > key->n_sz) + return -EINVAL; + + key->q = value; + key->q_sz = vlen; + + return 0; } -/** - * rsa_free_key() - frees rsa key allocated by rsa_parse_key() - * - * @rsa_key: struct rsa_key key representation - */ -void rsa_free_key(struct rsa_key *key) +int rsa_get_dp(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct rsa_key *key = context; + + /* invalid key provided */ + if (!value || !vlen || vlen > key->n_sz) + return -EINVAL; + + key->dp = value; + key->dp_sz = vlen; + + return 0; +} + +int rsa_get_dq(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) { - free_mpis(key); + struct rsa_key *key = context; + + /* invalid key provided */ + if (!value || !vlen || vlen > key->n_sz) + return -EINVAL; + + key->dq = value; + key->dq_sz = vlen; + + return 0; +} + +int rsa_get_qinv(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct rsa_key *key = context; + + /* invalid key provided */ + if (!value || !vlen || vlen > key->n_sz) + return -EINVAL; + + key->qinv = value; + key->qinv_sz = vlen; + + return 0; } -EXPORT_SYMBOL_GPL(rsa_free_key); /** - * rsa_parse_pub_key() - extracts an rsa public key from BER encoded buffer - * and stores it in the provided struct rsa_key + * rsa_parse_pub_key() - decodes the BER encoded buffer and stores in the + * provided struct rsa_key, pointers to the raw key as is, + * so that the caller can copy it or MPI parse it, etc. * * @rsa_key: struct rsa_key key representation * @key: key in BER format @@ -107,23 +167,15 @@ EXPORT_SYMBOL_GPL(rsa_free_key); int rsa_parse_pub_key(struct rsa_key *rsa_key, const void *key, unsigned int key_len) { - int ret; - - free_mpis(rsa_key); - ret = asn1_ber_decoder(&rsapubkey_decoder, rsa_key, key, key_len); - if (ret < 0) - goto error; - - return 0; -error: - free_mpis(rsa_key); - return ret; + return asn1_ber_decoder(&rsapubkey_decoder, rsa_key, key, key_len); } EXPORT_SYMBOL_GPL(rsa_parse_pub_key); /** - * rsa_parse_pub_key() - extracts an rsa private key from BER encoded buffer - * and stores it in the provided struct rsa_key + * rsa_parse_priv_key() - decodes the BER encoded buffer and stores in the + * provided struct rsa_key, pointers to the raw key + * as is, so that the caller can copy it or MPI parse it, + * etc. * * @rsa_key: struct rsa_key key representation * @key: key in BER format @@ -134,16 +186,6 @@ EXPORT_SYMBOL_GPL(rsa_parse_pub_key); int rsa_parse_priv_key(struct rsa_key *rsa_key, const void *key, unsigned int key_len) { - int ret; - - free_mpis(rsa_key); - ret = asn1_ber_decoder(&rsaprivkey_decoder, rsa_key, key, key_len); - if (ret < 0) - goto error; - - return 0; -error: - free_mpis(rsa_key); - return ret; + return asn1_ber_decoder(&rsaprivkey_decoder, rsa_key, key, key_len); } EXPORT_SYMBOL_GPL(rsa_parse_priv_key); diff --git a/crypto/rsaprivkey.asn1 b/crypto/rsaprivkey.asn1 index 731aea5edb0c..4ce06758e8af 100644 --- a/crypto/rsaprivkey.asn1 +++ b/crypto/rsaprivkey.asn1 @@ -3,9 +3,9 @@ RsaPrivKey ::= SEQUENCE { n INTEGER ({ rsa_get_n }), e INTEGER ({ rsa_get_e }), d INTEGER ({ rsa_get_d }), - prime1 INTEGER, - prime2 INTEGER, - exponent1 INTEGER, - exponent2 INTEGER, - coefficient INTEGER + prime1 INTEGER ({ rsa_get_p }), + prime2 INTEGER ({ rsa_get_q }), + exponent1 INTEGER ({ rsa_get_dp }), + exponent2 INTEGER ({ rsa_get_dq }), + coefficient INTEGER ({ rsa_get_qinv }) } diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index ea5815c5e128..52ce17a3dd63 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -18,8 +18,6 @@ #include #include #include -#include -#include #include static inline void memcpy_dir(void *buf, void *sgdata, size_t nbytes, int out) @@ -30,53 +28,6 @@ static inline void memcpy_dir(void *buf, void *sgdata, size_t nbytes, int out) memcpy(dst, src, nbytes); } -void scatterwalk_start(struct scatter_walk *walk, struct scatterlist *sg) -{ - walk->sg = sg; - - BUG_ON(!sg->length); - - walk->offset = sg->offset; -} -EXPORT_SYMBOL_GPL(scatterwalk_start); - -void *scatterwalk_map(struct scatter_walk *walk) -{ - return kmap_atomic(scatterwalk_page(walk)) + - offset_in_page(walk->offset); -} -EXPORT_SYMBOL_GPL(scatterwalk_map); - -static void scatterwalk_pagedone(struct scatter_walk *walk, int out, - unsigned int more) -{ - if (out) { - struct page *page; - - page = sg_page(walk->sg) + ((walk->offset - 1) >> PAGE_SHIFT); - /* Test ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE first as - * PageSlab cannot be optimised away per se due to - * use of volatile pointer. - */ - if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE && !PageSlab(page)) - flush_dcache_page(page); - } - - if (more) { - walk->offset += PAGE_SIZE - 1; - walk->offset &= PAGE_MASK; - if (walk->offset >= walk->sg->offset + walk->sg->length) - scatterwalk_start(walk, sg_next(walk->sg)); - } -} - -void scatterwalk_done(struct scatter_walk *walk, int out, int more) -{ - if (!(scatterwalk_pagelen(walk) & (PAGE_SIZE - 1)) || !more) - scatterwalk_pagedone(walk, out, more); -} -EXPORT_SYMBOL_GPL(scatterwalk_done); - void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, size_t nbytes, int out) { @@ -87,9 +38,11 @@ void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, if (len_this_page > nbytes) len_this_page = nbytes; - vaddr = scatterwalk_map(walk); - memcpy_dir(buf, vaddr, len_this_page, out); - scatterwalk_unmap(vaddr); + if (out != 2) { + vaddr = scatterwalk_map(walk); + memcpy_dir(buf, vaddr, len_this_page, out); + scatterwalk_unmap(vaddr); + } scatterwalk_advance(walk, len_this_page); @@ -99,7 +52,7 @@ void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, buf += len_this_page; nbytes -= len_this_page; - scatterwalk_pagedone(walk, out, 1); + scatterwalk_pagedone(walk, out & 1, 1); } } EXPORT_SYMBOL_GPL(scatterwalk_copychunks); @@ -125,28 +78,6 @@ void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, } EXPORT_SYMBOL_GPL(scatterwalk_map_and_copy); -int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes) -{ - int offset = 0, n = 0; - - /* num_bytes is too small */ - if (num_bytes < sg->length) - return -1; - - do { - offset += sg->length; - n++; - sg = sg_next(sg); - - /* num_bytes is too large */ - if (unlikely(!sg && (num_bytes < offset))) - return -1; - } while (sg && (num_bytes > offset)); - - return n; -} -EXPORT_SYMBOL_GPL(scatterwalk_bytes_sglen); - struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], struct scatterlist *src, unsigned int len) diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 15a749a5cab7..c7049231861f 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -14,50 +14,17 @@ */ #include -#include -#include #include +#include #include #include #include #include #include -#include #include -struct seqiv_ctx { - spinlock_t lock; - u8 salt[] __attribute__ ((aligned(__alignof__(u32)))); -}; - static void seqiv_free(struct crypto_instance *inst); -static void seqiv_complete2(struct skcipher_givcrypt_request *req, int err) -{ - struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req); - struct crypto_ablkcipher *geniv; - - if (err == -EINPROGRESS) - return; - - if (err) - goto out; - - geniv = skcipher_givcrypt_reqtfm(req); - memcpy(req->creq.info, subreq->info, crypto_ablkcipher_ivsize(geniv)); - -out: - kfree(subreq->info); -} - -static void seqiv_complete(struct crypto_async_request *base, int err) -{ - struct skcipher_givcrypt_request *req = base->data; - - seqiv_complete2(req, err); - skcipher_givcrypt_complete(req, err); -} - static void seqiv_aead_encrypt_complete2(struct aead_request *req, int err) { struct aead_request *subreq = aead_request_ctx(req); @@ -85,65 +52,6 @@ static void seqiv_aead_encrypt_complete(struct crypto_async_request *base, aead_request_complete(req, err); } -static void seqiv_geniv(struct seqiv_ctx *ctx, u8 *info, u64 seq, - unsigned int ivsize) -{ - unsigned int len = ivsize; - - if (ivsize > sizeof(u64)) { - memset(info, 0, ivsize - sizeof(u64)); - len = sizeof(u64); - } - seq = cpu_to_be64(seq); - memcpy(info + ivsize - len, &seq, len); - crypto_xor(info, ctx->salt, ivsize); -} - -static int seqiv_givencrypt(struct skcipher_givcrypt_request *req) -{ - struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req); - struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv); - struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req); - crypto_completion_t compl; - void *data; - u8 *info; - unsigned int ivsize; - int err; - - ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv)); - - compl = req->creq.base.complete; - data = req->creq.base.data; - info = req->creq.info; - - ivsize = crypto_ablkcipher_ivsize(geniv); - - if (unlikely(!IS_ALIGNED((unsigned long)info, - crypto_ablkcipher_alignmask(geniv) + 1))) { - info = kmalloc(ivsize, req->creq.base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL: - GFP_ATOMIC); - if (!info) - return -ENOMEM; - - compl = seqiv_complete; - data = req; - } - - ablkcipher_request_set_callback(subreq, req->creq.base.flags, compl, - data); - ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst, - req->creq.nbytes, info); - - seqiv_geniv(ctx, info, req->seq, ivsize); - memcpy(req->giv, info, ivsize); - - err = crypto_ablkcipher_encrypt(subreq); - if (unlikely(info != req->creq.info)) - seqiv_complete2(req, err); - return err; -} - static int seqiv_aead_encrypt(struct aead_request *req) { struct crypto_aead *geniv = crypto_aead_reqtfm(req); @@ -165,12 +73,16 @@ static int seqiv_aead_encrypt(struct aead_request *req) info = req->iv; if (req->src != req->dst) { - struct blkcipher_desc desc = { - .tfm = ctx->null, - }; + SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull); + + skcipher_request_set_tfm(nreq, ctx->sknull); + skcipher_request_set_callback(nreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(nreq, req->src, req->dst, + req->assoclen + req->cryptlen, + NULL); - err = crypto_blkcipher_encrypt(&desc, req->dst, req->src, - req->assoclen + req->cryptlen); + err = crypto_skcipher_encrypt(nreq); if (err) return err; } @@ -229,62 +141,6 @@ static int seqiv_aead_decrypt(struct aead_request *req) return crypto_aead_decrypt(subreq); } -static int seqiv_init(struct crypto_tfm *tfm) -{ - struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm); - struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv); - int err; - - spin_lock_init(&ctx->lock); - - tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request); - - err = 0; - if (!crypto_get_default_rng()) { - crypto_ablkcipher_crt(geniv)->givencrypt = seqiv_givencrypt; - err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt, - crypto_ablkcipher_ivsize(geniv)); - crypto_put_default_rng(); - } - - return err ?: skcipher_geniv_init(tfm); -} - -static int seqiv_ablkcipher_create(struct crypto_template *tmpl, - struct rtattr **tb) -{ - struct crypto_instance *inst; - int err; - - inst = skcipher_geniv_alloc(tmpl, tb, 0, 0); - - if (IS_ERR(inst)) - return PTR_ERR(inst); - - err = -EINVAL; - if (inst->alg.cra_ablkcipher.ivsize < sizeof(u64)) - goto free_inst; - - inst->alg.cra_init = seqiv_init; - inst->alg.cra_exit = skcipher_geniv_exit; - - inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize; - inst->alg.cra_ctxsize += sizeof(struct seqiv_ctx); - - inst->alg.cra_alignmask |= __alignof__(u32) - 1; - - err = crypto_register_instance(tmpl, inst); - if (err) - goto free_inst; - -out: - return err; - -free_inst: - skcipher_geniv_free(inst); - goto out; -} - static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) { struct aead_instance *inst; @@ -330,26 +186,20 @@ free_inst: static int seqiv_create(struct crypto_template *tmpl, struct rtattr **tb) { struct crypto_attr_type *algt; - int err; algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) return PTR_ERR(algt); if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK) - err = seqiv_ablkcipher_create(tmpl, tb); - else - err = seqiv_aead_create(tmpl, tb); + return -EINVAL; - return err; + return seqiv_aead_create(tmpl, tb); } static void seqiv_free(struct crypto_instance *inst) { - if ((inst->alg.cra_flags ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK) - skcipher_geniv_free(inst); - else - aead_geniv_free(aead_instance(inst)); + aead_geniv_free(aead_instance(inst)); } static struct crypto_template seqiv_tmpl = { diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c new file mode 100644 index 000000000000..62264397a2d2 --- /dev/null +++ b/crypto/sha3_generic.c @@ -0,0 +1,300 @@ +/* + * Cryptographic API. + * + * SHA-3, as specified in + * http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf + * + * SHA-3 code by Jeff Garzik + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option)• + * any later version. + * + */ +#include +#include +#include +#include +#include +#include + +#define KECCAK_ROUNDS 24 + +#define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) + +static const u64 keccakf_rndc[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 +}; + +static const int keccakf_rotc[24] = { + 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, + 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 +}; + +static const int keccakf_piln[24] = { + 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, + 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 +}; + +/* update the state with given number of rounds */ + +static void keccakf(u64 st[25]) +{ + int i, j, round; + u64 t, bc[5]; + + for (round = 0; round < KECCAK_ROUNDS; round++) { + + /* Theta */ + for (i = 0; i < 5; i++) + bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] + ^ st[i + 20]; + + for (i = 0; i < 5; i++) { + t = bc[(i + 4) % 5] ^ ROTL64(bc[(i + 1) % 5], 1); + for (j = 0; j < 25; j += 5) + st[j + i] ^= t; + } + + /* Rho Pi */ + t = st[1]; + for (i = 0; i < 24; i++) { + j = keccakf_piln[i]; + bc[0] = st[j]; + st[j] = ROTL64(t, keccakf_rotc[i]); + t = bc[0]; + } + + /* Chi */ + for (j = 0; j < 25; j += 5) { + for (i = 0; i < 5; i++) + bc[i] = st[j + i]; + for (i = 0; i < 5; i++) + st[j + i] ^= (~bc[(i + 1) % 5]) & + bc[(i + 2) % 5]; + } + + /* Iota */ + st[0] ^= keccakf_rndc[round]; + } +} + +static void sha3_init(struct sha3_state *sctx, unsigned int digest_sz) +{ + memset(sctx, 0, sizeof(*sctx)); + sctx->md_len = digest_sz; + sctx->rsiz = 200 - 2 * digest_sz; + sctx->rsizw = sctx->rsiz / 8; +} + +static int sha3_224_init(struct shash_desc *desc) +{ + struct sha3_state *sctx = shash_desc_ctx(desc); + + sha3_init(sctx, SHA3_224_DIGEST_SIZE); + return 0; +} + +static int sha3_256_init(struct shash_desc *desc) +{ + struct sha3_state *sctx = shash_desc_ctx(desc); + + sha3_init(sctx, SHA3_256_DIGEST_SIZE); + return 0; +} + +static int sha3_384_init(struct shash_desc *desc) +{ + struct sha3_state *sctx = shash_desc_ctx(desc); + + sha3_init(sctx, SHA3_384_DIGEST_SIZE); + return 0; +} + +static int sha3_512_init(struct shash_desc *desc) +{ + struct sha3_state *sctx = shash_desc_ctx(desc); + + sha3_init(sctx, SHA3_512_DIGEST_SIZE); + return 0; +} + +static int sha3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha3_state *sctx = shash_desc_ctx(desc); + unsigned int done; + const u8 *src; + + done = 0; + src = data; + + if ((sctx->partial + len) > (sctx->rsiz - 1)) { + if (sctx->partial) { + done = -sctx->partial; + memcpy(sctx->buf + sctx->partial, data, + done + sctx->rsiz); + src = sctx->buf; + } + + do { + unsigned int i; + + for (i = 0; i < sctx->rsizw; i++) + sctx->st[i] ^= ((u64 *) src)[i]; + keccakf(sctx->st); + + done += sctx->rsiz; + src = data + done; + } while (done + (sctx->rsiz - 1) < len); + + sctx->partial = 0; + } + memcpy(sctx->buf + sctx->partial, src, len - done); + sctx->partial += (len - done); + + return 0; +} + +static int sha3_final(struct shash_desc *desc, u8 *out) +{ + struct sha3_state *sctx = shash_desc_ctx(desc); + unsigned int i, inlen = sctx->partial; + + sctx->buf[inlen++] = 0x06; + memset(sctx->buf + inlen, 0, sctx->rsiz - inlen); + sctx->buf[sctx->rsiz - 1] |= 0x80; + + for (i = 0; i < sctx->rsizw; i++) + sctx->st[i] ^= ((u64 *) sctx->buf)[i]; + + keccakf(sctx->st); + + for (i = 0; i < sctx->rsizw; i++) + sctx->st[i] = cpu_to_le64(sctx->st[i]); + + memcpy(out, sctx->st, sctx->md_len); + + memset(sctx, 0, sizeof(*sctx)); + return 0; +} + +static struct shash_alg sha3_224 = { + .digestsize = SHA3_224_DIGEST_SIZE, + .init = sha3_224_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base = { + .cra_name = "sha3-224", + .cra_driver_name = "sha3-224-generic", + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA3_224_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static struct shash_alg sha3_256 = { + .digestsize = SHA3_256_DIGEST_SIZE, + .init = sha3_256_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base = { + .cra_name = "sha3-256", + .cra_driver_name = "sha3-256-generic", + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA3_256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static struct shash_alg sha3_384 = { + .digestsize = SHA3_384_DIGEST_SIZE, + .init = sha3_384_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base = { + .cra_name = "sha3-384", + .cra_driver_name = "sha3-384-generic", + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA3_384_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static struct shash_alg sha3_512 = { + .digestsize = SHA3_512_DIGEST_SIZE, + .init = sha3_512_init, + .update = sha3_update, + .final = sha3_final, + .descsize = sizeof(struct sha3_state), + .base = { + .cra_name = "sha3-512", + .cra_driver_name = "sha3-512-generic", + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA3_512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sha3_generic_mod_init(void) +{ + int ret; + + ret = crypto_register_shash(&sha3_224); + if (ret < 0) + goto err_out; + ret = crypto_register_shash(&sha3_256); + if (ret < 0) + goto err_out_224; + ret = crypto_register_shash(&sha3_384); + if (ret < 0) + goto err_out_256; + ret = crypto_register_shash(&sha3_512); + if (ret < 0) + goto err_out_384; + + return 0; + +err_out_384: + crypto_unregister_shash(&sha3_384); +err_out_256: + crypto_unregister_shash(&sha3_256); +err_out_224: + crypto_unregister_shash(&sha3_224); +err_out: + return ret; +} + +static void __exit sha3_generic_mod_fini(void) +{ + crypto_unregister_shash(&sha3_224); + crypto_unregister_shash(&sha3_256); + crypto_unregister_shash(&sha3_384); + crypto_unregister_shash(&sha3_512); +} + +module_init(sha3_generic_mod_init); +module_exit(sha3_generic_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-3 Secure Hash Algorithm"); + +MODULE_ALIAS_CRYPTO("sha3-224"); +MODULE_ALIAS_CRYPTO("sha3-224-generic"); +MODULE_ALIAS_CRYPTO("sha3-256"); +MODULE_ALIAS_CRYPTO("sha3-256-generic"); +MODULE_ALIAS_CRYPTO("sha3-384"); +MODULE_ALIAS_CRYPTO("sha3-384-generic"); +MODULE_ALIAS_CRYPTO("sha3-512"); +MODULE_ALIAS_CRYPTO("sha3-512-generic"); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 69230e9d4ac9..f7d0018dcaee 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -16,7 +16,11 @@ #include #include +#include #include +#include +#include +#include #include "internal.h" @@ -25,10 +29,11 @@ static unsigned int crypto_skcipher_extsize(struct crypto_alg *alg) if (alg->cra_type == &crypto_blkcipher_type) return sizeof(struct crypto_blkcipher *); - BUG_ON(alg->cra_type != &crypto_ablkcipher_type && - alg->cra_type != &crypto_givcipher_type); + if (alg->cra_type == &crypto_ablkcipher_type || + alg->cra_type == &crypto_givcipher_type) + return sizeof(struct crypto_ablkcipher *); - return sizeof(struct crypto_ablkcipher *); + return crypto_alg_extsize(alg); } static int skcipher_setkey_blkcipher(struct crypto_skcipher *tfm, @@ -216,26 +221,118 @@ static int crypto_init_skcipher_ops_ablkcipher(struct crypto_tfm *tfm) return 0; } +static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm) +{ + struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(skcipher); + + alg->exit(skcipher); +} + static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm) { + struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(skcipher); + if (tfm->__crt_alg->cra_type == &crypto_blkcipher_type) return crypto_init_skcipher_ops_blkcipher(tfm); - BUG_ON(tfm->__crt_alg->cra_type != &crypto_ablkcipher_type && - tfm->__crt_alg->cra_type != &crypto_givcipher_type); + if (tfm->__crt_alg->cra_type == &crypto_ablkcipher_type || + tfm->__crt_alg->cra_type == &crypto_givcipher_type) + return crypto_init_skcipher_ops_ablkcipher(tfm); + + skcipher->setkey = alg->setkey; + skcipher->encrypt = alg->encrypt; + skcipher->decrypt = alg->decrypt; + skcipher->ivsize = alg->ivsize; + skcipher->keysize = alg->max_keysize; + + if (alg->exit) + skcipher->base.exit = crypto_skcipher_exit_tfm; - return crypto_init_skcipher_ops_ablkcipher(tfm); + if (alg->init) + return alg->init(skcipher); + + return 0; +} + +static void crypto_skcipher_free_instance(struct crypto_instance *inst) +{ + struct skcipher_instance *skcipher = + container_of(inst, struct skcipher_instance, s.base); + + skcipher->free(skcipher); +} + +static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) + __attribute__ ((unused)); +static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) +{ + struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg, + base); + + seq_printf(m, "type : skcipher\n"); + seq_printf(m, "async : %s\n", + alg->cra_flags & CRYPTO_ALG_ASYNC ? "yes" : "no"); + seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); + seq_printf(m, "min keysize : %u\n", skcipher->min_keysize); + seq_printf(m, "max keysize : %u\n", skcipher->max_keysize); + seq_printf(m, "ivsize : %u\n", skcipher->ivsize); + seq_printf(m, "chunksize : %u\n", skcipher->chunksize); } +#ifdef CONFIG_NET +static int crypto_skcipher_report(struct sk_buff *skb, struct crypto_alg *alg) +{ + struct crypto_report_blkcipher rblkcipher; + struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg, + base); + + strncpy(rblkcipher.type, "skcipher", sizeof(rblkcipher.type)); + strncpy(rblkcipher.geniv, "", sizeof(rblkcipher.geniv)); + + rblkcipher.blocksize = alg->cra_blocksize; + rblkcipher.min_keysize = skcipher->min_keysize; + rblkcipher.max_keysize = skcipher->max_keysize; + rblkcipher.ivsize = skcipher->ivsize; + + if (nla_put(skb, CRYPTOCFGA_REPORT_BLKCIPHER, + sizeof(struct crypto_report_blkcipher), &rblkcipher)) + goto nla_put_failure; + return 0; + +nla_put_failure: + return -EMSGSIZE; +} +#else +static int crypto_skcipher_report(struct sk_buff *skb, struct crypto_alg *alg) +{ + return -ENOSYS; +} +#endif + static const struct crypto_type crypto_skcipher_type2 = { .extsize = crypto_skcipher_extsize, .init_tfm = crypto_skcipher_init_tfm, + .free = crypto_skcipher_free_instance, +#ifdef CONFIG_PROC_FS + .show = crypto_skcipher_show, +#endif + .report = crypto_skcipher_report, .maskclear = ~CRYPTO_ALG_TYPE_MASK, .maskset = CRYPTO_ALG_TYPE_BLKCIPHER_MASK, - .type = CRYPTO_ALG_TYPE_BLKCIPHER, + .type = CRYPTO_ALG_TYPE_SKCIPHER, .tfmsize = offsetof(struct crypto_skcipher, base), }; +int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn, + const char *name, u32 type, u32 mask) +{ + spawn->base.frontend = &crypto_skcipher_type2; + return crypto_grab_spawn(&spawn->base, name, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_grab_skcipher); + struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name, u32 type, u32 mask) { @@ -243,5 +340,90 @@ struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name, } EXPORT_SYMBOL_GPL(crypto_alloc_skcipher); +int crypto_has_skcipher2(const char *alg_name, u32 type, u32 mask) +{ + return crypto_type_has_alg(alg_name, &crypto_skcipher_type2, + type, mask); +} +EXPORT_SYMBOL_GPL(crypto_has_skcipher2); + +static int skcipher_prepare_alg(struct skcipher_alg *alg) +{ + struct crypto_alg *base = &alg->base; + + if (alg->ivsize > PAGE_SIZE / 8 || alg->chunksize > PAGE_SIZE / 8) + return -EINVAL; + + if (!alg->chunksize) + alg->chunksize = base->cra_blocksize; + + base->cra_type = &crypto_skcipher_type2; + base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; + base->cra_flags |= CRYPTO_ALG_TYPE_SKCIPHER; + + return 0; +} + +int crypto_register_skcipher(struct skcipher_alg *alg) +{ + struct crypto_alg *base = &alg->base; + int err; + + err = skcipher_prepare_alg(alg); + if (err) + return err; + + return crypto_register_alg(base); +} +EXPORT_SYMBOL_GPL(crypto_register_skcipher); + +void crypto_unregister_skcipher(struct skcipher_alg *alg) +{ + crypto_unregister_alg(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_unregister_skcipher); + +int crypto_register_skciphers(struct skcipher_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_register_skcipher(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + for (--i; i >= 0; --i) + crypto_unregister_skcipher(&algs[i]); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_register_skciphers); + +void crypto_unregister_skciphers(struct skcipher_alg *algs, int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_unregister_skcipher(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_unregister_skciphers); + +int skcipher_register_instance(struct crypto_template *tmpl, + struct skcipher_instance *inst) +{ + int err; + + err = skcipher_prepare_alg(&inst->alg); + if (err) + return err; + + return crypto_register_instance(tmpl, skcipher_crypto_instance(inst)); +} +EXPORT_SYMBOL_GPL(skcipher_register_instance); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Symmetric key cipher type"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 579dce071463..ae22f05d5936 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -72,7 +73,8 @@ static char *check[] = { "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea", "khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt", "camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320", - "lzo", "cts", "zlib", NULL + "lzo", "cts", "zlib", "sha3-224", "sha3-256", "sha3-384", "sha3-512", + NULL }; struct tcrypt_result { @@ -91,76 +93,6 @@ static void tcrypt_complete(struct crypto_async_request *req, int err) complete(&res->completion); } -static int test_cipher_jiffies(struct blkcipher_desc *desc, int enc, - struct scatterlist *sg, int blen, int secs) -{ - unsigned long start, end; - int bcount; - int ret; - - for (start = jiffies, end = start + secs * HZ, bcount = 0; - time_before(jiffies, end); bcount++) { - if (enc) - ret = crypto_blkcipher_encrypt(desc, sg, sg, blen); - else - ret = crypto_blkcipher_decrypt(desc, sg, sg, blen); - - if (ret) - return ret; - } - - printk("%d operations in %d seconds (%ld bytes)\n", - bcount, secs, (long)bcount * blen); - return 0; -} - -static int test_cipher_cycles(struct blkcipher_desc *desc, int enc, - struct scatterlist *sg, int blen) -{ - unsigned long cycles = 0; - int ret = 0; - int i; - - local_irq_disable(); - - /* Warm-up run. */ - for (i = 0; i < 4; i++) { - if (enc) - ret = crypto_blkcipher_encrypt(desc, sg, sg, blen); - else - ret = crypto_blkcipher_decrypt(desc, sg, sg, blen); - - if (ret) - goto out; - } - - /* The real thing. */ - for (i = 0; i < 8; i++) { - cycles_t start, end; - - start = get_cycles(); - if (enc) - ret = crypto_blkcipher_encrypt(desc, sg, sg, blen); - else - ret = crypto_blkcipher_decrypt(desc, sg, sg, blen); - end = get_cycles(); - - if (ret) - goto out; - - cycles += end - start; - } - -out: - local_irq_enable(); - - if (ret == 0) - printk("1 operation in %lu cycles (%d bytes)\n", - (cycles + 4) / 8, blen); - - return ret; -} - static inline int do_one_aead_op(struct aead_request *req, int ret) { if (ret == -EINPROGRESS || ret == -EBUSY) { @@ -454,127 +386,148 @@ out_noxbuf: return; } -static void test_cipher_speed(const char *algo, int enc, unsigned int secs, - struct cipher_speed_template *template, - unsigned int tcount, u8 *keysize) +static void test_hash_sg_init(struct scatterlist *sg) { - unsigned int ret, i, j, iv_len; - const char *key; - char iv[128]; - struct crypto_blkcipher *tfm; - struct blkcipher_desc desc; - const char *e; - u32 *b_size; + int i; - if (enc == ENCRYPT) - e = "encryption"; - else - e = "decryption"; + sg_init_table(sg, TVMEMSIZE); + for (i = 0; i < TVMEMSIZE; i++) { + sg_set_buf(sg + i, tvmem[i], PAGE_SIZE); + memset(tvmem[i], 0xff, PAGE_SIZE); + } +} - tfm = crypto_alloc_blkcipher(algo, 0, CRYPTO_ALG_ASYNC); +static inline int do_one_ahash_op(struct ahash_request *req, int ret) +{ + if (ret == -EINPROGRESS || ret == -EBUSY) { + struct tcrypt_result *tr = req->base.data; - if (IS_ERR(tfm)) { - printk("failed to load transform for %s: %ld\n", algo, - PTR_ERR(tfm)); + wait_for_completion(&tr->completion); + reinit_completion(&tr->completion); + ret = tr->err; + } + return ret; +} + +struct test_mb_ahash_data { + struct scatterlist sg[TVMEMSIZE]; + char result[64]; + struct ahash_request *req; + struct tcrypt_result tresult; + char *xbuf[XBUFSIZE]; +}; + +static void test_mb_ahash_speed(const char *algo, unsigned int sec, + struct hash_speed *speed) +{ + struct test_mb_ahash_data *data; + struct crypto_ahash *tfm; + unsigned long start, end; + unsigned long cycles; + unsigned int i, j, k; + int ret; + + data = kzalloc(sizeof(*data) * 8, GFP_KERNEL); + if (!data) return; + + tfm = crypto_alloc_ahash(algo, 0, 0); + if (IS_ERR(tfm)) { + pr_err("failed to load transform for %s: %ld\n", + algo, PTR_ERR(tfm)); + goto free_data; } - desc.tfm = tfm; - desc.flags = 0; - printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo, - get_driver_name(crypto_blkcipher, tfm), e); + for (i = 0; i < 8; ++i) { + if (testmgr_alloc_buf(data[i].xbuf)) + goto out; - i = 0; - do { + init_completion(&data[i].tresult.completion); - b_size = block_sizes; - do { - struct scatterlist sg[TVMEMSIZE]; + data[i].req = ahash_request_alloc(tfm, GFP_KERNEL); + if (!data[i].req) { + pr_err("alg: hash: Failed to allocate request for %s\n", + algo); + goto out; + } - if ((*keysize + *b_size) > TVMEMSIZE * PAGE_SIZE) { - printk("template (%u) too big for " - "tvmem (%lu)\n", *keysize + *b_size, - TVMEMSIZE * PAGE_SIZE); - goto out; - } + ahash_request_set_callback(data[i].req, 0, + tcrypt_complete, &data[i].tresult); + test_hash_sg_init(data[i].sg); + } - printk("test %u (%d bit key, %d byte blocks): ", i, - *keysize * 8, *b_size); + pr_info("\ntesting speed of multibuffer %s (%s)\n", algo, + get_driver_name(crypto_ahash, tfm)); - memset(tvmem[0], 0xff, PAGE_SIZE); + for (i = 0; speed[i].blen != 0; i++) { + /* For some reason this only tests digests. */ + if (speed[i].blen != speed[i].plen) + continue; - /* set key, plain text and IV */ - key = tvmem[0]; - for (j = 0; j < tcount; j++) { - if (template[j].klen == *keysize) { - key = template[j].key; - break; - } - } + if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) { + pr_err("template (%u) too big for tvmem (%lu)\n", + speed[i].blen, TVMEMSIZE * PAGE_SIZE); + goto out; + } - ret = crypto_blkcipher_setkey(tfm, key, *keysize); - if (ret) { - printk("setkey() failed flags=%x\n", - crypto_blkcipher_get_flags(tfm)); - goto out; - } + if (speed[i].klen) + crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen); - sg_init_table(sg, TVMEMSIZE); - sg_set_buf(sg, tvmem[0] + *keysize, - PAGE_SIZE - *keysize); - for (j = 1; j < TVMEMSIZE; j++) { - sg_set_buf(sg + j, tvmem[j], PAGE_SIZE); - memset (tvmem[j], 0xff, PAGE_SIZE); - } + for (k = 0; k < 8; k++) + ahash_request_set_crypt(data[k].req, data[k].sg, + data[k].result, speed[i].blen); - iv_len = crypto_blkcipher_ivsize(tfm); - if (iv_len) { - memset(&iv, 0xff, iv_len); - crypto_blkcipher_set_iv(tfm, iv, iv_len); - } + pr_info("test%3u " + "(%5u byte blocks,%5u bytes per update,%4u updates): ", + i, speed[i].blen, speed[i].plen, + speed[i].blen / speed[i].plen); - if (secs) - ret = test_cipher_jiffies(&desc, enc, sg, - *b_size, secs); - else - ret = test_cipher_cycles(&desc, enc, sg, - *b_size); + start = get_cycles(); - if (ret) { - printk("%s() failed flags=%x\n", e, desc.flags); - break; + for (k = 0; k < 8; k++) { + ret = crypto_ahash_digest(data[k].req); + if (ret == -EINPROGRESS) { + ret = 0; + continue; } - b_size++; - i++; - } while (*b_size); - keysize++; - } while (*keysize); -out: - crypto_free_blkcipher(tfm); -} + if (ret) + break; -static void test_hash_sg_init(struct scatterlist *sg) -{ - int i; + complete(&data[k].tresult.completion); + data[k].tresult.err = 0; + } - sg_init_table(sg, TVMEMSIZE); - for (i = 0; i < TVMEMSIZE; i++) { - sg_set_buf(sg + i, tvmem[i], PAGE_SIZE); - memset(tvmem[i], 0xff, PAGE_SIZE); - } -} + for (j = 0; j < k; j++) { + struct tcrypt_result *tr = &data[j].tresult; -static inline int do_one_ahash_op(struct ahash_request *req, int ret) -{ - if (ret == -EINPROGRESS || ret == -EBUSY) { - struct tcrypt_result *tr = req->base.data; + wait_for_completion(&tr->completion); + if (tr->err) + ret = tr->err; + } - wait_for_completion(&tr->completion); - reinit_completion(&tr->completion); - ret = tr->err; + end = get_cycles(); + cycles = end - start; + pr_cont("%6lu cycles/operation, %4lu cycles/byte\n", + cycles, cycles / (8 * speed[i].blen)); + + if (ret) { + pr_err("At least one hashing failed ret=%d\n", ret); + break; + } } - return ret; + +out: + for (k = 0; k < 8; ++k) + ahash_request_free(data[k].req); + + for (k = 0; k < 8; ++k) + testmgr_free_buf(data[k].xbuf); + + crypto_free_ahash(tfm); + +free_data: + kfree(data); } static int test_ahash_jiffies_digest(struct ahash_request *req, int blen, @@ -812,7 +765,7 @@ static void test_hash_speed(const char *algo, unsigned int secs, return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC); } -static inline int do_one_acipher_op(struct ablkcipher_request *req, int ret) +static inline int do_one_acipher_op(struct skcipher_request *req, int ret) { if (ret == -EINPROGRESS || ret == -EBUSY) { struct tcrypt_result *tr = req->base.data; @@ -825,7 +778,7 @@ static inline int do_one_acipher_op(struct ablkcipher_request *req, int ret) return ret; } -static int test_acipher_jiffies(struct ablkcipher_request *req, int enc, +static int test_acipher_jiffies(struct skcipher_request *req, int enc, int blen, int secs) { unsigned long start, end; @@ -836,10 +789,10 @@ static int test_acipher_jiffies(struct ablkcipher_request *req, int enc, time_before(jiffies, end); bcount++) { if (enc) ret = do_one_acipher_op(req, - crypto_ablkcipher_encrypt(req)); + crypto_skcipher_encrypt(req)); else ret = do_one_acipher_op(req, - crypto_ablkcipher_decrypt(req)); + crypto_skcipher_decrypt(req)); if (ret) return ret; @@ -850,7 +803,7 @@ static int test_acipher_jiffies(struct ablkcipher_request *req, int enc, return 0; } -static int test_acipher_cycles(struct ablkcipher_request *req, int enc, +static int test_acipher_cycles(struct skcipher_request *req, int enc, int blen) { unsigned long cycles = 0; @@ -861,10 +814,10 @@ static int test_acipher_cycles(struct ablkcipher_request *req, int enc, for (i = 0; i < 4; i++) { if (enc) ret = do_one_acipher_op(req, - crypto_ablkcipher_encrypt(req)); + crypto_skcipher_encrypt(req)); else ret = do_one_acipher_op(req, - crypto_ablkcipher_decrypt(req)); + crypto_skcipher_decrypt(req)); if (ret) goto out; @@ -877,10 +830,10 @@ static int test_acipher_cycles(struct ablkcipher_request *req, int enc, start = get_cycles(); if (enc) ret = do_one_acipher_op(req, - crypto_ablkcipher_encrypt(req)); + crypto_skcipher_encrypt(req)); else ret = do_one_acipher_op(req, - crypto_ablkcipher_decrypt(req)); + crypto_skcipher_decrypt(req)); end = get_cycles(); if (ret) @@ -897,16 +850,16 @@ out: return ret; } -static void test_acipher_speed(const char *algo, int enc, unsigned int secs, - struct cipher_speed_template *template, - unsigned int tcount, u8 *keysize) +static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, + struct cipher_speed_template *template, + unsigned int tcount, u8 *keysize, bool async) { unsigned int ret, i, j, k, iv_len; struct tcrypt_result tresult; const char *key; char iv[128]; - struct ablkcipher_request *req; - struct crypto_ablkcipher *tfm; + struct skcipher_request *req; + struct crypto_skcipher *tfm; const char *e; u32 *b_size; @@ -917,7 +870,7 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, init_completion(&tresult.completion); - tfm = crypto_alloc_ablkcipher(algo, 0, 0); + tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC); if (IS_ERR(tfm)) { pr_err("failed to load transform for %s: %ld\n", algo, @@ -926,17 +879,17 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, } pr_info("\ntesting speed of async %s (%s) %s\n", algo, - get_driver_name(crypto_ablkcipher, tfm), e); + get_driver_name(crypto_skcipher, tfm), e); - req = ablkcipher_request_alloc(tfm, GFP_KERNEL); + req = skcipher_request_alloc(tfm, GFP_KERNEL); if (!req) { pr_err("tcrypt: skcipher: Failed to allocate request for %s\n", algo); goto out; } - ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tcrypt_complete, &tresult); + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &tresult); i = 0; do { @@ -966,12 +919,12 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, } } - crypto_ablkcipher_clear_flags(tfm, ~0); + crypto_skcipher_clear_flags(tfm, ~0); - ret = crypto_ablkcipher_setkey(tfm, key, *keysize); + ret = crypto_skcipher_setkey(tfm, key, *keysize); if (ret) { pr_err("setkey() failed flags=%x\n", - crypto_ablkcipher_get_flags(tfm)); + crypto_skcipher_get_flags(tfm)); goto out_free_req; } @@ -995,11 +948,11 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, sg_set_buf(sg, tvmem[0] + *keysize, *b_size); } - iv_len = crypto_ablkcipher_ivsize(tfm); + iv_len = crypto_skcipher_ivsize(tfm); if (iv_len) memset(&iv, 0xff, iv_len); - ablkcipher_request_set_crypt(req, sg, sg, *b_size, iv); + skcipher_request_set_crypt(req, sg, sg, *b_size, iv); if (secs) ret = test_acipher_jiffies(req, enc, @@ -1010,7 +963,7 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, if (ret) { pr_err("%s() failed flags=%x\n", e, - crypto_ablkcipher_get_flags(tfm)); + crypto_skcipher_get_flags(tfm)); break; } b_size++; @@ -1020,9 +973,25 @@ static void test_acipher_speed(const char *algo, int enc, unsigned int secs, } while (*keysize); out_free_req: - ablkcipher_request_free(req); + skcipher_request_free(req); out: - crypto_free_ablkcipher(tfm); + crypto_free_skcipher(tfm); +} + +static void test_acipher_speed(const char *algo, int enc, unsigned int secs, + struct cipher_speed_template *template, + unsigned int tcount, u8 *keysize) +{ + return test_skcipher_speed(algo, enc, secs, template, tcount, keysize, + true); +} + +static void test_cipher_speed(const char *algo, int enc, unsigned int secs, + struct cipher_speed_template *template, + unsigned int tcount, u8 *keysize) +{ + return test_skcipher_speed(algo, enc, secs, template, tcount, keysize, + false); } static void test_available(void) @@ -1284,6 +1253,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) ret += tcrypt_test("crct10dif"); break; + case 48: + ret += tcrypt_test("sha3-224"); + break; + + case 49: + ret += tcrypt_test("sha3-256"); + break; + + case 50: + ret += tcrypt_test("sha3-384"); + break; + + case 51: + ret += tcrypt_test("sha3-512"); + break; + case 100: ret += tcrypt_test("hmac(md5)"); break; @@ -1328,6 +1313,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) ret += tcrypt_test("hmac(crc32)"); break; + case 111: + ret += tcrypt_test("hmac(sha3-224)"); + break; + + case 112: + ret += tcrypt_test("hmac(sha3-256)"); + break; + + case 113: + ret += tcrypt_test("hmac(sha3-384)"); + break; + + case 114: + ret += tcrypt_test("hmac(sha3-512)"); + break; + case 150: ret += tcrypt_test("ansi_cprng"); break; @@ -1406,6 +1407,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) speed_template_32_48_64); test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, speed_template_32_48_64); + test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0, @@ -1691,6 +1696,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) test_hash_speed("poly1305", sec, poly1305_speed_template); if (mode > 300 && mode < 400) break; + case 322: + test_hash_speed("sha3-224", sec, generic_hash_speed_template); + if (mode > 300 && mode < 400) break; + + case 323: + test_hash_speed("sha3-256", sec, generic_hash_speed_template); + if (mode > 300 && mode < 400) break; + + case 324: + test_hash_speed("sha3-384", sec, generic_hash_speed_template); + if (mode > 300 && mode < 400) break; + + case 325: + test_hash_speed("sha3-512", sec, generic_hash_speed_template); + if (mode > 300 && mode < 400) break; + case 399: break; @@ -1770,6 +1791,35 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) test_ahash_speed("rmd320", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; + case 418: + test_ahash_speed("sha3-224", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 419: + test_ahash_speed("sha3-256", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 420: + test_ahash_speed("sha3-384", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + + case 421: + test_ahash_speed("sha3-512", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 422: + test_mb_ahash_speed("sha1", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 423: + test_mb_ahash_speed("sha256", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 424: + test_mb_ahash_speed("sha512", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + case 499: break; @@ -1790,6 +1840,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m) speed_template_32_48_64); test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, speed_template_32_48_64); + test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0, diff --git a/crypto/testmgr.c b/crypto/testmgr.c index c727fb0cb021..5c9d5a5e7b65 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "internal.h" @@ -120,6 +121,11 @@ struct akcipher_test_suite { unsigned int count; }; +struct kpp_test_suite { + struct kpp_testvec *vecs; + unsigned int count; +}; + struct alg_test_desc { const char *alg; int (*test)(const struct alg_test_desc *desc, const char *driver, @@ -134,6 +140,7 @@ struct alg_test_desc { struct cprng_test_suite cprng; struct drbg_test_suite drbg; struct akcipher_test_suite akcipher; + struct kpp_test_suite kpp; } suite; }; @@ -1777,8 +1784,135 @@ static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver, } -static int do_test_rsa(struct crypto_akcipher *tfm, - struct akcipher_testvec *vecs) +static int do_test_kpp(struct crypto_kpp *tfm, struct kpp_testvec *vec, + const char *alg) +{ + struct kpp_request *req; + void *input_buf = NULL; + void *output_buf = NULL; + struct tcrypt_result result; + unsigned int out_len_max; + int err = -ENOMEM; + struct scatterlist src, dst; + + req = kpp_request_alloc(tfm, GFP_KERNEL); + if (!req) + return err; + + init_completion(&result.completion); + + err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size); + if (err < 0) + goto free_req; + + out_len_max = crypto_kpp_maxsize(tfm); + output_buf = kzalloc(out_len_max, GFP_KERNEL); + if (!output_buf) { + err = -ENOMEM; + goto free_req; + } + + /* Use appropriate parameter as base */ + kpp_request_set_input(req, NULL, 0); + sg_init_one(&dst, output_buf, out_len_max); + kpp_request_set_output(req, &dst, out_len_max); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &result); + + /* Compute public key */ + err = wait_async_op(&result, crypto_kpp_generate_public_key(req)); + if (err) { + pr_err("alg: %s: generate public key test failed. err %d\n", + alg, err); + goto free_output; + } + /* Verify calculated public key */ + if (memcmp(vec->expected_a_public, sg_virt(req->dst), + vec->expected_a_public_size)) { + pr_err("alg: %s: generate public key test failed. Invalid output\n", + alg); + err = -EINVAL; + goto free_output; + } + + /* Calculate shared secret key by using counter part (b) public key. */ + input_buf = kzalloc(vec->b_public_size, GFP_KERNEL); + if (!input_buf) { + err = -ENOMEM; + goto free_output; + } + + memcpy(input_buf, vec->b_public, vec->b_public_size); + sg_init_one(&src, input_buf, vec->b_public_size); + sg_init_one(&dst, output_buf, out_len_max); + kpp_request_set_input(req, &src, vec->b_public_size); + kpp_request_set_output(req, &dst, out_len_max); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &result); + err = wait_async_op(&result, crypto_kpp_compute_shared_secret(req)); + if (err) { + pr_err("alg: %s: compute shard secret test failed. err %d\n", + alg, err); + goto free_all; + } + /* + * verify shared secret from which the user will derive + * secret key by executing whatever hash it has chosen + */ + if (memcmp(vec->expected_ss, sg_virt(req->dst), + vec->expected_ss_size)) { + pr_err("alg: %s: compute shared secret test failed. Invalid output\n", + alg); + err = -EINVAL; + } + +free_all: + kfree(input_buf); +free_output: + kfree(output_buf); +free_req: + kpp_request_free(req); + return err; +} + +static int test_kpp(struct crypto_kpp *tfm, const char *alg, + struct kpp_testvec *vecs, unsigned int tcount) +{ + int ret, i; + + for (i = 0; i < tcount; i++) { + ret = do_test_kpp(tfm, vecs++, alg); + if (ret) { + pr_err("alg: %s: test failed on vector %d, err=%d\n", + alg, i + 1, ret); + return ret; + } + } + return 0; +} + +static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver, + u32 type, u32 mask) +{ + struct crypto_kpp *tfm; + int err = 0; + + tfm = crypto_alloc_kpp(driver, type | CRYPTO_ALG_INTERNAL, mask); + if (IS_ERR(tfm)) { + pr_err("alg: kpp: Failed to load tfm for %s: %ld\n", + driver, PTR_ERR(tfm)); + return PTR_ERR(tfm); + } + if (desc->suite.kpp.vecs) + err = test_kpp(tfm, desc->alg, desc->suite.kpp.vecs, + desc->suite.kpp.count); + + crypto_free_kpp(tfm); + return err; +} + +static int test_akcipher_one(struct crypto_akcipher *tfm, + struct akcipher_testvec *vecs) { char *xbuf[XBUFSIZE]; struct akcipher_request *req; @@ -1807,6 +1941,7 @@ static int do_test_rsa(struct crypto_akcipher *tfm, if (err) goto free_req; + err = -ENOMEM; out_len_max = crypto_akcipher_maxsize(tfm); outbuf_enc = kzalloc(out_len_max, GFP_KERNEL); if (!outbuf_enc) @@ -1829,17 +1964,18 @@ static int do_test_rsa(struct crypto_akcipher *tfm, /* Run RSA encrypt - c = m^e mod n;*/ err = wait_async_op(&result, crypto_akcipher_encrypt(req)); if (err) { - pr_err("alg: rsa: encrypt test failed. err %d\n", err); + pr_err("alg: akcipher: encrypt test failed. err %d\n", err); goto free_all; } if (req->dst_len != vecs->c_size) { - pr_err("alg: rsa: encrypt test failed. Invalid output len\n"); + pr_err("alg: akcipher: encrypt test failed. Invalid output len\n"); err = -EINVAL; goto free_all; } /* verify that encrypted message is equal to expected */ if (memcmp(vecs->c, outbuf_enc, vecs->c_size)) { - pr_err("alg: rsa: encrypt test failed. Invalid output\n"); + pr_err("alg: akcipher: encrypt test failed. Invalid output\n"); + hexdump(outbuf_enc, vecs->c_size); err = -EINVAL; goto free_all; } @@ -1867,18 +2003,22 @@ static int do_test_rsa(struct crypto_akcipher *tfm, /* Run RSA decrypt - m = c^d mod n;*/ err = wait_async_op(&result, crypto_akcipher_decrypt(req)); if (err) { - pr_err("alg: rsa: decrypt test failed. err %d\n", err); + pr_err("alg: akcipher: decrypt test failed. err %d\n", err); goto free_all; } out_len = req->dst_len; - if (out_len != vecs->m_size) { - pr_err("alg: rsa: decrypt test failed. Invalid output len\n"); + if (out_len < vecs->m_size) { + pr_err("alg: akcipher: decrypt test failed. " + "Invalid output len %u\n", out_len); err = -EINVAL; goto free_all; } /* verify that decrypted message is equal to the original msg */ - if (memcmp(vecs->m, outbuf_dec, vecs->m_size)) { - pr_err("alg: rsa: decrypt test failed. Invalid output\n"); + if (memchr_inv(outbuf_dec, 0, out_len - vecs->m_size) || + memcmp(vecs->m, outbuf_dec + out_len - vecs->m_size, + vecs->m_size)) { + pr_err("alg: akcipher: decrypt test failed. Invalid output\n"); + hexdump(outbuf_dec, out_len); err = -EINVAL; } free_all: @@ -1891,28 +2031,22 @@ free_xbuf: return err; } -static int test_rsa(struct crypto_akcipher *tfm, struct akcipher_testvec *vecs, - unsigned int tcount) +static int test_akcipher(struct crypto_akcipher *tfm, const char *alg, + struct akcipher_testvec *vecs, unsigned int tcount) { + const char *algo = + crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm)); int ret, i; for (i = 0; i < tcount; i++) { - ret = do_test_rsa(tfm, vecs++); - if (ret) { - pr_err("alg: rsa: test failed on vector %d, err=%d\n", - i + 1, ret); - return ret; - } - } - return 0; -} - -static int test_akcipher(struct crypto_akcipher *tfm, const char *alg, - struct akcipher_testvec *vecs, unsigned int tcount) -{ - if (strncmp(alg, "rsa", 3) == 0) - return test_rsa(tfm, vecs, tcount); + ret = test_akcipher_one(tfm, vecs++); + if (!ret) + continue; + pr_err("alg: akcipher: test %d failed for %s, err=%d\n", + i + 1, algo, ret); + return ret; + } return 0; } @@ -2728,6 +2862,16 @@ static const struct alg_test_desc alg_test_descs[] = { } } } + }, { + .alg = "dh", + .test = alg_test_kpp, + .fips_allowed = 1, + .suite = { + .kpp = { + .vecs = dh_tv_template, + .count = DH_TEST_VECTORS + } + } }, { .alg = "digest_null", .test = alg_test_null, @@ -3156,6 +3300,16 @@ static const struct alg_test_desc alg_test_descs[] = { } } } + }, { + .alg = "ecdh", + .test = alg_test_kpp, + .fips_allowed = 1, + .suite = { + .kpp = { + .vecs = ecdh_tv_template, + .count = ECDH_TEST_VECTORS + } + } }, { .alg = "gcm(aes)", .test = alg_test_aead, @@ -3248,6 +3402,46 @@ static const struct alg_test_desc alg_test_descs[] = { .count = HMAC_SHA256_TEST_VECTORS } } + }, { + .alg = "hmac(sha3-224)", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = { + .vecs = hmac_sha3_224_tv_template, + .count = HMAC_SHA3_224_TEST_VECTORS + } + } + }, { + .alg = "hmac(sha3-256)", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = { + .vecs = hmac_sha3_256_tv_template, + .count = HMAC_SHA3_256_TEST_VECTORS + } + } + }, { + .alg = "hmac(sha3-384)", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = { + .vecs = hmac_sha3_384_tv_template, + .count = HMAC_SHA3_384_TEST_VECTORS + } + } + }, { + .alg = "hmac(sha3-512)", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = { + .vecs = hmac_sha3_512_tv_template, + .count = HMAC_SHA3_512_TEST_VECTORS + } + } }, { .alg = "hmac(sha384)", .test = alg_test_hash, @@ -3658,6 +3852,46 @@ static const struct alg_test_desc alg_test_descs[] = { .count = SHA256_TEST_VECTORS } } + }, { + .alg = "sha3-224", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = { + .vecs = sha3_224_tv_template, + .count = SHA3_224_TEST_VECTORS + } + } + }, { + .alg = "sha3-256", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = { + .vecs = sha3_256_tv_template, + .count = SHA3_256_TEST_VECTORS + } + } + }, { + .alg = "sha3-384", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = { + .vecs = sha3_384_tv_template, + .count = SHA3_384_TEST_VECTORS + } + } + }, { + .alg = "sha3-512", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = { + .vecs = sha3_512_tv_template, + .count = SHA3_512_TEST_VECTORS + } + } }, { .alg = "sha384", .test = alg_test_hash, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 487ec880e889..acb6bbff781a 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -133,6 +133,17 @@ struct akcipher_testvec { bool public_key_vec; }; +struct kpp_testvec { + unsigned char *secret; + unsigned char *b_public; + unsigned char *expected_a_public; + unsigned char *expected_ss; + unsigned short secret_size; + unsigned short b_public_size; + unsigned short expected_a_public_size; + unsigned short expected_ss_size; +}; + static char zeroed_string[48]; /* @@ -141,7 +152,7 @@ static char zeroed_string[48]; #ifdef CONFIG_CRYPTO_FIPS #define RSA_TEST_VECTORS 2 #else -#define RSA_TEST_VECTORS 4 +#define RSA_TEST_VECTORS 5 #endif static struct akcipher_testvec rsa_tv_template[] = { { @@ -327,6 +338,516 @@ static struct akcipher_testvec rsa_tv_template[] = { .m_size = 8, .c_size = 256, .public_key_vec = true, + }, { + .key = + "\x30\x82\x09\x29" /* sequence of 2345 bytes */ + "\x02\x01\x00" /* version integer of 1 byte */ + "\x02\x82\x02\x01" /* modulus - integer of 513 bytes */ + "\x00\xC3\x8B\x55\x7B\x73\x4D\xFF\xE9\x9B\xC6\xDC\x67\x3C\xB4\x8E" + "\xA0\x86\xED\xF2\xB9\x50\x5C\x54\x5C\xBA\xE4\xA1\xB2\xA7\xAE\x2F" + "\x1B\x7D\xF1\xFB\xAC\x79\xC5\xDF\x1A\x00\xC9\xB2\xC1\x61\x25\x33" + "\xE6\x9C\xE9\xCF\xD6\x27\xC4\x4E\x44\x30\x44\x5E\x08\xA1\x87\x52" + "\xCC\x6B\x97\x70\x8C\xBC\xA5\x06\x31\x0C\xD4\x2F\xD5\x7D\x26\x24" + "\xA2\xE2\xAC\x78\xF4\x53\x14\xCE\xF7\x19\x2E\xD7\xF7\xE6\x0C\xB9" + "\x56\x7F\x0B\xF1\xB1\xE2\x43\x70\xBD\x86\x1D\xA1\xCC\x2B\x19\x08" + "\x76\xEF\x91\xAC\xBF\x20\x24\x0D\x38\xC0\x89\xB8\x9A\x70\xB3\x64" + "\xD9\x8F\x80\x41\x10\x5B\x9F\xB1\xCB\x76\x43\x00\x21\x25\x36\xD4" + "\x19\xFC\x55\x95\x10\xE4\x26\x74\x98\x2C\xD9\xBD\x0B\x2B\x04\xC2" + "\xAC\x82\x38\xB4\xDD\x4C\x04\x7E\x51\x36\x40\x1E\x0B\xC4\x7C\x25" + "\xDD\x4B\xB2\xE7\x20\x0A\x57\xF9\xB4\x94\xC3\x08\x33\x22\x6F\x8B" + "\x48\xDB\x03\x68\x5A\x5B\xBA\xAE\xF3\xAD\xCF\xC3\x6D\xBA\xF1\x28" + "\x67\x7E\x6C\x79\x07\xDE\xFC\xED\xE7\x96\xE3\x6C\xE0\x2C\x87\xF8" + "\x02\x01\x28\x38\x43\x21\x53\x84\x69\x75\x78\x15\x7E\xEE\xD2\x1B" + "\xB9\x23\x40\xA8\x86\x1E\x38\x83\xB2\x73\x1D\x53\xFB\x9E\x2A\x8A" + "\xB2\x75\x35\x01\xC3\xC3\xC4\x94\xE8\x84\x86\x64\x81\xF4\x42\xAA" + "\x3C\x0E\xD6\x4F\xBC\x0A\x09\x2D\xE7\x1B\xD4\x10\xA8\x54\xEA\x89" + "\x84\x8A\xCB\xF7\x5A\x3C\xCA\x76\x08\x29\x62\xB4\x6A\x22\xDF\x14" + "\x95\x71\xFD\xB6\x86\x39\xB8\x8B\xF8\x91\x7F\x38\xAA\x14\xCD\xE5" + "\xF5\x1D\xC2\x6D\x53\x69\x52\x84\x7F\xA3\x1A\x5E\x26\x04\x83\x06" + "\x73\x52\x56\xCF\x76\x26\xC9\xDD\x75\xD7\xFC\xF4\x69\xD8\x7B\x55" + "\xB7\x68\x13\x53\xB9\xE7\x89\xC3\xE8\xD6\x6E\xA7\x6D\xEA\x81\xFD" + "\xC4\xB7\x05\x5A\xB7\x41\x0A\x23\x8E\x03\x8A\x1C\xAE\xD3\x1E\xCE" + "\xE3\x5E\xFC\x19\x4A\xEE\x61\x9B\x8E\xE5\xE5\xDD\x85\xF9\x41\xEC" + "\x14\x53\x92\xF7\xDD\x06\x85\x02\x91\xE3\xEB\x6C\x43\x03\xB1\x36" + "\x7B\x89\x5A\xA8\xEB\xFC\xD5\xA8\x35\xDC\x81\xD9\x5C\xBD\xCA\xDC" + "\x9B\x98\x0B\x06\x5D\x0C\x5B\xEE\xF3\xD5\xCC\x57\xC9\x71\x2F\x90" + "\x3B\x3C\xF0\x8E\x4E\x35\x48\xAE\x63\x74\xA9\xFC\x72\x75\x8E\x34" + "\xA8\xF2\x1F\xEA\xDF\x3A\x37\x2D\xE5\x39\x39\xF8\x57\x58\x3C\x04" + "\xFE\x87\x06\x98\xBC\x7B\xD3\x21\x36\x60\x25\x54\xA7\x3D\xFA\x91" + "\xCC\xA8\x0B\x92\x8E\xB4\xF7\x06\xFF\x1E\x95\xCB\x07\x76\x97\x3B" + "\x9D" + "\x02\x03\x01\x00\x01" /* public key integer of 3 bytes */ + "\x02\x82\x02\x00" /* private key integer of 512 bytes */ + "\x74\xA9\xE0\x6A\x32\xB4\xCA\x85\xD9\x86\x9F\x60\x88\x7B\x40\xCC" + "\xCD\x33\x91\xA8\xB6\x25\x1F\xBF\xE3\x51\x1C\x97\xB6\x2A\xD9\xB8" + "\x11\x40\x19\xE3\x21\x13\xC8\xB3\x7E\xDC\xD7\x65\x40\x4C\x2D\xD6" + "\xDC\xAF\x32\x6C\x96\x75\x2C\x2C\xCA\x8F\x3F\x7A\xEE\xC4\x09\xC6" + "\x24\x3A\xC9\xCF\x6D\x8D\x17\x50\x94\x52\xD3\xE7\x0F\x2F\x7E\x94" + "\x1F\xA0\xBE\xD9\x25\xE8\x38\x42\x7C\x27\xD2\x79\xF8\x2A\x87\x38" + "\xEF\xBB\x74\x8B\xA8\x6E\x8C\x08\xC6\xC7\x4F\x0C\xBC\x79\xC6\xEF" + "\x0E\xA7\x5E\xE4\xF8\x8C\x09\xC7\x5E\x37\xCC\x87\x77\xCD\xCF\xD1" + "\x6D\x28\x1B\xA9\x62\xC0\xB8\x16\xA7\x8B\xF9\xBB\xCC\xB4\x15\x7F" + "\x1B\x69\x03\xF2\x7B\xEB\xE5\x8C\x14\xD6\x23\x4F\x52\x6F\x18\xA6" + "\x4B\x5B\x01\xAD\x35\xF9\x48\x53\xB3\x86\x35\x66\xD7\xE7\x29\xC0" + "\x09\xB5\xC6\xE6\xFA\xC4\xDA\x19\xBE\xD7\x4D\x41\x14\xBE\x6F\xDF" + "\x1B\xAB\xC0\xCA\x88\x07\xAC\xF1\x7D\x35\x83\x67\x28\x2D\x50\xE9" + "\xCE\x27\x71\x5E\x1C\xCF\xD2\x30\x65\x79\x72\x2F\x9C\xE1\xD2\x39" + "\x7F\xEF\x3B\x01\xF2\x14\x1D\xDF\xBD\x51\xD3\xA1\x53\x62\xCF\x5F" + "\x79\x84\xCE\x06\x96\x69\x29\x49\x82\x1C\x71\x4A\xA1\x66\xC8\x2F" + "\xFD\x7B\x96\x7B\xFC\xC4\x26\x58\xC4\xFC\x7C\xAF\xB5\xE8\x95\x83" + "\x87\xCB\x46\xDE\x97\xA7\xB3\xA2\x54\x5B\xD7\xAF\xAB\xEB\xC8\xF3" + "\x55\x9D\x48\x2B\x30\x9C\xDC\x26\x4B\xC2\x89\x45\x13\xB2\x01\x9A" + "\xA4\x65\xC3\xEC\x24\x2D\x26\x97\xEB\x80\x8A\x9D\x03\xBC\x59\x66" + "\x9E\xE2\xBB\xBB\x63\x19\x64\x93\x11\x7B\x25\x65\x30\xCD\x5B\x4B" + "\x2C\xFF\xDC\x2D\x30\x87\x1F\x3C\x88\x07\xD0\xFC\x48\xCC\x05\x8A" + "\xA2\xC8\x39\x3E\xD5\x51\xBC\x0A\xBE\x6D\xA8\xA0\xF6\x88\x06\x79" + "\x13\xFF\x1B\x45\xDA\x54\xC9\x24\x25\x8A\x75\x0A\x26\xD1\x69\x81" + "\x14\x14\xD1\x79\x7D\x8E\x76\xF2\xE0\xEB\xDD\x0F\xDE\xC2\xEC\x80" + "\xD7\xDC\x16\x99\x92\xBE\xCB\x40\x0C\xCE\x7C\x3B\x46\xA2\x5B\x5D" + "\x0C\x45\xEB\xE1\x00\xDE\x72\x50\xB1\xA6\x0B\x76\xC5\x8D\xFC\x82" + "\x38\x6D\x99\x14\x1D\x1A\x4A\xD3\x7C\x53\xB8\x12\x46\xA2\x30\x38" + "\x82\xF4\x96\x6E\x8C\xCE\x47\x0D\xAF\x0A\x3B\x45\xB7\x43\x95\x43" + "\x9E\x02\x2C\x44\x07\x6D\x1F\x3C\x66\x89\x09\xB6\x1F\x06\x30\xCC" + "\xAD\xCE\x7D\x9A\xDE\x3E\xFB\x6C\xE4\x58\x43\xD2\x4F\xA5\x9E\x5E" + "\xA7\x7B\xAE\x3A\xF6\x7E\xD9\xDB\xD3\xF5\xC5\x41\xAF\xE6\x9C\x91" + "\x02\x82\x01\x01" /* prime1 - integer of 257 bytes */ + "\x00\xE0\xA6\x6C\xF0\xA2\xF8\x81\x85\x36\x43\xD0\x13\x0B\x33\x8B" + "\x8F\x78\x3D\xAC\xC7\x5E\x46\x6A\x7F\x05\xAE\x3E\x26\x0A\xA6\xD0" + "\x51\xF3\xC8\x61\xF5\x77\x22\x48\x10\x87\x4C\xD5\xA4\xD5\xAE\x2D" + "\x4E\x7A\xFE\x1C\x31\xE7\x6B\xFF\xA4\x69\x20\xF9\x2A\x0B\x99\xBE" + "\x7C\x32\x68\xAD\xB0\xC6\x94\x81\x41\x75\xDC\x06\x78\x0A\xB4\xCF" + "\xCD\x1B\x2D\x31\xE4\x7B\xEA\xA8\x35\x99\x75\x57\xC6\x0E\xF6\x78" + "\x4F\xA0\x92\x4A\x00\x1B\xE7\x96\xF2\x5B\xFD\x2C\x0A\x0A\x13\x81" + "\xAF\xCB\x59\x87\x31\xD9\x83\x65\xF2\x22\x48\xD0\x03\x67\x39\xF6" + "\xFF\xA8\x36\x07\x3A\x68\xE3\x7B\xA9\x64\xFD\x9C\xF7\xB1\x3D\xBF" + "\x26\x5C\xCC\x7A\xFC\xA2\x8F\x51\xD1\xE1\xE2\x3C\xEC\x06\x75\x7C" + "\x34\xF9\xA9\x33\x70\x11\xAD\x5A\xDC\x5F\xCF\x50\xF6\x23\x2F\x39" + "\xAC\x92\x48\x53\x4D\x01\x96\x3C\xD8\xDC\x1F\x23\x23\x78\x80\x34" + "\x54\x14\x76\x8B\xB6\xBB\xFB\x88\x78\x31\x59\x28\xD2\xB1\x75\x17" + "\x88\x04\x4A\x78\x62\x18\x2E\xF5\xFB\x9B\xEF\x15\xD8\x16\x47\xC6" + "\x42\xB1\x02\xDA\x9E\xE3\x84\x90\xB4\x2D\xC3\xCE\x13\xC9\x12\x7D" + "\x3E\xCD\x39\x39\xC9\xAD\xA1\x1A\xE6\xD5\xAD\x5A\x09\x4D\x1B\x0C" + "\xAB" + "\x02\x82\x01\x01" /* prime 2 - integer of 257 bytes */ + "\x00\xDE\xD5\x1B\xF6\xCD\x83\xB1\xC6\x47\x7E\xB9\xC0\x6B\xA9\xB8" + "\x02\xF3\xAE\x40\x5D\xFC\xD3\xE5\x4E\xF1\xE3\x39\x04\x52\x84\x89" + "\x40\x37\xBB\xC2\xCD\x7F\x71\x77\x17\xDF\x6A\x4C\x31\x24\x7F\xB9" + "\x7E\x7F\xC8\x43\x4A\x3C\xEB\x8D\x1B\x7F\x21\x51\x67\x45\x8F\xA0" + "\x36\x29\x3A\x18\x45\xA5\x32\xEC\x74\x88\x3C\x98\x5D\x67\x3B\xD7" + "\x51\x1F\xE9\xAE\x09\x01\xDE\xDE\x7C\xFB\x60\xD1\xA5\x6C\xE9\x6A" + "\x93\x04\x02\x3A\xBB\x67\x02\xB9\xFD\x23\xF0\x02\x2B\x49\x85\xC9" + "\x5B\xE7\x4B\xDF\xA3\xF4\xEE\x59\x4C\x45\xEF\x8B\xC1\x6B\xDE\xDE" + "\xBC\x1A\xFC\xD2\x76\x3F\x33\x74\xA9\x8E\xA3\x7E\x0C\xC6\xCE\x70" + "\xA1\x5B\xA6\x77\xEA\x76\xEB\x18\xCE\xB9\xD7\x78\x8D\xAE\x06\xBB" + "\xD3\x1F\x16\x0D\x05\xAB\x4F\xC6\x52\xC8\x6B\x36\x51\x7D\x1D\x27" + "\xAF\x88\x9A\x6F\xCC\x25\x2E\x74\x06\x72\xCE\x9E\xDB\xE0\x9D\x30" + "\xEF\x55\xA5\x58\x21\xA7\x42\x12\x2C\x2C\x23\x87\xC1\x0F\xE8\x51" + "\xDA\x53\xDA\xFC\x05\x36\xDF\x08\x0E\x08\x36\xBE\x5C\x86\x9E\xCA" + "\x68\x90\x33\x12\x0B\x14\x82\xAB\x90\x1A\xD4\x49\x32\x9C\xBD\xAA" + "\xAB\x4E\x38\xF1\xEE\xED\x3D\x3F\xE8\xBD\x48\x56\xA6\x64\xEE\xC8" + "\xD7" + "\x02\x82\x01\x01" /* exponent 1 - integer of 257 bytes */ + "\x00\x96\x5E\x6F\x8F\x06\xD6\xE6\x03\x1F\x96\x76\x81\x38\xBF\x30" + "\xCC\x40\x84\xAF\xD0\xE7\x06\xA5\x24\x0E\xCE\x59\xA5\x26\xFE\x0F" + "\x74\xBB\x83\xC6\x26\x02\xAF\x3C\xA3\x6B\x9C\xFF\x68\x0C\xEB\x40" + "\x42\x46\xCB\x2E\x5E\x2C\xF4\x3A\x32\x77\x77\xED\xAF\xBA\x02\x17" + "\xE1\x93\xF0\x43\x4A\x8F\x31\x39\xEF\x72\x0F\x6B\x79\x10\x59\x84" + "\xBA\x5A\x55\x7F\x0E\xDB\xEE\xEE\xD6\xA9\xB8\x44\x9F\x3A\xC6\xB9" + "\x33\x3B\x5C\x90\x11\xD0\x9B\xCC\x8A\xBF\x0E\x10\x5B\x4B\xF1\x50" + "\x9E\x35\xB3\xE0\x6D\x7A\x95\x9C\x38\x5D\xC0\x75\x13\xC2\x15\xA7" + "\x81\xEA\xBA\xF7\x4D\x9E\x85\x9D\xF1\x7D\xBA\xD0\x45\x6F\x2A\xD0" + "\x76\xC2\x28\xD0\xAD\xA7\xB5\xDC\xE3\x6A\x99\xFF\x83\x50\xB3\x75" + "\x07\x14\x91\xAF\xEF\x74\xB5\x9F\x9A\xE0\xBA\xA9\x0B\x87\xF3\x85" + "\x5C\x40\xB2\x0E\xA7\xFD\xC6\xED\x45\x8E\xD9\x7C\xB0\xB2\x68\xC6" + "\x1D\xFD\x70\x78\x06\x41\x7F\x95\x12\x36\x9D\xE2\x58\x5D\x15\xEE" + "\x41\x49\xF5\xFA\xEC\x56\x19\xA0\xE6\xE0\xB2\x40\xE1\xD9\xD0\x03" + "\x22\x02\xCF\xD1\x3C\x07\x38\x65\x8F\x65\x0E\xAA\x32\xCE\x25\x05" + "\x16\x73\x51\xB9\x9F\x88\x0B\xCD\x30\xF3\x97\xCC\x2B\x6B\xA4\x0E" + "\x6F" + "\x02\x82\x01\x00" /* exponent 2 - integer of 256 bytes */ + "\x2A\x5F\x3F\xB8\x08\x90\x58\x47\xA9\xE4\xB1\x11\xA3\xE7\x5B\xF4" + "\x43\xBE\x08\xC3\x56\x86\x3C\x7E\x6C\x84\x96\x9C\xF9\xCB\xF6\x05" + "\x5E\x13\xB8\x11\x37\x80\xAD\xF2\xBE\x2B\x0A\x5D\xF5\xE0\xCB\xB7" + "\x00\x39\x66\x82\x41\x5F\x51\x2F\xBF\x56\xE8\x91\xC8\xAA\x6C\xFE" + "\x9F\x8C\x4A\x7D\x43\xD2\x91\x1F\xFF\x9F\xF6\x21\x1C\xB6\x46\x55" + "\x48\xCA\x38\xAB\xC1\xCD\x4D\x65\x5A\xAF\xA8\x6D\xDA\x6D\xF0\x34" + "\x10\x79\x14\x0D\xFA\xA2\x8C\x17\x54\xB4\x18\xD5\x7E\x5F\x90\x50" + "\x87\x84\xE7\xFB\xD7\x61\x53\x5D\xAB\x96\xC7\x6E\x7A\x42\xA0\xFC" + "\x07\xED\xB7\x5F\x80\xD9\x19\xFF\xFB\xFD\x9E\xC4\x73\x31\x62\x3D" + "\x6C\x9E\x15\x03\x62\xA5\x85\xCC\x19\x8E\x9D\x7F\xE3\x6D\xA8\x5D" + "\x96\xF5\xAC\x78\x3D\x81\x27\xE7\x29\xF1\x29\x1D\x09\xBB\x77\x86" + "\x6B\x65\x62\x88\xE1\x31\x1A\x22\xF7\xC5\xCE\x73\x65\x1C\xBE\xE7" + "\x63\xD3\xD3\x14\x63\x27\xAF\x28\xF3\x23\xB6\x76\xC1\xBD\x9D\x82" + "\xF4\x9B\x19\x7D\x2C\x57\xF0\xC2\x2A\x51\xAE\x95\x0D\x8C\x38\x54" + "\xF5\xC6\xA0\x51\xB7\x0E\xB9\xEC\xE7\x0D\x22\xF6\x1A\xD3\xFE\x16" + "\x21\x03\xB7\x0D\x85\xD3\x35\xC9\xDD\xE4\x59\x85\xBE\x7F\xA1\x75" + "\x02\x82\x01\x01" /* coefficient - integer of 257 bytes */ + "\x00\xB9\x48\xD2\x54\x2F\x19\x54\x64\xAE\x62\x80\x61\x89\x80\xB4" + "\x48\x0B\x8D\x7E\x1B\x0F\x50\x08\x82\x3F\xED\x75\x84\xB7\x13\xE4" + "\xF8\x8D\xA8\xBB\x54\x21\x4C\x5A\x54\x07\x16\x4B\xB4\xA4\x9E\x30" + "\xBF\x7A\x30\x1B\x39\x60\xA3\x21\x53\xFB\xB0\xDC\x0F\x7C\x2C\xFB" + "\xAA\x95\x7D\x51\x39\x28\x33\x1F\x25\x31\x53\xF5\xD2\x64\x2B\xF2" + "\x1E\xB3\xC0\x6A\x0B\xC9\xA4\x42\x64\x5C\xFB\x15\xA3\xE8\x4C\x3A" + "\x9C\x3C\xBE\xA3\x39\x83\x23\xE3\x6D\x18\xCC\xC2\xDC\x63\x8D\xBA" + "\x98\xE0\xE0\x31\x4A\x2B\x37\x9C\x4D\x6B\xF3\x9F\x51\xE4\x43\x5C" + "\x83\x5F\xBF\x5C\xFE\x92\x45\x01\xAF\xF5\xC2\xF4\xB7\x56\x93\xA5" + "\xF4\xAA\x67\x3C\x48\x37\xBD\x9A\x3C\xFE\xA5\x9A\xB0\xD1\x6B\x85" + "\xDD\x81\xD4\xFA\xAD\x31\x83\xA8\x22\x9B\xFD\xB4\x61\xDC\x7A\x51" + "\x59\x62\x10\x1B\x7E\x44\xA3\xFE\x90\x51\x5A\x3E\x02\x87\xAD\xFA" + "\xDD\x0B\x1F\x3D\x35\xAF\xEE\x13\x85\x51\xA7\x42\xC0\xEE\x9E\x20" + "\xE9\xD0\x29\xB2\xE4\x21\xE4\x6D\x62\xB9\xF4\x48\x4A\xD8\x46\x8E" + "\x61\xA6\x2C\x5D\xDF\x8F\x97\x2B\x3A\x75\x1D\x83\x17\x6F\xC6\xB0" + "\xDE\xFC\x14\x25\x06\x5A\x60\xBB\xB8\x21\x89\xD1\xEF\x57\xF1\x71" + "\x3D", + .m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a", + .c = + "\x5c\xce\x9c\xd7\x9a\x9e\xa1\xfe\x7a\x82\x3c\x68\x27\x98\xe3\x5d" + "\xd5\xd7\x07\x29\xf5\xfb\xc3\x1a\x7f\x63\x1e\x62\x31\x3b\x19\x87" + "\x79\x4f\xec\x7b\xf3\xcb\xea\x9b\x95\x52\x3a\x40\xe5\x87\x7b\x72" + "\xd1\x72\xc9\xfb\x54\x63\xd8\xc9\xd7\x2c\xfc\x7b\xc3\x14\x1e\xbc" + "\x18\xb4\x34\xa1\xbf\x14\xb1\x37\x31\x6e\xf0\x1b\x35\x19\x54\x07" + "\xf7\x99\xec\x3e\x63\xe2\xcd\x61\x28\x65\xc3\xcd\xb1\x38\x36\xa5" + "\xb2\xd7\xb0\xdc\x1f\xf5\xef\x19\xc7\x53\x32\x2d\x1c\x26\xda\xe4" + "\x0d\xd6\x90\x7e\x28\xd8\xdc\xe4\x61\x05\xd2\x25\x90\x01\xd3\x96" + "\x6d\xa6\xcf\x58\x20\xbb\x03\xf4\x01\xbc\x79\xb9\x18\xd8\xb8\xba" + "\xbd\x93\xfc\xf2\x62\x5d\x8c\x66\x1e\x0e\x84\x59\x93\xdd\xe2\x93" + "\xa2\x62\x7d\x08\x82\x7a\xdd\xfc\xb8\xbc\xc5\x4f\x9c\x4e\xbf\xb4" + "\xfc\xf4\xc5\x01\xe8\x00\x70\x4d\x28\x26\xcc\x2e\xfe\x0e\x58\x41" + "\x8b\xec\xaf\x7c\x4b\x54\xd0\xa0\x64\xf9\x32\xf4\x2e\x47\x65\x0a" + "\x67\x88\x39\x3a\xdb\xb2\xdb\x7b\xb5\xf6\x17\xa8\xd9\xc6\x5e\x28" + "\x13\x82\x8a\x99\xdb\x60\x08\xa5\x23\x37\xfa\x88\x90\x31\xc8\x9d" + "\x8f\xec\xfb\x85\x9f\xb1\xce\xa6\x24\x50\x46\x44\x47\xcb\x65\xd1" + "\xdf\xc0\xb1\x6c\x90\x1f\x99\x8e\x4d\xd5\x9e\x31\x07\x66\x87\xdf" + "\x01\xaa\x56\x3c\x71\xe0\x2b\x6f\x67\x3b\x23\xed\xc2\xbd\x03\x30" + "\x79\x76\x02\x10\x10\x98\x85\x8a\xff\xfd\x0b\xda\xa5\xd9\x32\x48" + "\x02\xa0\x0b\xb9\x2a\x8a\x18\xca\xc6\x8f\x3f\xbb\x16\xb2\xaa\x98" + "\x27\xe3\x60\x43\xed\x15\x70\xd4\x57\x15\xfe\x19\xd4\x9b\x13\x78" + "\x8a\xf7\x21\xf1\xa2\xa2\x2d\xb3\x09\xcf\x44\x91\x6e\x08\x3a\x30" + "\x81\x3e\x90\x93\x8a\x67\x33\x00\x59\x54\x9a\x25\xd3\x49\x8e\x9f" + "\xc1\x4b\xe5\x86\xf3\x50\x4c\xbc\xc5\xd3\xf5\x3a\x54\xe1\x36\x3f" + "\xe2\x5a\xb4\x37\xc0\xeb\x70\x35\xec\xf6\xb7\xe8\x44\x3b\x7b\xf3" + "\xf1\xf2\x1e\xdb\x60\x7d\xd5\xbe\xf0\x71\x34\x90\x4c\xcb\xd4\x35" + "\x51\xc7\xdd\xd8\xc9\x81\xf5\x5d\x57\x46\x2c\xb1\x7b\x9b\xaa\xcb" + "\xd1\x22\x25\x49\x44\xa3\xd4\x6b\x29\x7b\xd8\xb2\x07\x93\xbf\x3d" + "\x52\x49\x84\x79\xef\xb8\xe5\xc4\xad\xca\xa8\xc6\xf6\xa6\x76\x70" + "\x5b\x0b\xe5\x83\xc6\x0e\xef\x55\xf2\xe7\xff\x04\xea\xe6\x13\xbe" + "\x40\xe1\x40\x45\x48\x66\x75\x31\xae\x35\x64\x91\x11\x6f\xda\xee" + "\x26\x86\x45\x6f\x0b\xd5\x9f\x03\xb1\x65\x5b\xdb\xa4\xe4\xf9\x45", + .key_len = 2349, + .m_size = 8, + .c_size = 512, + } +}; + +#define DH_TEST_VECTORS 2 + +struct kpp_testvec dh_tv_template[] = { + { + .secret = +#ifdef __LITTLE_ENDIAN + "\x01\x00" /* type */ + "\x11\x02" /* len */ + "\x00\x01\x00\x00" /* key_size */ + "\x00\x01\x00\x00" /* p_size */ + "\x01\x00\x00\x00" /* g_size */ +#else + "\x00\x01" /* type */ + "\x02\x11" /* len */ + "\x00\x00\x01\x00" /* key_size */ + "\x00\x00\x01\x00" /* p_size */ + "\x00\x00\x00\x01" /* g_size */ +#endif + /* xa */ + "\x44\xc1\x48\x36\xa7\x2b\x6f\x4e\x43\x03\x68\xad\x31\x00\xda\xf3" + "\x2a\x01\xa8\x32\x63\x5f\x89\x32\x1f\xdf\x4c\xa1\x6a\xbc\x10\x15" + "\x90\x35\xc9\x26\x41\xdf\x7b\xaa\x56\x56\x3d\x85\x44\xb5\xc0\x8e" + "\x37\x83\x06\x50\xb3\x5f\x0e\x28\x2c\xd5\x46\x15\xe3\xda\x7d\x74" + "\x87\x13\x91\x4f\xd4\x2d\xf6\xc7\x5e\x14\x2c\x11\xc2\x26\xb4\x3a" + "\xe3\xb2\x36\x20\x11\x3b\x22\xf2\x06\x65\x66\xe2\x57\x58\xf8\x22" + "\x1a\x94\xbd\x2b\x0e\x8c\x55\xad\x61\x23\x45\x2b\x19\x1e\x63\x3a" + "\x13\x61\xe3\xa0\x79\x70\x3e\x6d\x98\x32\xbc\x7f\x82\xc3\x11\xd8" + "\xeb\x53\xb5\xfc\xb5\xd5\x3c\x4a\xea\x92\x3e\x01\xce\x15\x65\xd4" + "\xaa\x85\xc1\x11\x90\x83\x31\x6e\xfe\xe7\x7f\x7d\xed\xab\xf9\x29" + "\xf8\xc7\xf1\x68\xc6\xb7\xe4\x1f\x2f\x28\xa0\xc9\x1a\x50\x64\x29" + "\x4b\x01\x6d\x1a\xda\x46\x63\x21\x07\x40\x8c\x8e\x4c\x6f\xb5\xe5" + "\x12\xf3\xc2\x1b\x48\x27\x5e\x27\x01\xb1\xaa\xed\x68\x9b\x83\x18" + "\x8f\xb1\xeb\x1f\x04\xd1\x3c\x79\xed\x4b\xf7\x0a\x33\xdc\xe0\xc6" + "\xd8\x02\x51\x59\x00\x74\x30\x07\x4c\x2d\xac\xe4\x13\xf1\x80\xf0" + "\xce\xfa\xff\xa9\xce\x29\x46\xdd\x9d\xad\xd1\xc3\xc6\x58\x1a\x63" + /* p */ + "\xb9\x36\x3a\xf1\x82\x1f\x60\xd3\x22\x47\xb8\xbc\x2d\x22\x6b\x81" + "\x7f\xe8\x20\x06\x09\x23\x73\x49\x9a\x59\x8b\x35\x25\xf8\x31\xbc" + "\x7d\xa8\x1c\x9d\x56\x0d\x1a\xf7\x4b\x4f\x96\xa4\x35\x77\x6a\x89" + "\xab\x42\x00\x49\x21\x71\xed\x28\x16\x1d\x87\x5a\x10\xa7\x9c\x64" + "\x94\xd4\x87\x3d\x28\xef\x44\xfe\x4b\xe2\xb4\x15\x8c\x82\xa6\xf3" + "\x50\x5f\xa8\xe8\xa2\x60\xe7\x00\x86\x78\x05\xd4\x78\x19\xa1\x98" + "\x62\x4e\x4a\x00\x78\x56\x96\xe6\xcf\xd7\x10\x1b\x74\x5d\xd0\x26" + "\x61\xdb\x6b\x32\x09\x51\xd8\xa5\xfd\x54\x16\x71\x01\xb3\x39\xe6" + "\x4e\x69\xb1\xd7\x06\x8f\xd6\x1e\xdc\x72\x25\x26\x74\xc8\x41\x06" + "\x5c\xd1\x26\x5c\xb0\x2f\xf9\x59\x13\xc1\x2a\x0f\x78\xea\x7b\xf7" + "\xbd\x59\xa0\x90\x1d\xfc\x33\x5b\x4c\xbf\x05\x9c\x3a\x3f\x69\xa2" + "\x45\x61\x4e\x10\x6a\xb3\x17\xc5\x68\x30\xfb\x07\x5f\x34\xc6\xfb" + "\x73\x07\x3c\x70\xf6\xae\xe7\x72\x84\xc3\x18\x81\x8f\xe8\x11\x1f" + "\x3d\x83\x83\x01\x2a\x14\x73\xbf\x32\x32\x2e\xc9\x4d\xdb\x2a\xca" + "\xee\x71\xf9\xda\xad\xe8\x82\x0b\x4d\x0c\x1f\xb6\x1d\xef\x00\x67" + "\x74\x3d\x95\xe0\xb7\xc4\x30\x8a\x24\x87\x12\x47\x27\x70\x0d\x73" + /* g */ + "\x02", + .b_public = + "\x2a\x67\x5c\xfd\x63\x5d\xc0\x97\x0a\x8b\xa2\x1f\xf8\x8a\xcb\x54" + "\xca\x2f\xd3\x49\x3f\x01\x8e\x87\xfe\xcc\x94\xa0\x3e\xd4\x26\x79" + "\x9a\x94\x3c\x11\x81\x58\x5c\x60\x3d\xf5\x98\x90\x89\x64\x62\x1f" + "\xbd\x05\x6d\x2b\xcd\x84\x40\x9b\x4a\x1f\xe0\x19\xf1\xca\x20\xb3" + "\x4e\xa0\x4f\x15\xcc\xa5\xfe\xa5\xb4\xf5\x0b\x18\x7a\x5a\x37\xaa" + "\x58\x00\x19\x7f\xe2\xa3\xd9\x1c\x44\x57\xcc\xde\x2e\xc1\x38\xea" + "\xeb\xe3\x90\x40\xc4\x6c\xf7\xcd\xe9\x22\x50\x71\xf5\x7c\xdb\x37" + "\x0e\x80\xc3\xed\x7e\xb1\x2b\x2f\xbe\x71\xa6\x11\xa5\x9d\xf5\x39" + "\xf1\xa2\xe5\x85\xbc\x25\x91\x4e\x84\x8d\x26\x9f\x4f\xe6\x0f\xa6" + "\x2b\x6b\xf9\x0d\xaf\x6f\xbb\xfa\x2d\x79\x15\x31\x57\xae\x19\x60" + "\x22\x0a\xf5\xfd\x98\x0e\xbf\x5d\x49\x75\x58\x37\xbc\x7f\xf5\x21" + "\x56\x1e\xd5\xb3\x50\x0b\xca\x96\xf3\xd1\x3f\xb3\x70\xa8\x6d\x63" + "\x48\xfb\x3d\xd7\x29\x91\x45\xb5\x48\xcd\xb6\x78\x30\xf2\x3f\x1e" + "\xd6\x22\xd6\x35\x9b\xf9\x1f\x85\xae\xab\x4b\xd7\xe0\xc7\x86\x67" + "\x3f\x05\x7f\xa6\x0d\x2f\x0d\xbf\x53\x5f\x4d\x2c\x6d\x5e\x57\x40" + "\x30\x3a\x23\x98\xf9\xb4\x32\xf5\x32\x83\xdd\x0b\xae\x33\x97\x2f", + .expected_a_public = + "\x5c\x24\xdf\xeb\x5b\x4b\xf8\xc5\xef\x39\x48\x82\xe0\x1e\x62\xee" + "\x8a\xae\xdf\x93\x6c\x2b\x16\x95\x92\x16\x3f\x16\x7b\x75\x03\x85" + "\xd9\xf1\x69\xc2\x14\x87\x45\xfc\xa4\x19\xf6\xf0\xa4\xf3\xec\xd4" + "\x6c\x5c\x03\x3b\x94\xc2\x2f\x92\xe4\xce\xb3\xe4\x72\xe8\x17\xe6" + "\x23\x7e\x00\x01\x09\x59\x13\xbf\xc1\x2f\x99\xa9\x07\xaa\x02\x23" + "\x4a\xca\x39\x4f\xbc\xec\x0f\x27\x4f\x19\x93\x6c\xb9\x30\x52\xfd" + "\x2b\x9d\x86\xf1\x06\x1e\xb6\x56\x27\x4a\xc9\x8a\xa7\x8a\x48\x5e" + "\xb5\x60\xcb\xdf\xff\x03\x26\x10\xbf\x90\x8f\x46\x60\xeb\x9b\x9a" + "\xd6\x6f\x44\x91\x03\x92\x18\x2c\x96\x5e\x40\x19\xfb\xf4\x4f\x3a" + "\x02\x7b\xaf\xcc\x22\x20\x79\xb9\xf8\x9f\x8f\x85\x6b\xec\x44\xbb" + "\xe6\xa8\x8e\xb1\xe8\x2c\xee\x64\xee\xf8\xbd\x00\xf3\xe2\x2b\x93" + "\xcd\xe7\xc4\xdf\xc9\x19\x46\xfe\xb6\x07\x73\xc1\x8a\x64\x79\x26" + "\xe7\x30\xad\x2a\xdf\xe6\x8f\x59\xf5\x81\xbf\x4a\x29\x91\xe7\xb7" + "\xcf\x48\x13\x27\x75\x79\x40\xd9\xd6\x32\x52\x4e\x6a\x86\xae\x6f" + "\xc2\xbf\xec\x1f\xc2\x69\xb2\xb6\x59\xe5\xa5\x17\xa4\x77\xb7\x62" + "\x46\xde\xe8\xd2\x89\x78\x9a\xef\xa3\xb5\x8f\x26\xec\x80\xda\x39", + .expected_ss = + "\x8f\xf3\xac\xa2\xea\x22\x11\x5c\x45\x65\x1a\x77\x75\x2e\xcf\x46" + "\x23\x14\x1e\x67\x53\x4d\x35\xb0\x38\x1d\x4e\xb9\x41\x9a\x21\x24" + "\x6e\x9f\x40\xfe\x90\x51\xb1\x06\xa4\x7b\x87\x17\x2f\xe7\x5e\x22" + "\xf0\x7b\x54\x84\x0a\xac\x0a\x90\xd2\xd7\xe8\x7f\xe7\xe3\x30\x75" + "\x01\x1f\x24\x75\x56\xbe\xcc\x8d\x1e\x68\x0c\x41\x72\xd3\xfa\xbb" + "\xe5\x9c\x60\xc7\x28\x77\x0c\xbe\x89\xab\x08\xd6\x21\xe7\x2e\x1a" + "\x58\x7a\xca\x4f\x22\xf3\x2b\x30\xfd\xf4\x98\xc1\xa3\xf8\xf6\xcc" + "\xa9\xe4\xdb\x5b\xee\xd5\x5c\x6f\x62\x4c\xd1\x1a\x02\x2a\x23\xe4" + "\xb5\x57\xf3\xf9\xec\x04\x83\x54\xfe\x08\x5e\x35\xac\xfb\xa8\x09" + "\x82\x32\x60\x11\xb2\x16\x62\x6b\xdf\xda\xde\x9c\xcb\x63\x44\x6c" + "\x59\x26\x6a\x8f\xb0\x24\xcb\xa6\x72\x48\x1e\xeb\xe0\xe1\x09\x44" + "\xdd\xee\x66\x6d\x84\xcf\xa5\xc1\xb8\x36\x74\xd3\x15\x96\xc3\xe4" + "\xc6\x5a\x4d\x23\x97\x0c\x5c\xcb\xa9\xf5\x29\xc2\x0e\xff\x93\x82" + "\xd3\x34\x49\xad\x64\xa6\xb1\xc0\x59\x28\x75\x60\xa7\x8a\xb0\x11" + "\x56\x89\x42\x74\x11\xf5\xf6\x5e\x6f\x16\x54\x6a\xb1\x76\x4d\x50" + "\x8a\x68\xc1\x5b\x82\xb9\x0d\x00\x32\x50\xed\x88\x87\x48\x92\x17", + .secret_size = 529, + .b_public_size = 256, + .expected_a_public_size = 256, + .expected_ss_size = 256, + }, + { + .secret = +#ifdef __LITTLE_ENDIAN + "\x01\x00" /* type */ + "\x11\x02" /* len */ + "\x00\x01\x00\x00" /* key_size */ + "\x00\x01\x00\x00" /* p_size */ + "\x01\x00\x00\x00" /* g_size */ +#else + "\x00\x01" /* type */ + "\x02\x11" /* len */ + "\x00\x00\x01\x00" /* key_size */ + "\x00\x00\x01\x00" /* p_size */ + "\x00\x00\x00\x01" /* g_size */ +#endif + /* xa */ + "\x4d\x75\xa8\x6e\xba\x23\x3a\x0c\x63\x56\xc8\xc9\x5a\xa7\xd6\x0e" + "\xed\xae\x40\x78\x87\x47\x5f\xe0\xa7\x7b\xba\x84\x88\x67\x4e\xe5" + "\x3c\xcc\x5c\x6a\xe7\x4a\x20\xec\xbe\xcb\xf5\x52\x62\x9f\x37\x80" + "\x0c\x72\x7b\x83\x66\xa4\xf6\x7f\x95\x97\x1c\x6a\x5c\x7e\xf1\x67" + "\x37\xb3\x93\x39\x3d\x0b\x55\x35\xd9\xe5\x22\x04\x9f\xf8\xc1\x04" + "\xce\x13\xa5\xac\xe1\x75\x05\xd1\x2b\x53\xa2\x84\xef\xb1\x18\xf4" + "\x66\xdd\xea\xe6\x24\x69\x5a\x49\xe0\x7a\xd8\xdf\x1b\xb7\xf1\x6d" + "\x9b\x50\x2c\xc8\x1c\x1c\xa3\xb4\x37\xfb\x66\x3f\x67\x71\x73\xa9" + "\xff\x5f\xd9\xa2\x25\x6e\x25\x1b\x26\x54\xbf\x0c\xc6\xdb\xea\x0a" + "\x52\x6c\x16\x7c\x27\x68\x15\x71\x58\x73\x9d\xe6\xc2\x80\xaa\x97" + "\x31\x66\xfb\xa6\xfb\xfd\xd0\x9c\x1d\xbe\x81\x48\xf5\x9a\x32\xf1" + "\x69\x62\x18\x78\xae\x72\x36\xe6\x94\x27\xd1\xff\x18\x4f\x28\x6a" + "\x16\xbd\x6a\x60\xee\xe5\xf9\x6d\x16\xe4\xb8\xa6\x41\x9b\x23\x7e" + "\xf7\x9d\xd1\x1d\x03\x15\x66\x3a\xcf\xb6\x2c\x13\x96\x2c\x52\x21" + "\xe4\x2d\x48\x7a\x8a\x5d\xb2\x88\xed\x98\x61\x79\x8b\x6a\x1e\x5f" + "\xd0\x8a\x2d\x99\x5a\x2b\x0f\xbc\xef\x53\x8f\x32\xc1\xa2\x99\x26" + /* p */ + "\xb9\x36\x3a\xf1\x82\x1f\x60\xd3\x22\x47\xb8\xbc\x2d\x22\x6b\x81" + "\x7f\xe8\x20\x06\x09\x23\x73\x49\x9a\x59\x8b\x35\x25\xf8\x31\xbc" + "\x7d\xa8\x1c\x9d\x56\x0d\x1a\xf7\x4b\x4f\x96\xa4\x35\x77\x6a\x89" + "\xab\x42\x00\x49\x21\x71\xed\x28\x16\x1d\x87\x5a\x10\xa7\x9c\x64" + "\x94\xd4\x87\x3d\x28\xef\x44\xfe\x4b\xe2\xb4\x15\x8c\x82\xa6\xf3" + "\x50\x5f\xa8\xe8\xa2\x60\xe7\x00\x86\x78\x05\xd4\x78\x19\xa1\x98" + "\x62\x4e\x4a\x00\x78\x56\x96\xe6\xcf\xd7\x10\x1b\x74\x5d\xd0\x26" + "\x61\xdb\x6b\x32\x09\x51\xd8\xa5\xfd\x54\x16\x71\x01\xb3\x39\xe6" + "\x4e\x69\xb1\xd7\x06\x8f\xd6\x1e\xdc\x72\x25\x26\x74\xc8\x41\x06" + "\x5c\xd1\x26\x5c\xb0\x2f\xf9\x59\x13\xc1\x2a\x0f\x78\xea\x7b\xf7" + "\xbd\x59\xa0\x90\x1d\xfc\x33\x5b\x4c\xbf\x05\x9c\x3a\x3f\x69\xa2" + "\x45\x61\x4e\x10\x6a\xb3\x17\xc5\x68\x30\xfb\x07\x5f\x34\xc6\xfb" + "\x73\x07\x3c\x70\xf6\xae\xe7\x72\x84\xc3\x18\x81\x8f\xe8\x11\x1f" + "\x3d\x83\x83\x01\x2a\x14\x73\xbf\x32\x32\x2e\xc9\x4d\xdb\x2a\xca" + "\xee\x71\xf9\xda\xad\xe8\x82\x0b\x4d\x0c\x1f\xb6\x1d\xef\x00\x67" + "\x74\x3d\x95\xe0\xb7\xc4\x30\x8a\x24\x87\x12\x47\x27\x70\x0d\x73" + /* g */ + "\x02", + .b_public = + "\x99\x4d\xd9\x01\x84\x8e\x4a\x5b\xb8\xa5\x64\x8c\x6c\x00\x5c\x0e" + "\x1e\x1b\xee\x5d\x9f\x53\xe3\x16\x70\x01\xed\xbf\x4f\x14\x36\x6e" + "\xe4\x43\x45\x43\x49\xcc\xb1\xb0\x2a\xc0\x6f\x22\x55\x42\x17\x94" + "\x18\x83\xd7\x2a\x5c\x51\x54\xf8\x4e\x7c\x10\xda\x76\x68\x57\x77" + "\x1e\x62\x03\x30\x04\x7b\x4c\x39\x9c\x54\x01\x54\xec\xef\xb3\x55" + "\xa4\xc0\x24\x6d\x3d\xbd\xcc\x46\x5b\x00\x96\xc7\xea\x93\xd1\x3f" + "\xf2\x6a\x72\xe3\xf2\xc1\x92\x24\x5b\xda\x48\x70\x2c\xa9\x59\x97" + "\x19\xb1\xd6\x54\xb3\x9c\x2e\xb0\x63\x07\x9b\x5e\xac\xb5\xf2\xb1" + "\x5b\xf8\xf3\xd7\x2d\x37\x9b\x68\x6c\xf8\x90\x07\xbc\x37\x9a\xa5" + "\xe2\x91\x12\x25\x47\x77\xe3\x3d\xb2\x95\x69\x44\x0b\x91\x1e\xaf" + "\x7c\x8c\x7c\x34\x41\x6a\xab\x60\x6e\xc6\x52\xec\x7e\x94\x0a\x37" + "\xec\x98\x90\xdf\x3f\x02\xbd\x23\x52\xdd\xd9\xe5\x31\x80\x74\x25" + "\xb6\xd2\xd3\xcc\xd5\xcc\x6d\xf9\x7e\x4d\x78\xab\x77\x51\xfa\x77" + "\x19\x94\x49\x8c\x05\xd4\x75\xed\xd2\xb3\x64\x57\xe0\x52\x99\xc0" + "\x83\xe3\xbb\x5e\x2b\xf1\xd2\xc0\xb1\x37\x36\x0b\x7c\xb5\x63\x96" + "\x8e\xde\x04\x23\x11\x95\x62\x11\x9a\xce\x6f\x63\xc8\xd5\xd1\x8f", + .expected_a_public = + "\x90\x89\xe4\x82\xd6\x0a\xcf\x1a\xae\xce\x1b\x66\xa7\x19\x71\x18" + "\x8f\x95\x4b\x5b\x80\x45\x4a\x5a\x43\x99\x4d\x37\xcf\xa3\xa7\x28" + "\x9c\xc7\x73\xf1\xb2\x17\xf6\x99\xe3\x6b\x56\xcb\x3e\x35\x60\x7d" + "\x65\xc7\x84\x6b\x3e\x60\xee\xcd\xd2\x70\xe7\xc9\x32\x1c\xf0\xb4" + "\xf9\x52\xd9\x88\x75\xfd\x40\x2c\xa7\xbe\x19\x1c\x0a\xae\x93\xe1" + "\x71\xc7\xcd\x4f\x33\x5c\x10\x7d\x39\x56\xfc\x73\x84\xb2\x67\xc3" + "\x77\x26\x20\x97\x2b\xf8\x13\x43\x93\x9c\x9a\xa4\x08\xc7\x34\x83" + "\xe6\x98\x61\xe7\x16\x30\x2c\xb1\xdb\x2a\xb2\xcc\xc3\x02\xa5\x3c" + "\x71\x50\x14\x83\xc7\xbb\xa4\xbe\x98\x1b\xfe\xcb\x43\xe9\x97\x62" + "\xd6\xf0\x8c\xcb\x1c\xba\x1e\xa8\xa6\xa6\x50\xfc\x85\x7d\x47\xbf" + "\xf4\x3e\x23\xd3\x5f\xb2\x71\x3e\x40\x94\xaa\x87\x83\x2c\x6c\x8e" + "\x60\xfd\xdd\xf7\xf4\x76\x03\xd3\x1d\xec\x18\x51\xa3\xf2\x44\x1a" + "\x3f\xb4\x7c\x18\x0d\x68\x65\x92\x54\x0d\x2d\x81\x16\xf1\x84\x66" + "\x89\x92\xd0\x1a\x5e\x1f\x42\x46\x5b\xe5\x83\x86\x80\xd9\xcd\x3a" + "\x5a\x2f\xb9\x59\x9b\xe4\x43\x84\x64\xf3\x09\x1a\x0a\xa2\x64\x0f" + "\x77\x4e\x8d\x8b\xe6\x88\xd1\xfc\xaf\x8f\xdf\x1d\xbc\x31\xb3\xbd", + .expected_ss = + "\x34\xc3\x35\x14\x88\x46\x26\x23\x97\xbb\xdd\x28\x5c\x94\xf6\x47" + "\xca\xb3\x19\xaf\xca\x44\x9b\xc2\x7d\x89\xfd\x96\x14\xfd\x6d\x58" + "\xd8\xc4\x6b\x61\x2a\x0d\xf2\x36\x45\xc8\xe4\xa4\xed\x81\x53\x81" + "\x66\x1e\xe0\x5a\xb1\x78\x2d\x0b\x5c\xb4\xd1\xfc\x90\xc6\x9c\xdb" + "\x5a\x30\x0b\x14\x7d\xbe\xb3\x7d\xb1\xb2\x76\x3c\x6c\xef\x74\x6b" + "\xe7\x1f\x64\x0c\xab\x65\xe1\x76\x5c\x3d\x83\xb5\x8a\xfb\xaf\x0f" + "\xf2\x06\x14\x8f\xa0\xf6\xc1\x89\x78\xf2\xba\x72\x73\x3c\xf7\x76" + "\x21\x67\xbc\x24\x31\xb8\x09\x65\x0f\x0c\x02\x32\x4a\x98\x14\xfc" + "\x72\x2c\x25\x60\x68\x5f\x2f\x30\x1e\x5b\xf0\x3b\xd1\xa2\x87\xa0" + "\x54\xdf\xdb\xc0\xee\x0a\x0f\x47\xc9\x90\x20\x2c\xf9\xe3\x52\xad" + "\x27\x65\x8d\x54\x8d\xa8\xa1\xf3\xed\x15\xd4\x94\x28\x90\x31\x93" + "\x1b\xc0\x51\xbb\x43\x5d\x76\x3b\x1d\x2a\x71\x50\xea\x5d\x48\x94" + "\x7f\x6f\xf1\x48\xdb\x30\xe5\xae\x64\x79\xd9\x7a\xdb\xc6\xff\xd8" + "\x5e\x5a\x64\xbd\xf6\x85\x04\xe8\x28\x6a\xac\xef\xce\x19\x8e\x9a" + "\xfe\x75\xc0\x27\x69\xe3\xb3\x7b\x21\xa7\xb1\x16\xa4\x85\x23\xee" + "\xb0\x1b\x04\x6e\xbd\xab\x16\xde\xfd\x86\x6b\xa9\x95\xd7\x0b\xfd", + .secret_size = 529, + .b_public_size = 256, + .expected_a_public_size = 256, + .expected_ss_size = 256, + } +}; + +#ifdef CONFIG_CRYPTO_FIPS +#define ECDH_TEST_VECTORS 1 +#else +#define ECDH_TEST_VECTORS 2 +#endif +struct kpp_testvec ecdh_tv_template[] = { + { +#ifndef CONFIG_CRYPTO_FIPS + .secret = +#ifdef __LITTLE_ENDIAN + "\x02\x00" /* type */ + "\x20\x00" /* len */ + "\x01\x00" /* curve_id */ + "\x18\x00" /* key_size */ +#else + "\x00\x02" /* type */ + "\x00\x20" /* len */ + "\x00\x01" /* curve_id */ + "\x00\x18" /* key_size */ +#endif + "\xb5\x05\xb1\x71\x1e\xbf\x8c\xda" + "\x4e\x19\x1e\x62\x1f\x23\x23\x31" + "\x36\x1e\xd3\x84\x2f\xcc\x21\x72", + .b_public = + "\xc3\xba\x67\x4b\x71\xec\xd0\x76" + "\x7a\x99\x75\x64\x36\x13\x9a\x94" + "\x5d\x8b\xdc\x60\x90\x91\xfd\x3f" + "\xb0\x1f\x8a\x0a\x68\xc6\x88\x6e" + "\x83\x87\xdd\x67\x09\xf8\x8d\x96" + "\x07\xd6\xbd\x1c\xe6\x8d\x9d\x67", + .expected_a_public = + "\x1a\x04\xdb\xa5\xe1\xdd\x4e\x79" + "\xa3\xe6\xef\x0e\x5c\x80\x49\x85" + "\xfa\x78\xb4\xef\x49\xbd\x4c\x7c" + "\x22\x90\x21\x02\xf9\x1b\x81\x5d" + "\x0c\x8a\xa8\x98\xd6\x27\x69\x88" + "\x5e\xbc\x94\xd8\x15\x9e\x21\xce", + .expected_ss = + "\xf4\x57\xcc\x4f\x1f\x4e\x31\xcc" + "\xe3\x40\x60\xc8\x06\x93\xc6\x2e" + "\x99\x80\x81\x28\xaf\xc5\x51\x74", + .secret_size = 32, + .b_public_size = 48, + .expected_a_public_size = 48, + .expected_ss_size = 24 + }, { +#endif + .secret = +#ifdef __LITTLE_ENDIAN + "\x02\x00" /* type */ + "\x28\x00" /* len */ + "\x02\x00" /* curve_id */ + "\x20\x00" /* key_size */ +#else + "\x00\x02" /* type */ + "\x00\x28" /* len */ + "\x00\x02" /* curve_id */ + "\x00\x20" /* key_size */ +#endif + "\x24\xd1\x21\xeb\xe5\xcf\x2d\x83" + "\xf6\x62\x1b\x6e\x43\x84\x3a\xa3" + "\x8b\xe0\x86\xc3\x20\x19\xda\x92" + "\x50\x53\x03\xe1\xc0\xea\xb8\x82", + .expected_a_public = + "\x1a\x7f\xeb\x52\x00\xbd\x3c\x31" + "\x7d\xb6\x70\xc1\x86\xa6\xc7\xc4" + "\x3b\xc5\x5f\x6c\x6f\x58\x3c\xf5" + "\xb6\x63\x82\x77\x33\x24\xa1\x5f" + "\x6a\xca\x43\x6f\xf7\x7e\xff\x02" + "\x37\x08\xcc\x40\x5e\x7a\xfd\x6a" + "\x6a\x02\x6e\x41\x87\x68\x38\x77" + "\xfa\xa9\x44\x43\x2d\xef\x09\xdf", + .expected_ss = + "\xea\x17\x6f\x7e\x6e\x57\x26\x38" + "\x8b\xfb\x41\xeb\xba\xc8\x6d\xa5" + "\xa8\x72\xd1\xff\xc9\x47\x3d\xaa" + "\x58\x43\x9f\x34\x0f\x8c\xf3\xc9", + .b_public = + "\xcc\xb4\xda\x74\xb1\x47\x3f\xea" + "\x6c\x70\x9e\x38\x2d\xc7\xaa\xb7" + "\x29\xb2\x47\x03\x19\xab\xdd\x34" + "\xbd\xa8\x2c\x93\xe1\xa4\x74\xd9" + "\x64\x63\xf7\x70\x20\x2f\xa4\xe6" + "\x9f\x4a\x38\xcc\xc0\x2c\x49\x2f" + "\xb1\x32\xbb\xaf\x22\x61\xda\xcb" + "\x6f\xdb\xa9\xaa\xfc\x77\x81\xf3", + .secret_size = 40, + .b_public_size = 64, + .expected_a_public_size = 64, + .expected_ss_size = 32 } }; @@ -376,6 +897,131 @@ static struct hash_testvec md4_tv_template [] = { }, }; +#define SHA3_224_TEST_VECTORS 3 +static struct hash_testvec sha3_224_tv_template[] = { + { + .plaintext = "", + .digest = "\x6b\x4e\x03\x42\x36\x67\xdb\xb7" + "\x3b\x6e\x15\x45\x4f\x0e\xb1\xab" + "\xd4\x59\x7f\x9a\x1b\x07\x8e\x3f" + "\x5b\x5a\x6b\xc7", + }, { + .plaintext = "a", + .psize = 1, + .digest = "\x9e\x86\xff\x69\x55\x7c\xa9\x5f" + "\x40\x5f\x08\x12\x69\x68\x5b\x38" + "\xe3\xa8\x19\xb3\x09\xee\x94\x2f" + "\x48\x2b\x6a\x8b", + }, { + .plaintext = "abcdbcdecdefdefgefghfghighijhijkijkl" + "jklmklmnlmnomnopnopq", + .psize = 56, + .digest = "\x8a\x24\x10\x8b\x15\x4a\xda\x21" + "\xc9\xfd\x55\x74\x49\x44\x79\xba" + "\x5c\x7e\x7a\xb7\x6e\xf2\x64\xea" + "\xd0\xfc\xce\x33", + }, +}; + +#define SHA3_256_TEST_VECTORS 3 +static struct hash_testvec sha3_256_tv_template[] = { + { + .plaintext = "", + .digest = "\xa7\xff\xc6\xf8\xbf\x1e\xd7\x66" + "\x51\xc1\x47\x56\xa0\x61\xd6\x62" + "\xf5\x80\xff\x4d\xe4\x3b\x49\xfa" + "\x82\xd8\x0a\x4b\x80\xf8\x43\x4a", + }, { + .plaintext = "a", + .psize = 1, + .digest = "\x80\x08\x4b\xf2\xfb\xa0\x24\x75" + "\x72\x6f\xeb\x2c\xab\x2d\x82\x15" + "\xea\xb1\x4b\xc6\xbd\xd8\xbf\xb2" + "\xc8\x15\x12\x57\x03\x2e\xcd\x8b", + }, { + .plaintext = "abcdbcdecdefdefgefghfghighijhijkijkl" + "jklmklmnlmnomnopnopq", + .psize = 56, + .digest = "\x41\xc0\xdb\xa2\xa9\xd6\x24\x08" + "\x49\x10\x03\x76\xa8\x23\x5e\x2c" + "\x82\xe1\xb9\x99\x8a\x99\x9e\x21" + "\xdb\x32\xdd\x97\x49\x6d\x33\x76", + }, +}; + + +#define SHA3_384_TEST_VECTORS 3 +static struct hash_testvec sha3_384_tv_template[] = { + { + .plaintext = "", + .digest = "\x0c\x63\xa7\x5b\x84\x5e\x4f\x7d" + "\x01\x10\x7d\x85\x2e\x4c\x24\x85" + "\xc5\x1a\x50\xaa\xaa\x94\xfc\x61" + "\x99\x5e\x71\xbb\xee\x98\x3a\x2a" + "\xc3\x71\x38\x31\x26\x4a\xdb\x47" + "\xfb\x6b\xd1\xe0\x58\xd5\xf0\x04", + }, { + .plaintext = "a", + .psize = 1, + .digest = "\x18\x15\xf7\x74\xf3\x20\x49\x1b" + "\x48\x56\x9e\xfe\xc7\x94\xd2\x49" + "\xee\xb5\x9a\xae\x46\xd2\x2b\xf7" + "\x7d\xaf\xe2\x5c\x5e\xdc\x28\xd7" + "\xea\x44\xf9\x3e\xe1\x23\x4a\xa8" + "\x8f\x61\xc9\x19\x12\xa4\xcc\xd9", + }, { + .plaintext = "abcdbcdecdefdefgefghfghighijhijkijkl" + "jklmklmnlmnomnopnopq", + .psize = 56, + .digest = "\x99\x1c\x66\x57\x55\xeb\x3a\x4b" + "\x6b\xbd\xfb\x75\xc7\x8a\x49\x2e" + "\x8c\x56\xa2\x2c\x5c\x4d\x7e\x42" + "\x9b\xfd\xbc\x32\xb9\xd4\xad\x5a" + "\xa0\x4a\x1f\x07\x6e\x62\xfe\xa1" + "\x9e\xef\x51\xac\xd0\x65\x7c\x22", + }, +}; + + +#define SHA3_512_TEST_VECTORS 3 +static struct hash_testvec sha3_512_tv_template[] = { + { + .plaintext = "", + .digest = "\xa6\x9f\x73\xcc\xa2\x3a\x9a\xc5" + "\xc8\xb5\x67\xdc\x18\x5a\x75\x6e" + "\x97\xc9\x82\x16\x4f\xe2\x58\x59" + "\xe0\xd1\xdc\xc1\x47\x5c\x80\xa6" + "\x15\xb2\x12\x3a\xf1\xf5\xf9\x4c" + "\x11\xe3\xe9\x40\x2c\x3a\xc5\x58" + "\xf5\x00\x19\x9d\x95\xb6\xd3\xe3" + "\x01\x75\x85\x86\x28\x1d\xcd\x26", + }, { + .plaintext = "a", + .psize = 1, + .digest = "\x69\x7f\x2d\x85\x61\x72\xcb\x83" + "\x09\xd6\xb8\xb9\x7d\xac\x4d\xe3" + "\x44\xb5\x49\xd4\xde\xe6\x1e\xdf" + "\xb4\x96\x2d\x86\x98\xb7\xfa\x80" + "\x3f\x4f\x93\xff\x24\x39\x35\x86" + "\xe2\x8b\x5b\x95\x7a\xc3\xd1\xd3" + "\x69\x42\x0c\xe5\x33\x32\x71\x2f" + "\x99\x7b\xd3\x36\xd0\x9a\xb0\x2a", + }, { + .plaintext = "abcdbcdecdefdefgefghfghighijhijkijkl" + "jklmklmnlmnomnopnopq", + .psize = 56, + .digest = "\x04\xa3\x71\xe8\x4e\xcf\xb5\xb8" + "\xb7\x7c\xb4\x86\x10\xfc\xa8\x18" + "\x2d\xd4\x57\xce\x6f\x32\x6a\x0f" + "\xd3\xd7\xec\x2f\x1e\x91\x63\x6d" + "\xee\x69\x1f\xbe\x0c\x98\x53\x02" + "\xba\x1b\x0d\x8d\xc7\x8c\x08\x63" + "\x46\xb5\x33\xb4\x9c\x03\x0d\x99" + "\xa2\x7d\xaf\x11\x39\xd6\xe7\x5e", + }, +}; + + /* * MD5 test vectors from RFC1321 */ @@ -3246,6 +3892,394 @@ static struct hash_testvec hmac_sha512_tv_template[] = { }, }; +#define HMAC_SHA3_224_TEST_VECTORS 4 + +static struct hash_testvec hmac_sha3_224_tv_template[] = { + { + .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b\x0b", + .ksize = 20, + .plaintext = "Hi There", + .psize = 8, + .digest = "\x3b\x16\x54\x6b\xbc\x7b\xe2\x70" + "\x6a\x03\x1d\xca\xfd\x56\x37\x3d" + "\x98\x84\x36\x76\x41\xd8\xc5\x9a" + "\xf3\xc8\x60\xf7", + }, { + .key = "Jefe", + .ksize = 4, + .plaintext = "what do ya want for nothing?", + .psize = 28, + .digest = "\x7f\xdb\x8d\xd8\x8b\xd2\xf6\x0d" + "\x1b\x79\x86\x34\xad\x38\x68\x11" + "\xc2\xcf\xc8\x5b\xfa\xf5\xd5\x2b" + "\xba\xce\x5e\x66", + .np = 4, + .tap = { 7, 7, 7, 7 } + }, { + .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + .ksize = 131, + .plaintext = "Test Using Large" + "r Than Block-Siz" + "e Key - Hash Key" + " First", + .psize = 54, + .digest = "\xb4\xa1\xf0\x4c\x00\x28\x7a\x9b" + "\x7f\x60\x75\xb3\x13\xd2\x79\xb8" + "\x33\xbc\x8f\x75\x12\x43\x52\xd0" + "\x5f\xb9\x99\x5f", + }, { + .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + .ksize = 131, + .plaintext = + "This is a test u" + "sing a larger th" + "an block-size ke" + "y and a larger t" + "han block-size d" + "ata. The key nee" + "ds to be hashed " + "before being use" + "d by the HMAC al" + "gorithm.", + .psize = 152, + .digest = "\x05\xd8\xcd\x6d\x00\xfa\xea\x8d" + "\x1e\xb6\x8a\xde\x28\x73\x0b\xbd" + "\x3c\xba\xb6\x92\x9f\x0a\x08\x6b" + "\x29\xcd\x62\xa0", + }, +}; + +#define HMAC_SHA3_256_TEST_VECTORS 4 + +static struct hash_testvec hmac_sha3_256_tv_template[] = { + { + .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b\x0b", + .ksize = 20, + .plaintext = "Hi There", + .psize = 8, + .digest = "\xba\x85\x19\x23\x10\xdf\xfa\x96" + "\xe2\xa3\xa4\x0e\x69\x77\x43\x51" + "\x14\x0b\xb7\x18\x5e\x12\x02\xcd" + "\xcc\x91\x75\x89\xf9\x5e\x16\xbb", + }, { + .key = "Jefe", + .ksize = 4, + .plaintext = "what do ya want for nothing?", + .psize = 28, + .digest = "\xc7\xd4\x07\x2e\x78\x88\x77\xae" + "\x35\x96\xbb\xb0\xda\x73\xb8\x87" + "\xc9\x17\x1f\x93\x09\x5b\x29\x4a" + "\xe8\x57\xfb\xe2\x64\x5e\x1b\xa5", + .np = 4, + .tap = { 7, 7, 7, 7 } + }, { + .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + .ksize = 131, + .plaintext = "Test Using Large" + "r Than Block-Siz" + "e Key - Hash Key" + " First", + .psize = 54, + .digest = "\xed\x73\xa3\x74\xb9\x6c\x00\x52" + "\x35\xf9\x48\x03\x2f\x09\x67\x4a" + "\x58\xc0\xce\x55\x5c\xfc\x1f\x22" + "\x3b\x02\x35\x65\x60\x31\x2c\x3b", + }, { + .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + .ksize = 131, + .plaintext = + "This is a test u" + "sing a larger th" + "an block-size ke" + "y and a larger t" + "han block-size d" + "ata. The key nee" + "ds to be hashed " + "before being use" + "d by the HMAC al" + "gorithm.", + .psize = 152, + .digest = "\x65\xc5\xb0\x6d\x4c\x3d\xe3\x2a" + "\x7a\xef\x87\x63\x26\x1e\x49\xad" + "\xb6\xe2\x29\x3e\xc8\xe7\xc6\x1e" + "\x8d\xe6\x17\x01\xfc\x63\xe1\x23", + }, +}; + +#define HMAC_SHA3_384_TEST_VECTORS 4 + +static struct hash_testvec hmac_sha3_384_tv_template[] = { + { + .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b\x0b", + .ksize = 20, + .plaintext = "Hi There", + .psize = 8, + .digest = "\x68\xd2\xdc\xf7\xfd\x4d\xdd\x0a" + "\x22\x40\xc8\xa4\x37\x30\x5f\x61" + "\xfb\x73\x34\xcf\xb5\xd0\x22\x6e" + "\x1b\xc2\x7d\xc1\x0a\x2e\x72\x3a" + "\x20\xd3\x70\xb4\x77\x43\x13\x0e" + "\x26\xac\x7e\x3d\x53\x28\x86\xbd", + }, { + .key = "Jefe", + .ksize = 4, + .plaintext = "what do ya want for nothing?", + .psize = 28, + .digest = "\xf1\x10\x1f\x8c\xbf\x97\x66\xfd" + "\x67\x64\xd2\xed\x61\x90\x3f\x21" + "\xca\x9b\x18\xf5\x7c\xf3\xe1\xa2" + "\x3c\xa1\x35\x08\xa9\x32\x43\xce" + "\x48\xc0\x45\xdc\x00\x7f\x26\xa2" + "\x1b\x3f\x5e\x0e\x9d\xf4\xc2\x0a", + .np = 4, + .tap = { 7, 7, 7, 7 } + }, { + .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + .ksize = 131, + .plaintext = "Test Using Large" + "r Than Block-Siz" + "e Key - Hash Key" + " First", + .psize = 54, + .digest = "\x0f\xc1\x95\x13\xbf\x6b\xd8\x78" + "\x03\x70\x16\x70\x6a\x0e\x57\xbc" + "\x52\x81\x39\x83\x6b\x9a\x42\xc3" + "\xd4\x19\xe4\x98\xe0\xe1\xfb\x96" + "\x16\xfd\x66\x91\x38\xd3\x3a\x11" + "\x05\xe0\x7c\x72\xb6\x95\x3b\xcc", + }, { + .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + .ksize = 131, + .plaintext = + "This is a test u" + "sing a larger th" + "an block-size ke" + "y and a larger t" + "han block-size d" + "ata. The key nee" + "ds to be hashed " + "before being use" + "d by the HMAC al" + "gorithm.", + .psize = 152, + .digest = "\x02\x6f\xdf\x6b\x50\x74\x1e\x37" + "\x38\x99\xc9\xf7\xd5\x40\x6d\x4e" + "\xb0\x9f\xc6\x66\x56\x36\xfc\x1a" + "\x53\x00\x29\xdd\xf5\xcf\x3c\xa5" + "\xa9\x00\xed\xce\x01\xf5\xf6\x1e" + "\x2f\x40\x8c\xdf\x2f\xd3\xe7\xe8", + }, +}; + +#define HMAC_SHA3_512_TEST_VECTORS 4 + +static struct hash_testvec hmac_sha3_512_tv_template[] = { + { + .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" + "\x0b\x0b\x0b\x0b", + .ksize = 20, + .plaintext = "Hi There", + .psize = 8, + .digest = "\xeb\x3f\xbd\x4b\x2e\xaa\xb8\xf5" + "\xc5\x04\xbd\x3a\x41\x46\x5a\xac" + "\xec\x15\x77\x0a\x7c\xab\xac\x53" + "\x1e\x48\x2f\x86\x0b\x5e\xc7\xba" + "\x47\xcc\xb2\xc6\xf2\xaf\xce\x8f" + "\x88\xd2\x2b\x6d\xc6\x13\x80\xf2" + "\x3a\x66\x8f\xd3\x88\x8b\xb8\x05" + "\x37\xc0\xa0\xb8\x64\x07\x68\x9e", + }, { + .key = "Jefe", + .ksize = 4, + .plaintext = "what do ya want for nothing?", + .psize = 28, + .digest = "\x5a\x4b\xfe\xab\x61\x66\x42\x7c" + "\x7a\x36\x47\xb7\x47\x29\x2b\x83" + "\x84\x53\x7c\xdb\x89\xaf\xb3\xbf" + "\x56\x65\xe4\xc5\xe7\x09\x35\x0b" + "\x28\x7b\xae\xc9\x21\xfd\x7c\xa0" + "\xee\x7a\x0c\x31\xd0\x22\xa9\x5e" + "\x1f\xc9\x2b\xa9\xd7\x7d\xf8\x83" + "\x96\x02\x75\xbe\xb4\xe6\x20\x24", + .np = 4, + .tap = { 7, 7, 7, 7 } + }, { + .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + .ksize = 131, + .plaintext = "Test Using Large" + "r Than Block-Siz" + "e Key - Hash Key" + " First", + .psize = 54, + .digest = "\x00\xf7\x51\xa9\xe5\x06\x95\xb0" + "\x90\xed\x69\x11\xa4\xb6\x55\x24" + "\x95\x1c\xdc\x15\xa7\x3a\x5d\x58" + "\xbb\x55\x21\x5e\xa2\xcd\x83\x9a" + "\xc7\x9d\x2b\x44\xa3\x9b\xaf\xab" + "\x27\xe8\x3f\xde\x9e\x11\xf6\x34" + "\x0b\x11\xd9\x91\xb1\xb9\x1b\xf2" + "\xee\xe7\xfc\x87\x24\x26\xc3\xa4", + }, { + .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" + "\xaa\xaa\xaa", + .ksize = 131, + .plaintext = + "This is a test u" + "sing a larger th" + "an block-size ke" + "y and a larger t" + "han block-size d" + "ata. The key nee" + "ds to be hashed " + "before being use" + "d by the HMAC al" + "gorithm.", + .psize = 152, + .digest = "\x38\xa4\x56\xa0\x04\xbd\x10\xd3" + "\x2c\x9a\xb8\x33\x66\x84\x11\x28" + "\x62\xc3\xdb\x61\xad\xcc\xa3\x18" + "\x29\x35\x5e\xaf\x46\xfd\x5c\x73" + "\xd0\x6a\x1f\x0d\x13\xfe\xc9\xa6" + "\x52\xfb\x38\x11\xb5\x77\xb1\xb1" + "\xd1\xb9\x78\x9f\x97\xae\x5b\x83" + "\xc6\xf4\x4d\xfc\xf1\xd6\x7e\xba", + }, +}; + /* * Poly1305 test vectors from RFC7539 A.3. */ diff --git a/drivers/Makefile b/drivers/Makefile index 6e7d458ac7d4..6b0ad573e0d7 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -12,7 +12,7 @@ obj-$(CONFIG_GENERIC_PHY) += phy/ # GPIO must come after pinctrl as gpios may need to mux pins etc obj-$(CONFIG_PINCTRL) += pinctrl/ -obj-y += gpio/ +obj-$(CONFIG_GPIOLIB) += gpio/ obj-y += pwm/ obj-$(CONFIG_PCI) += pci/ obj-$(CONFIG_PARISC) += parisc/ @@ -128,7 +128,6 @@ obj-$(CONFIG_SGI_SN) += sn/ obj-y += firmware/ obj-$(CONFIG_CRYPTO) += crypto/ obj-$(CONFIG_SUPERH) += sh/ -obj-$(CONFIG_ARCH_SHMOBILE) += sh/ ifndef CONFIG_ARCH_USES_GETTIMEOFFSET obj-y += clocksource/ endif diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index b7e2e776397d..acad70a0bb0d 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -213,6 +213,10 @@ config ACPI_CPU_FREQ_PSS bool select THERMAL +config ACPI_PROCESSOR_CSTATE + def_bool y + depends on IA64 || X86 + config ACPI_PROCESSOR_IDLE bool select CPU_IDLE @@ -234,7 +238,7 @@ config ACPI_CPPC_LIB config ACPI_PROCESSOR tristate "Processor" depends on X86 || IA64 || ARM64 - select ACPI_PROCESSOR_IDLE if X86 || IA64 + select ACPI_PROCESSOR_IDLE select ACPI_CPU_FREQ_PSS if X86 || IA64 default y help @@ -291,8 +295,8 @@ config ACPI_THERMAL config ACPI_NUMA bool "NUMA support" depends on NUMA - depends on (X86 || IA64) - default y if IA64_GENERIC || IA64_SGI_SN2 + depends on (X86 || IA64 || ARM64) + default y if IA64_GENERIC || IA64_SGI_SN2 || ARM64 config ACPI_CUSTOM_DSDT_FILE string "Custom DSDT Table file to include" @@ -311,9 +315,12 @@ config ACPI_CUSTOM_DSDT bool default ACPI_CUSTOM_DSDT_FILE != "" +config ARCH_HAS_ACPI_TABLE_UPGRADE + def_bool n + config ACPI_TABLE_UPGRADE bool "Allow upgrading ACPI tables via initrd" - depends on BLK_DEV_INITRD && X86 + depends on BLK_DEV_INITRD && ARCH_HAS_ACPI_TABLE_UPGRADE default y help This option provides functionality to upgrade arbitrary ACPI tables @@ -475,6 +482,7 @@ config ACPI_NFIT_DEBUG issue. source "drivers/acpi/apei/Kconfig" +source "drivers/acpi/dptf/Kconfig" config ACPI_EXTLOG tristate "Extended Error Log support" @@ -519,6 +527,20 @@ config XPOWER_PMIC_OPREGION help This config adds ACPI operation region support for XPower AXP288 PMIC. +config BXT_WC_PMIC_OPREGION + bool "ACPI operation region support for BXT WhiskeyCove PMIC" + depends on INTEL_SOC_PMIC + help + This config adds ACPI operation region support for BXT WhiskeyCove PMIC. + endif +config ACPI_CONFIGFS + tristate "ACPI configfs support" + select CONFIGFS_FS + help + Select this option to enable support for ACPI configuration from + userspace. The configurable ACPI groups will be visible under + /config/acpi, assuming configfs is mounted under /config. + endif # ACPI diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 251ce85a66fb..88f54f03e3d2 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -44,7 +44,6 @@ acpi-y += acpi_lpss.o acpi_apd.o acpi-y += acpi_platform.o acpi-y += acpi_pnp.o acpi-$(CONFIG_ARM_AMBA) += acpi_amba.o -acpi-y += int340x_thermal.o acpi-y += power.o acpi-y += event.o acpi-$(CONFIG_ACPI_REDUCED_HARDWARE_ONLY) += evged.o @@ -99,5 +98,9 @@ obj-$(CONFIG_ACPI_EXTLOG) += acpi_extlog.o obj-$(CONFIG_PMIC_OPREGION) += pmic/intel_pmic.o obj-$(CONFIG_CRC_PMIC_OPREGION) += pmic/intel_pmic_crc.o obj-$(CONFIG_XPOWER_PMIC_OPREGION) += pmic/intel_pmic_xpower.o +obj-$(CONFIG_BXT_WC_PMIC_OPREGION) += pmic/intel_pmic_bxtwc.o + +obj-$(CONFIG_ACPI_CONFIGFS) += acpi_configfs.o video-objs += acpi_video.o video_detect.o +obj-y += dptf/ diff --git a/drivers/acpi/acpi_configfs.c b/drivers/acpi/acpi_configfs.c new file mode 100644 index 000000000000..146a77fb762d --- /dev/null +++ b/drivers/acpi/acpi_configfs.c @@ -0,0 +1,267 @@ +/* + * ACPI configfs support + * + * Copyright (c) 2016 Intel Corporation + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + */ + +#define pr_fmt(fmt) "ACPI configfs: " fmt + +#include +#include +#include +#include + +static struct config_group *acpi_table_group; + +struct acpi_table { + struct config_item cfg; + struct acpi_table_header *header; +}; + +static ssize_t acpi_table_aml_write(struct config_item *cfg, + const void *data, size_t size) +{ + const struct acpi_table_header *header = data; + struct acpi_table *table; + int ret; + + table = container_of(cfg, struct acpi_table, cfg); + + if (table->header) { + pr_err("table already loaded\n"); + return -EBUSY; + } + + if (header->length != size) { + pr_err("invalid table length\n"); + return -EINVAL; + } + + if (memcmp(header->signature, ACPI_SIG_SSDT, 4)) { + pr_err("invalid table signature\n"); + return -EINVAL; + } + + table = container_of(cfg, struct acpi_table, cfg); + + table->header = kmemdup(header, header->length, GFP_KERNEL); + if (!table->header) + return -ENOMEM; + + ret = acpi_load_table(table->header); + if (ret) { + kfree(table->header); + table->header = NULL; + } + + return ret; +} + +static inline struct acpi_table_header *get_header(struct config_item *cfg) +{ + struct acpi_table *table = container_of(cfg, struct acpi_table, cfg); + + if (!table->header) + pr_err("table not loaded\n"); + + return table->header; +} + +static ssize_t acpi_table_aml_read(struct config_item *cfg, + void *data, size_t size) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + if (data) + memcpy(data, h, h->length); + + return h->length; +} + +#define MAX_ACPI_TABLE_SIZE (128 * 1024) + +CONFIGFS_BIN_ATTR(acpi_table_, aml, NULL, MAX_ACPI_TABLE_SIZE); + +struct configfs_bin_attribute *acpi_table_bin_attrs[] = { + &acpi_table_attr_aml, + NULL, +}; + +ssize_t acpi_table_signature_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%.*s\n", ACPI_NAME_SIZE, h->signature); +} + +ssize_t acpi_table_length_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%d\n", h->length); +} + +ssize_t acpi_table_revision_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%d\n", h->revision); +} + +ssize_t acpi_table_oem_id_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%.*s\n", ACPI_OEM_ID_SIZE, h->oem_id); +} + +ssize_t acpi_table_oem_table_id_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%.*s\n", ACPI_OEM_TABLE_ID_SIZE, h->oem_table_id); +} + +ssize_t acpi_table_oem_revision_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%d\n", h->oem_revision); +} + +ssize_t acpi_table_asl_compiler_id_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%.*s\n", ACPI_NAME_SIZE, h->asl_compiler_id); +} + +ssize_t acpi_table_asl_compiler_revision_show(struct config_item *cfg, + char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%d\n", h->asl_compiler_revision); +} + +CONFIGFS_ATTR_RO(acpi_table_, signature); +CONFIGFS_ATTR_RO(acpi_table_, length); +CONFIGFS_ATTR_RO(acpi_table_, revision); +CONFIGFS_ATTR_RO(acpi_table_, oem_id); +CONFIGFS_ATTR_RO(acpi_table_, oem_table_id); +CONFIGFS_ATTR_RO(acpi_table_, oem_revision); +CONFIGFS_ATTR_RO(acpi_table_, asl_compiler_id); +CONFIGFS_ATTR_RO(acpi_table_, asl_compiler_revision); + +struct configfs_attribute *acpi_table_attrs[] = { + &acpi_table_attr_signature, + &acpi_table_attr_length, + &acpi_table_attr_revision, + &acpi_table_attr_oem_id, + &acpi_table_attr_oem_table_id, + &acpi_table_attr_oem_revision, + &acpi_table_attr_asl_compiler_id, + &acpi_table_attr_asl_compiler_revision, + NULL, +}; + +static struct config_item_type acpi_table_type = { + .ct_owner = THIS_MODULE, + .ct_bin_attrs = acpi_table_bin_attrs, + .ct_attrs = acpi_table_attrs, +}; + +static struct config_item *acpi_table_make_item(struct config_group *group, + const char *name) +{ + struct acpi_table *table; + + table = kzalloc(sizeof(*table), GFP_KERNEL); + if (!table) + return ERR_PTR(-ENOMEM); + + config_item_init_type_name(&table->cfg, name, &acpi_table_type); + return &table->cfg; +} + +struct configfs_group_operations acpi_table_group_ops = { + .make_item = acpi_table_make_item, +}; + +static struct config_item_type acpi_tables_type = { + .ct_owner = THIS_MODULE, + .ct_group_ops = &acpi_table_group_ops, +}; + +static struct config_item_type acpi_root_group_type = { + .ct_owner = THIS_MODULE, +}; + +static struct configfs_subsystem acpi_configfs = { + .su_group = { + .cg_item = { + .ci_namebuf = "acpi", + .ci_type = &acpi_root_group_type, + }, + }, + .su_mutex = __MUTEX_INITIALIZER(acpi_configfs.su_mutex), +}; + +static int __init acpi_configfs_init(void) +{ + int ret; + struct config_group *root = &acpi_configfs.su_group; + + config_group_init(root); + + ret = configfs_register_subsystem(&acpi_configfs); + if (ret) + return ret; + + acpi_table_group = configfs_register_default_group(root, "table", + &acpi_tables_type); + return PTR_ERR_OR_ZERO(acpi_table_group); +} +module_init(acpi_configfs_init); + +static void __exit acpi_configfs_exit(void) +{ + configfs_unregister_default_group(acpi_table_group); + configfs_unregister_subsystem(&acpi_configfs); +} +module_exit(acpi_configfs_exit); + +MODULE_AUTHOR("Octavian Purdila "); +MODULE_DESCRIPTION("ACPI configfs support"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/acpi/acpi_dbg.c b/drivers/acpi/acpi_dbg.c index 1f4128487dd4..dee86925a9a1 100644 --- a/drivers/acpi/acpi_dbg.c +++ b/drivers/acpi/acpi_dbg.c @@ -602,7 +602,7 @@ static int acpi_aml_read_user(char __user *buf, int len) crc->tail = (crc->tail + n) & (ACPI_AML_BUF_SIZE - 1); ret = n; out: - acpi_aml_unlock_fifo(ACPI_AML_OUT_USER, !ret); + acpi_aml_unlock_fifo(ACPI_AML_OUT_USER, ret >= 0); return ret; } @@ -672,7 +672,7 @@ static int acpi_aml_write_user(const char __user *buf, int len) crc->head = (crc->head + n) & (ACPI_AML_BUF_SIZE - 1); ret = n; out: - acpi_aml_unlock_fifo(ACPI_AML_IN_USER, !ret); + acpi_aml_unlock_fifo(ACPI_AML_IN_USER, ret >= 0); return n; } diff --git a/drivers/acpi/acpi_lpat.c b/drivers/acpi/acpi_lpat.c index feb61c1630eb..c1c4877ca96c 100644 --- a/drivers/acpi/acpi_lpat.c +++ b/drivers/acpi/acpi_lpat.c @@ -13,7 +13,7 @@ * GNU General Public License for more details. */ -#include +#include #include #include @@ -157,5 +157,3 @@ void acpi_lpat_free_conversion_table(struct acpi_lpat_conversion_table } } EXPORT_SYMBOL_GPL(acpi_lpat_free_conversion_table); - -MODULE_LICENSE("GPL"); diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c index 0872d5fecb82..357a0b8f860b 100644 --- a/drivers/acpi/acpi_lpss.c +++ b/drivers/acpi/acpi_lpss.c @@ -29,6 +29,7 @@ ACPI_MODULE_NAME("acpi_lpss"); #ifdef CONFIG_X86_INTEL_LPSS #include +#include #include #include @@ -229,8 +230,8 @@ static const struct lpss_device_desc bsw_spi_dev_desc = { #define ICPU(model) { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, } static const struct x86_cpu_id lpss_cpu_ids[] = { - ICPU(0x37), /* Valleyview, Bay Trail */ - ICPU(0x4c), /* Braswell, Cherry Trail */ + ICPU(INTEL_FAM6_ATOM_SILVERMONT1), /* Valleyview, Bay Trail */ + ICPU(INTEL_FAM6_ATOM_AIRMONT), /* Braswell, Cherry Trail */ {} }; diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index c1d138e128cb..c5557d070954 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -1246,6 +1246,9 @@ static int acpi_video_device_enumerate(struct acpi_video_bus *video) union acpi_object *dod = NULL; union acpi_object *obj; + if (!video->cap._DOD) + return AE_NOT_EXIST; + status = acpi_evaluate_object(video->device->handle, "_DOD", NULL, &buffer); if (!ACPI_SUCCESS(status)) { ACPI_EXCEPTION((AE_INFO, status, "Evaluating _DOD")); diff --git a/drivers/acpi/apei/Makefile b/drivers/acpi/apei/Makefile index 5d575a955940..e50573de25f1 100644 --- a/drivers/acpi/apei/Makefile +++ b/drivers/acpi/apei/Makefile @@ -3,4 +3,4 @@ obj-$(CONFIG_ACPI_APEI_GHES) += ghes.o obj-$(CONFIG_ACPI_APEI_EINJ) += einj.o obj-$(CONFIG_ACPI_APEI_ERST_DEBUG) += erst-dbg.o -apei-y := apei-base.o hest.o erst.o +apei-y := apei-base.o hest.o erst.o bert.o diff --git a/drivers/acpi/apei/apei-internal.h b/drivers/acpi/apei/apei-internal.h index 16129c78b489..6e9f14c0a71b 100644 --- a/drivers/acpi/apei/apei-internal.h +++ b/drivers/acpi/apei/apei-internal.h @@ -1,6 +1,6 @@ /* * apei-internal.h - ACPI Platform Error Interface internal - * definations. + * definitions. */ #ifndef APEI_INTERNAL_H diff --git a/drivers/acpi/apei/bert.c b/drivers/acpi/apei/bert.c new file mode 100644 index 000000000000..a05b5c0cf181 --- /dev/null +++ b/drivers/acpi/apei/bert.c @@ -0,0 +1,150 @@ +/* + * APEI Boot Error Record Table (BERT) support + * + * Copyright 2011 Intel Corp. + * Author: Huang Ying + * + * Under normal circumstances, when a hardware error occurs, the error + * handler receives control and processes the error. This gives OSPM a + * chance to process the error condition, report it, and optionally attempt + * recovery. In some cases, the system is unable to process an error. + * For example, system firmware or a management controller may choose to + * reset the system or the system might experience an uncontrolled crash + * or reset.The boot error source is used to report unhandled errors that + * occurred in a previous boot. This mechanism is described in the BERT + * table. + * + * For more information about BERT, please refer to ACPI Specification + * version 4.0, section 17.3.1 + * + * This file is licensed under GPLv2. + * + */ + +#include +#include +#include +#include +#include + +#include "apei-internal.h" + +#undef pr_fmt +#define pr_fmt(fmt) "BERT: " fmt + +static int bert_disable; + +static void __init bert_print_all(struct acpi_bert_region *region, + unsigned int region_len) +{ + struct acpi_hest_generic_status *estatus = + (struct acpi_hest_generic_status *)region; + int remain = region_len; + u32 estatus_len; + + if (!estatus->block_status) + return; + + while (remain > sizeof(struct acpi_bert_region)) { + if (cper_estatus_check(estatus)) { + pr_err(FW_BUG "Invalid error record.\n"); + return; + } + + estatus_len = cper_estatus_len(estatus); + if (remain < estatus_len) { + pr_err(FW_BUG "Truncated status block (length: %u).\n", + estatus_len); + return; + } + + pr_info_once("Error records from previous boot:\n"); + + cper_estatus_print(KERN_INFO HW_ERR, estatus); + + /* + * Because the boot error source is "one-time polled" type, + * clear Block Status of current Generic Error Status Block, + * once it's printed. + */ + estatus->block_status = 0; + + estatus = (void *)estatus + estatus_len; + /* No more error records. */ + if (!estatus->block_status) + return; + + remain -= estatus_len; + } +} + +static int __init setup_bert_disable(char *str) +{ + bert_disable = 1; + + return 0; +} +__setup("bert_disable", setup_bert_disable); + +static int __init bert_check_table(struct acpi_table_bert *bert_tab) +{ + if (bert_tab->header.length < sizeof(struct acpi_table_bert) || + bert_tab->region_length < sizeof(struct acpi_bert_region)) + return -EINVAL; + + return 0; +} + +static int __init bert_init(void) +{ + struct acpi_bert_region *boot_error_region; + struct acpi_table_bert *bert_tab; + unsigned int region_len; + acpi_status status; + int rc = 0; + + if (acpi_disabled) + return 0; + + if (bert_disable) { + pr_info("Boot Error Record Table support is disabled.\n"); + return 0; + } + + status = acpi_get_table(ACPI_SIG_BERT, 0, (struct acpi_table_header **)&bert_tab); + if (status == AE_NOT_FOUND) + return 0; + + if (ACPI_FAILURE(status)) { + pr_err("get table failed, %s.\n", acpi_format_exception(status)); + return -EINVAL; + } + + rc = bert_check_table(bert_tab); + if (rc) { + pr_err(FW_BUG "table invalid.\n"); + return rc; + } + + region_len = bert_tab->region_length; + if (!request_mem_region(bert_tab->address, region_len, "APEI BERT")) { + pr_err("Can't request iomem region <%016llx-%016llx>.\n", + (unsigned long long)bert_tab->address, + (unsigned long long)bert_tab->address + region_len - 1); + return -EIO; + } + + boot_error_region = ioremap_cache(bert_tab->address, region_len); + if (boot_error_region) { + bert_print_all(boot_error_region, region_len); + iounmap(boot_error_region); + } else { + rc = -ENOMEM; + } + + release_mem_region(bert_tab->address, region_len); + + return rc; +} + +late_initcall(bert_init); diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj.c index 559c1173de1c..eebb7e39c49c 100644 --- a/drivers/acpi/apei/einj.c +++ b/drivers/acpi/apei/einj.c @@ -33,7 +33,8 @@ #include "apei-internal.h" -#define EINJ_PFX "EINJ: " +#undef pr_fmt +#define pr_fmt(fmt) "EINJ: " fmt #define SPIN_UNIT 100 /* 100ns */ /* Firmware should respond within 1 milliseconds */ @@ -179,8 +180,7 @@ static int einj_get_available_error_type(u32 *type) static int einj_timedout(u64 *t) { if ((s64)*t < SPIN_UNIT) { - pr_warning(FW_WARN EINJ_PFX - "Firmware does not respond in time\n"); + pr_warning(FW_WARN "Firmware does not respond in time\n"); return 1; } *t -= SPIN_UNIT; @@ -307,8 +307,7 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, r = request_mem_region(trigger_paddr, sizeof(*trigger_tab), "APEI EINJ Trigger Table"); if (!r) { - pr_err(EINJ_PFX - "Can not request [mem %#010llx-%#010llx] for Trigger table\n", + pr_err("Can not request [mem %#010llx-%#010llx] for Trigger table\n", (unsigned long long)trigger_paddr, (unsigned long long)trigger_paddr + sizeof(*trigger_tab) - 1); @@ -316,13 +315,12 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, } trigger_tab = ioremap_cache(trigger_paddr, sizeof(*trigger_tab)); if (!trigger_tab) { - pr_err(EINJ_PFX "Failed to map trigger table!\n"); + pr_err("Failed to map trigger table!\n"); goto out_rel_header; } rc = einj_check_trigger_header(trigger_tab); if (rc) { - pr_warning(FW_BUG EINJ_PFX - "The trigger error action table is invalid\n"); + pr_warning(FW_BUG "Invalid trigger error action table.\n"); goto out_rel_header; } @@ -336,8 +334,7 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, table_size - sizeof(*trigger_tab), "APEI EINJ Trigger Table"); if (!r) { - pr_err(EINJ_PFX -"Can not request [mem %#010llx-%#010llx] for Trigger Table Entry\n", + pr_err("Can not request [mem %#010llx-%#010llx] for Trigger Table Entry\n", (unsigned long long)trigger_paddr + sizeof(*trigger_tab), (unsigned long long)trigger_paddr + table_size - 1); goto out_rel_header; @@ -345,7 +342,7 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, iounmap(trigger_tab); trigger_tab = ioremap_cache(trigger_paddr, table_size); if (!trigger_tab) { - pr_err(EINJ_PFX "Failed to map trigger table!\n"); + pr_err("Failed to map trigger table!\n"); goto out_rel_entry; } trigger_entry = (struct acpi_whea_header *) @@ -695,34 +692,42 @@ static int __init einj_init(void) struct dentry *fentry; struct apei_exec_context ctx; - if (acpi_disabled) + if (acpi_disabled) { + pr_warn("ACPI disabled.\n"); return -ENODEV; + } status = acpi_get_table(ACPI_SIG_EINJ, 0, (struct acpi_table_header **)&einj_tab); - if (status == AE_NOT_FOUND) + if (status == AE_NOT_FOUND) { + pr_warn("EINJ table not found.\n"); return -ENODEV; + } else if (ACPI_FAILURE(status)) { - const char *msg = acpi_format_exception(status); - pr_err(EINJ_PFX "Failed to get table, %s\n", msg); + pr_err("Failed to get EINJ table: %s\n", + acpi_format_exception(status)); return -EINVAL; } rc = einj_check_table(einj_tab); if (rc) { - pr_warning(FW_BUG EINJ_PFX "EINJ table is invalid\n"); + pr_warn(FW_BUG "Invalid EINJ table.n"); return -EINVAL; } rc = -ENOMEM; einj_debug_dir = debugfs_create_dir("einj", apei_get_debugfs_dir()); - if (!einj_debug_dir) + if (!einj_debug_dir) { + pr_err("Error creating debugfs node.\n"); goto err_cleanup; + } + fentry = debugfs_create_file("available_error_type", S_IRUSR, einj_debug_dir, NULL, &available_error_type_fops); if (!fentry) goto err_cleanup; + fentry = debugfs_create_file("error_type", S_IRUSR | S_IWUSR, einj_debug_dir, NULL, &error_type_fops); if (!fentry) @@ -735,14 +740,22 @@ static int __init einj_init(void) apei_resources_init(&einj_resources); einj_exec_ctx_init(&ctx); rc = apei_exec_collect_resources(&ctx, &einj_resources); - if (rc) + if (rc) { + pr_err("Error collecting EINJ resources.\n"); goto err_fini; + } + rc = apei_resources_request(&einj_resources, "APEI EINJ"); - if (rc) + if (rc) { + pr_err("Error requesting memory/port resources.\n"); goto err_fini; + } + rc = apei_exec_pre_map_gars(&ctx); - if (rc) + if (rc) { + pr_err("Error pre-mapping GARs.\n"); goto err_release; + } rc = -ENOMEM; einj_param = einj_get_parameter_address(); @@ -787,7 +800,7 @@ static int __init einj_init(void) goto err_unmap; } - pr_info(EINJ_PFX "Error INJection is initialized.\n"); + pr_info("Error INJection is initialized.\n"); return 0; @@ -798,6 +811,7 @@ err_unmap: sizeof(struct einj_parameter); acpi_os_unmap_iomem(einj_param, size); + pr_err("Error creating param extension debugfs nodes.\n"); } apei_exec_post_unmap_gars(&ctx); err_release: @@ -805,6 +819,7 @@ err_release: err_fini: apei_resources_fini(&einj_resources); err_cleanup: + pr_err("Error creating primary debugfs nodes.\n"); debugfs_remove_recursive(einj_debug_dir); return rc; diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c index 006c3894c6ea..f096ab3cb54d 100644 --- a/drivers/acpi/apei/erst.c +++ b/drivers/acpi/apei/erst.c @@ -927,7 +927,8 @@ static int erst_open_pstore(struct pstore_info *psi); static int erst_close_pstore(struct pstore_info *psi); static ssize_t erst_reader(u64 *id, enum pstore_type_id *type, int *count, struct timespec *time, char **buf, - bool *compressed, struct pstore_info *psi); + bool *compressed, ssize_t *ecc_notice_size, + struct pstore_info *psi); static int erst_writer(enum pstore_type_id type, enum kmsg_dump_reason reason, u64 *id, unsigned int part, int count, bool compressed, size_t size, struct pstore_info *psi); @@ -987,7 +988,8 @@ static int erst_close_pstore(struct pstore_info *psi) static ssize_t erst_reader(u64 *id, enum pstore_type_id *type, int *count, struct timespec *time, char **buf, - bool *compressed, struct pstore_info *psi) + bool *compressed, ssize_t *ecc_notice_size, + struct pstore_info *psi) { int rc; ssize_t len = 0; @@ -1033,6 +1035,7 @@ skip: memcpy(*buf, rcd->data, len - sizeof(*rcd)); *id = record_id; *compressed = false; + *ecc_notice_size = 0; if (uuid_le_cmp(rcd->sec_hdr.section_type, CPER_SECTION_TYPE_DMESG_Z) == 0) { *type = PSTORE_TYPE_DMESG; diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 262ca31b86d9..85b7d07fe5c8 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -30,6 +30,9 @@ #include #include #include +#include +#include +#include #ifdef CONFIG_X86 #include #endif @@ -174,22 +177,17 @@ void acpi_bus_detach_private_data(acpi_handle handle) EXPORT_SYMBOL_GPL(acpi_bus_detach_private_data); static void acpi_print_osc_error(acpi_handle handle, - struct acpi_osc_context *context, char *error) + struct acpi_osc_context *context, char *error) { - struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER}; int i; - if (ACPI_FAILURE(acpi_get_name(handle, ACPI_FULL_PATHNAME, &buffer))) - printk(KERN_DEBUG "%s: %s\n", context->uuid_str, error); - else { - printk(KERN_DEBUG "%s (%s): %s\n", - (char *)buffer.pointer, context->uuid_str, error); - kfree(buffer.pointer); - } - printk(KERN_DEBUG "_OSC request data:"); + acpi_handle_debug(handle, "(%s): %s\n", context->uuid_str, error); + + pr_debug("_OSC request data:"); for (i = 0; i < context->cap.length; i += sizeof(u32)) - printk(" %x", *((u32 *)(context->cap.pointer + i))); - printk("\n"); + pr_debug(" %x", *((u32 *)(context->cap.pointer + i))); + + pr_debug("\n"); } acpi_status acpi_str_to_uuid(char *str, u8 *uuid) @@ -302,6 +300,14 @@ out_kfree: EXPORT_SYMBOL(acpi_run_osc); bool osc_sb_apei_support_acked; + +/* + * ACPI 6.0 Section 8.4.4.2 Idle State Coordination + * OSPM supports platform coordinated low power idle(LPI) states + */ +bool osc_pc_lpi_support_confirmed; +EXPORT_SYMBOL_GPL(osc_pc_lpi_support_confirmed); + static u8 sb_uuid_str[] = "0811B06E-4A27-44F9-8D60-3CBBC22E7B48"; static void acpi_bus_osc_support(void) { @@ -322,6 +328,7 @@ static void acpi_bus_osc_support(void) capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PPC_OST_SUPPORT; capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_HOTPLUG_OST_SUPPORT; + capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PCLPI_SUPPORT; if (!ghes_disable) capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_APEI_SUPPORT; @@ -329,9 +336,12 @@ static void acpi_bus_osc_support(void) return; if (ACPI_SUCCESS(acpi_run_osc(handle, &context))) { u32 *capbuf_ret = context.ret.pointer; - if (context.ret.length > OSC_SUPPORT_DWORD) + if (context.ret.length > OSC_SUPPORT_DWORD) { osc_sb_apei_support_acked = capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_APEI_SUPPORT; + osc_pc_lpi_support_confirmed = + capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_PCLPI_SUPPORT; + } kfree(context.ret.pointer); } /* do we need to check other returned cap? Sounds no */ @@ -475,6 +485,56 @@ static void acpi_device_remove_notify_handler(struct acpi_device *device) acpi_device_notify); } +/* Handle events targeting \_SB device (at present only graceful shutdown) */ + +#define ACPI_SB_NOTIFY_SHUTDOWN_REQUEST 0x81 +#define ACPI_SB_INDICATE_INTERVAL 10000 + +static void sb_notify_work(struct work_struct *dummy) +{ + acpi_handle sb_handle; + + orderly_poweroff(true); + + /* + * After initiating graceful shutdown, the ACPI spec requires OSPM + * to evaluate _OST method once every 10seconds to indicate that + * the shutdown is in progress + */ + acpi_get_handle(NULL, "\\_SB", &sb_handle); + while (1) { + pr_info("Graceful shutdown in progress.\n"); + acpi_evaluate_ost(sb_handle, ACPI_OST_EC_OSPM_SHUTDOWN, + ACPI_OST_SC_OS_SHUTDOWN_IN_PROGRESS, NULL); + msleep(ACPI_SB_INDICATE_INTERVAL); + } +} + +static void acpi_sb_notify(acpi_handle handle, u32 event, void *data) +{ + static DECLARE_WORK(acpi_sb_work, sb_notify_work); + + if (event == ACPI_SB_NOTIFY_SHUTDOWN_REQUEST) { + if (!work_busy(&acpi_sb_work)) + schedule_work(&acpi_sb_work); + } else + pr_warn("event %x is not supported by \\_SB device\n", event); +} + +static int __init acpi_setup_sb_notify_handler(void) +{ + acpi_handle sb_handle; + + if (ACPI_FAILURE(acpi_get_handle(NULL, "\\_SB", &sb_handle))) + return -ENXIO; + + if (ACPI_FAILURE(acpi_install_notify_handler(sb_handle, ACPI_DEVICE_NOTIFY, + acpi_sb_notify, NULL))) + return -EINVAL; + + return 0; +} + /* -------------------------------------------------------------------------- Device Matching -------------------------------------------------------------------------- */ @@ -961,8 +1021,7 @@ void __init acpi_early_init(void) /** * acpi_subsystem_init - Finalize the early initialization of ACPI. * - * Switch over the platform to the ACPI mode (if possible), initialize the - * handling of ACPI events, install the interrupt and global lock handlers. + * Switch over the platform to the ACPI mode (if possible). * * Doing this too early is generally unsafe, but at the same time it needs to be * done before all things that really depend on ACPI. The right spot appears to @@ -990,6 +1049,13 @@ void __init acpi_subsystem_init(void) } } +static acpi_status acpi_bus_table_handler(u32 event, void *table, void *context) +{ + acpi_scan_table_handler(event, table, context); + + return acpi_sysfs_table_handler(event, table, context); +} + static int __init acpi_bus_init(void) { int result; @@ -1043,6 +1109,8 @@ static int __init acpi_bus_init(void) * _PDC control method may load dynamic SSDT tables, * and we need to install the table handler before that. */ + status = acpi_install_table_handler(acpi_bus_table_handler, NULL); + acpi_sysfs_init(); acpi_early_processor_set_pdc(); @@ -1124,6 +1192,7 @@ static int __init acpi_init(void) acpi_sleep_proc_init(); acpi_wakeup_device_init(); acpi_debugger_init(); + acpi_setup_sb_notify_handler(); return 0; } diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c index 5c3b0918d5fd..148f4e5ca104 100644 --- a/drivers/acpi/button.c +++ b/drivers/acpi/button.c @@ -53,6 +53,10 @@ #define ACPI_BUTTON_DEVICE_NAME_LID "Lid Switch" #define ACPI_BUTTON_TYPE_LID 0x05 +#define ACPI_BUTTON_LID_INIT_IGNORE 0x00 +#define ACPI_BUTTON_LID_INIT_OPEN 0x01 +#define ACPI_BUTTON_LID_INIT_METHOD 0x02 + #define _COMPONENT ACPI_BUTTON_COMPONENT ACPI_MODULE_NAME("button"); @@ -105,6 +109,7 @@ struct acpi_button { static BLOCKING_NOTIFIER_HEAD(acpi_lid_notifier); static struct acpi_device *lid_device; +static u8 lid_init_state = ACPI_BUTTON_LID_INIT_METHOD; /* -------------------------------------------------------------------------- FS Interface (/proc) @@ -113,16 +118,52 @@ static struct acpi_device *lid_device; static struct proc_dir_entry *acpi_button_dir; static struct proc_dir_entry *acpi_lid_dir; +static int acpi_lid_evaluate_state(struct acpi_device *device) +{ + unsigned long long lid_state; + acpi_status status; + + status = acpi_evaluate_integer(device->handle, "_LID", NULL, &lid_state); + if (ACPI_FAILURE(status)) + return -ENODEV; + + return lid_state ? 1 : 0; +} + +static int acpi_lid_notify_state(struct acpi_device *device, int state) +{ + struct acpi_button *button = acpi_driver_data(device); + int ret; + + /* input layer checks if event is redundant */ + input_report_switch(button->input, SW_LID, !state); + input_sync(button->input); + + if (state) + pm_wakeup_event(&device->dev, 0); + + ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, device); + if (ret == NOTIFY_DONE) + ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, + device); + if (ret == NOTIFY_DONE || ret == NOTIFY_OK) { + /* + * It is also regarded as success if the notifier_chain + * returns NOTIFY_OK or NOTIFY_DONE. + */ + ret = 0; + } + return ret; +} + static int acpi_button_state_seq_show(struct seq_file *seq, void *offset) { struct acpi_device *device = seq->private; - acpi_status status; - unsigned long long state; + int state; - status = acpi_evaluate_integer(device->handle, "_LID", NULL, &state); + state = acpi_lid_evaluate_state(device); seq_printf(seq, "state: %s\n", - ACPI_FAILURE(status) ? "unsupported" : - (state ? "open" : "closed")); + state < 0 ? "unsupported" : (state ? "open" : "closed")); return 0; } @@ -231,51 +272,37 @@ EXPORT_SYMBOL(acpi_lid_notifier_unregister); int acpi_lid_open(void) { - acpi_status status; - unsigned long long state; - if (!lid_device) return -ENODEV; - status = acpi_evaluate_integer(lid_device->handle, "_LID", NULL, - &state); - if (ACPI_FAILURE(status)) - return -ENODEV; - - return !!state; + return acpi_lid_evaluate_state(lid_device); } EXPORT_SYMBOL(acpi_lid_open); -static int acpi_lid_send_state(struct acpi_device *device) +static int acpi_lid_update_state(struct acpi_device *device) { - struct acpi_button *button = acpi_driver_data(device); - unsigned long long state; - acpi_status status; - int ret; - - status = acpi_evaluate_integer(device->handle, "_LID", NULL, &state); - if (ACPI_FAILURE(status)) - return -ENODEV; + int state; - /* input layer checks if event is redundant */ - input_report_switch(button->input, SW_LID, !state); - input_sync(button->input); + state = acpi_lid_evaluate_state(device); + if (state < 0) + return state; - if (state) - pm_wakeup_event(&device->dev, 0); + return acpi_lid_notify_state(device, state); +} - ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, device); - if (ret == NOTIFY_DONE) - ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, - device); - if (ret == NOTIFY_DONE || ret == NOTIFY_OK) { - /* - * It is also regarded as success if the notifier_chain - * returns NOTIFY_OK or NOTIFY_DONE. - */ - ret = 0; +static void acpi_lid_initialize_state(struct acpi_device *device) +{ + switch (lid_init_state) { + case ACPI_BUTTON_LID_INIT_OPEN: + (void)acpi_lid_notify_state(device, 1); + break; + case ACPI_BUTTON_LID_INIT_METHOD: + (void)acpi_lid_update_state(device); + break; + case ACPI_BUTTON_LID_INIT_IGNORE: + default: + break; } - return ret; } static void acpi_button_notify(struct acpi_device *device, u32 event) @@ -290,7 +317,7 @@ static void acpi_button_notify(struct acpi_device *device, u32 event) case ACPI_BUTTON_NOTIFY_STATUS: input = button->input; if (button->type == ACPI_BUTTON_TYPE_LID) { - acpi_lid_send_state(device); + acpi_lid_update_state(device); } else { int keycode; @@ -335,7 +362,7 @@ static int acpi_button_resume(struct device *dev) button->suspended = false; if (button->type == ACPI_BUTTON_TYPE_LID) - return acpi_lid_send_state(device); + acpi_lid_initialize_state(device); return 0; } #endif @@ -416,7 +443,7 @@ static int acpi_button_add(struct acpi_device *device) if (error) goto err_remove_fs; if (button->type == ACPI_BUTTON_TYPE_LID) { - acpi_lid_send_state(device); + acpi_lid_initialize_state(device); /* * This assumes there's only one lid device, or if there are * more we only care about the last one... @@ -446,4 +473,42 @@ static int acpi_button_remove(struct acpi_device *device) return 0; } +static int param_set_lid_init_state(const char *val, struct kernel_param *kp) +{ + int result = 0; + + if (!strncmp(val, "open", sizeof("open") - 1)) { + lid_init_state = ACPI_BUTTON_LID_INIT_OPEN; + pr_info("Notify initial lid state as open\n"); + } else if (!strncmp(val, "method", sizeof("method") - 1)) { + lid_init_state = ACPI_BUTTON_LID_INIT_METHOD; + pr_info("Notify initial lid state with _LID return value\n"); + } else if (!strncmp(val, "ignore", sizeof("ignore") - 1)) { + lid_init_state = ACPI_BUTTON_LID_INIT_IGNORE; + pr_info("Do not notify initial lid state\n"); + } else + result = -EINVAL; + return result; +} + +static int param_get_lid_init_state(char *buffer, struct kernel_param *kp) +{ + switch (lid_init_state) { + case ACPI_BUTTON_LID_INIT_OPEN: + return sprintf(buffer, "open"); + case ACPI_BUTTON_LID_INIT_METHOD: + return sprintf(buffer, "method"); + case ACPI_BUTTON_LID_INIT_IGNORE: + return sprintf(buffer, "ignore"); + default: + return sprintf(buffer, "invalid"); + } + return 0; +} + +module_param_call(lid_init_state, + param_set_lid_init_state, param_get_lid_init_state, + NULL, 0644); +MODULE_PARM_DESC(lid_init_state, "Behavior for reporting LID initial state"); + module_acpi_driver(acpi_button_driver); diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 8adac69dba3d..2e981732805b 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -299,8 +299,10 @@ int acpi_get_psd_map(struct cpudata **all_cpu_data) continue; cpc_ptr = per_cpu(cpc_desc_ptr, i); - if (!cpc_ptr) - continue; + if (!cpc_ptr) { + retval = -EFAULT; + goto err_ret; + } pdomain = &(cpc_ptr->domain_info); cpumask_set_cpu(i, pr->shared_cpu_map); @@ -322,8 +324,10 @@ int acpi_get_psd_map(struct cpudata **all_cpu_data) continue; match_cpc_ptr = per_cpu(cpc_desc_ptr, j); - if (!match_cpc_ptr) - continue; + if (!match_cpc_ptr) { + retval = -EFAULT; + goto err_ret; + } match_pdomain = &(match_cpc_ptr->domain_info); if (match_pdomain->domain != pdomain->domain) @@ -353,8 +357,10 @@ int acpi_get_psd_map(struct cpudata **all_cpu_data) continue; match_cpc_ptr = per_cpu(cpc_desc_ptr, j); - if (!match_cpc_ptr) - continue; + if (!match_cpc_ptr) { + retval = -EFAULT; + goto err_ret; + } match_pdomain = &(match_cpc_ptr->domain_info); if (match_pdomain->domain != pdomain->domain) @@ -595,9 +601,6 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) /* Store CPU Logical ID */ cpc_ptr->cpu_id = pr->id; - /* Plug it into this CPUs CPC descriptor. */ - per_cpu(cpc_desc_ptr, pr->id) = cpc_ptr; - /* Parse PSD data for this CPU */ ret = acpi_get_psd(cpc_ptr, handle); if (ret) @@ -610,6 +613,9 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) goto out_free; } + /* Plug PSD data into this CPUs CPC descriptor. */ + per_cpu(cpc_desc_ptr, pr->id) = cpc_ptr; + /* Everything looks okay */ pr_debug("Parsed CPC struct for CPU: %d\n", pr->id); diff --git a/drivers/acpi/dock.c b/drivers/acpi/dock.c index e8e128dede29..0c00208b423e 100644 --- a/drivers/acpi/dock.c +++ b/drivers/acpi/dock.c @@ -21,7 +21,7 @@ */ #include -#include +#include #include #include #include @@ -33,12 +33,7 @@ #include "internal.h" -#define ACPI_DOCK_DRIVER_DESCRIPTION "ACPI Dock Station Driver" - ACPI_MODULE_NAME("dock"); -MODULE_AUTHOR("Kristen Carlson Accardi"); -MODULE_DESCRIPTION(ACPI_DOCK_DRIVER_DESCRIPTION); -MODULE_LICENSE("GPL"); static bool immediate_undock = 1; module_param(immediate_undock, bool, 0644); diff --git a/drivers/acpi/dptf/Kconfig b/drivers/acpi/dptf/Kconfig new file mode 100644 index 000000000000..ac0a6ed0cf46 --- /dev/null +++ b/drivers/acpi/dptf/Kconfig @@ -0,0 +1,15 @@ +config DPTF_POWER + tristate "DPTF Platform Power Participant" + depends on X86 + help + This driver adds support for Dynamic Platform and Thermal Framework + (DPTF) Platform Power Participant device (INT3407) support. + This participant is responsible for exposing platform telemetry: + max_platform_power + platform_power_source + adapter_rating + battery_steady_power + charger_type + + To compile this driver as a module, choose M here: + the module will be called dptf_power. diff --git a/drivers/acpi/dptf/Makefile b/drivers/acpi/dptf/Makefile new file mode 100644 index 000000000000..06ea8809583d --- /dev/null +++ b/drivers/acpi/dptf/Makefile @@ -0,0 +1,4 @@ +obj-$(CONFIG_ACPI) += int340x_thermal.o +obj-$(CONFIG_DPTF_POWER) += dptf_power.o + +ccflags-y += -Idrivers/acpi diff --git a/drivers/acpi/dptf/dptf_power.c b/drivers/acpi/dptf/dptf_power.c new file mode 100644 index 000000000000..734642dc5008 --- /dev/null +++ b/drivers/acpi/dptf/dptf_power.c @@ -0,0 +1,128 @@ +/* + * dptf_power: DPTF platform power driver + * Copyright (c) 2016, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ + +#include +#include +#include +#include + +/* + * Presentation of attributes which are defined for INT3407. They are: + * PMAX : Maximum platform powe + * PSRC : Platform power source + * ARTG : Adapter rating + * CTYP : Charger type + * PBSS : Battery steady power + */ +#define DPTF_POWER_SHOW(name, object) \ +static ssize_t name##_show(struct device *dev,\ + struct device_attribute *attr,\ + char *buf)\ +{\ + struct platform_device *pdev = to_platform_device(dev);\ + struct acpi_device *acpi_dev = platform_get_drvdata(pdev);\ + unsigned long long val;\ + acpi_status status;\ +\ + status = acpi_evaluate_integer(acpi_dev->handle, #object,\ + NULL, &val);\ + if (ACPI_SUCCESS(status))\ + return sprintf(buf, "%d\n", (int)val);\ + else \ + return -EINVAL;\ +} + +DPTF_POWER_SHOW(max_platform_power_mw, PMAX) +DPTF_POWER_SHOW(platform_power_source, PSRC) +DPTF_POWER_SHOW(adapter_rating_mw, ARTG) +DPTF_POWER_SHOW(battery_steady_power_mw, PBSS) +DPTF_POWER_SHOW(charger_type, CTYP) + +static DEVICE_ATTR_RO(max_platform_power_mw); +static DEVICE_ATTR_RO(platform_power_source); +static DEVICE_ATTR_RO(adapter_rating_mw); +static DEVICE_ATTR_RO(battery_steady_power_mw); +static DEVICE_ATTR_RO(charger_type); + +static struct attribute *dptf_power_attrs[] = { + &dev_attr_max_platform_power_mw.attr, + &dev_attr_platform_power_source.attr, + &dev_attr_adapter_rating_mw.attr, + &dev_attr_battery_steady_power_mw.attr, + &dev_attr_charger_type.attr, + NULL +}; + +static struct attribute_group dptf_power_attribute_group = { + .attrs = dptf_power_attrs, + .name = "dptf_power" +}; + +static int dptf_power_add(struct platform_device *pdev) +{ + struct acpi_device *acpi_dev; + acpi_status status; + unsigned long long ptype; + int result; + + acpi_dev = ACPI_COMPANION(&(pdev->dev)); + if (!acpi_dev) + return -ENODEV; + + status = acpi_evaluate_integer(acpi_dev->handle, "PTYP", NULL, &ptype); + if (ACPI_FAILURE(status)) + return -ENODEV; + + if (ptype != 0x11) + return -ENODEV; + + result = sysfs_create_group(&pdev->dev.kobj, + &dptf_power_attribute_group); + if (result) + return result; + + platform_set_drvdata(pdev, acpi_dev); + + return 0; +} + +static int dptf_power_remove(struct platform_device *pdev) +{ + + sysfs_remove_group(&pdev->dev.kobj, &dptf_power_attribute_group); + + return 0; +} + +static const struct acpi_device_id int3407_device_ids[] = { + {"INT3407", 0}, + {"", 0}, +}; +MODULE_DEVICE_TABLE(acpi, int3407_device_ids); + +static struct platform_driver dptf_power_driver = { + .probe = dptf_power_add, + .remove = dptf_power_remove, + .driver = { + .name = "DPTF Platform Power", + .acpi_match_table = int3407_device_ids, + }, +}; + +module_platform_driver(dptf_power_driver); + +MODULE_AUTHOR("Srinivas Pandruvada "); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ACPI DPTF platform power driver"); diff --git a/drivers/acpi/int340x_thermal.c b/drivers/acpi/dptf/int340x_thermal.c similarity index 100% rename from drivers/acpi/int340x_thermal.c rename to drivers/acpi/dptf/int340x_thermal.c diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 73c76d646064..999a10914678 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -1331,8 +1331,6 @@ static int ec_install_handlers(struct acpi_ec *ec) static void ec_remove_handlers(struct acpi_ec *ec) { - acpi_ec_stop(ec, false); - if (test_bit(EC_FLAGS_EC_HANDLER_INSTALLED, &ec->flags)) { if (ACPI_FAILURE(acpi_remove_address_space_handler(ec->handle, ACPI_ADR_SPACE_EC, &acpi_ec_space_handler))) @@ -1340,6 +1338,19 @@ static void ec_remove_handlers(struct acpi_ec *ec) clear_bit(EC_FLAGS_EC_HANDLER_INSTALLED, &ec->flags); } + /* + * Stops handling the EC transactions after removing the operation + * region handler. This is required because _REG(DISCONNECT) + * invoked during the removal can result in new EC transactions. + * + * Flushes the EC requests and thus disables the GPE before + * removing the GPE handler. This is required by the current ACPICA + * GPE core. ACPICA GPE core will automatically disable a GPE when + * it is indicated but there is no way to handle it. So the drivers + * must disable the GPEs prior to removing the GPE handlers. + */ + acpi_ec_stop(ec, false); + if (test_bit(EC_FLAGS_GPE_HANDLER_INSTALLED, &ec->flags)) { if (ACPI_FAILURE(acpi_remove_gpe_handler(NULL, ec->gpe, &acpi_ec_gpe_handler))) @@ -1348,13 +1359,9 @@ static void ec_remove_handlers(struct acpi_ec *ec) } } -static int acpi_ec_add(struct acpi_device *device) +static struct acpi_ec *acpi_ec_alloc(void) { - struct acpi_ec *ec = NULL; - int ret; - - strcpy(acpi_device_name(device), ACPI_EC_DEVICE_NAME); - strcpy(acpi_device_class(device), ACPI_EC_CLASS); + struct acpi_ec *ec; /* Check for boot EC */ if (boot_ec) { @@ -1365,9 +1372,21 @@ static int acpi_ec_add(struct acpi_device *device) first_ec = NULL; } else { ec = make_acpi_ec(); - if (!ec) - return -ENOMEM; } + return ec; +} + +static int acpi_ec_add(struct acpi_device *device) +{ + struct acpi_ec *ec = NULL; + int ret; + + strcpy(acpi_device_name(device), ACPI_EC_DEVICE_NAME); + strcpy(acpi_device_class(device), ACPI_EC_CLASS); + + ec = acpi_ec_alloc(); + if (!ec) + return -ENOMEM; if (ec_parse_device(device->handle, 0, ec, NULL) != AE_CTRL_TERMINATE) { kfree(ec); @@ -1454,27 +1473,31 @@ static const struct acpi_device_id ec_device_ids[] = { int __init acpi_ec_dsdt_probe(void) { acpi_status status; + struct acpi_ec *ec; + int ret; - if (boot_ec) - return 0; - + ec = acpi_ec_alloc(); + if (!ec) + return -ENOMEM; /* * Finding EC from DSDT if there is no ECDT EC available. When this * function is invoked, ACPI tables have been fully loaded, we can * walk namespace now. */ - boot_ec = make_acpi_ec(); - if (!boot_ec) - return -ENOMEM; status = acpi_get_devices(ec_device_ids[0].id, - ec_parse_device, boot_ec, NULL); - if (ACPI_FAILURE(status) || !boot_ec->handle) - return -ENODEV; - if (!ec_install_handlers(boot_ec)) { - first_ec = boot_ec; - return 0; + ec_parse_device, ec, NULL); + if (ACPI_FAILURE(status) || !ec->handle) { + ret = -ENODEV; + goto error; } - return -EFAULT; + ret = ec_install_handlers(ec); + +error: + if (ret) + kfree(ec); + else + first_ec = boot_ec = ec; + return ret; } #if 0 @@ -1518,6 +1541,11 @@ static int ec_clear_on_resume(const struct dmi_system_id *id) return 0; } +/* + * Some ECDTs contain wrong register addresses. + * MSI MS-171F + * https://bugzilla.kernel.org/show_bug.cgi?id=12461 + */ static int ec_correct_ecdt(const struct dmi_system_id *id) { pr_debug("Detected system needing ECDT address correction.\n"); @@ -1526,16 +1554,6 @@ static int ec_correct_ecdt(const struct dmi_system_id *id) } static struct dmi_system_id ec_dmi_table[] __initdata = { - { - ec_correct_ecdt, "Asus L4R", { - DMI_MATCH(DMI_BIOS_VERSION, "1008.006"), - DMI_MATCH(DMI_PRODUCT_NAME, "L4R"), - DMI_MATCH(DMI_BOARD_NAME, "L4R") }, NULL}, - { - ec_correct_ecdt, "Asus M6R", { - DMI_MATCH(DMI_BIOS_VERSION, "0207"), - DMI_MATCH(DMI_PRODUCT_NAME, "M6R"), - DMI_MATCH(DMI_BOARD_NAME, "M6R") }, NULL}, { ec_correct_ecdt, "MSI MS-171F", { DMI_MATCH(DMI_SYS_VENDOR, "Micro-Star"), @@ -1548,12 +1566,13 @@ static struct dmi_system_id ec_dmi_table[] __initdata = { int __init acpi_ec_ecdt_probe(void) { - int ret = 0; + int ret; acpi_status status; struct acpi_table_ecdt *ecdt_ptr; + struct acpi_ec *ec; - boot_ec = make_acpi_ec(); - if (!boot_ec) + ec = acpi_ec_alloc(); + if (!ec) return -ENOMEM; /* * Generate a boot ec context @@ -1577,28 +1596,20 @@ int __init acpi_ec_ecdt_probe(void) pr_info("EC description table is found, configuring boot EC\n"); if (EC_FLAGS_CORRECT_ECDT) { - /* - * Asus L4R, Asus M6R - * https://bugzilla.kernel.org/show_bug.cgi?id=9399 - * MSI MS-171F - * https://bugzilla.kernel.org/show_bug.cgi?id=12461 - */ - boot_ec->command_addr = ecdt_ptr->data.address; - boot_ec->data_addr = ecdt_ptr->control.address; + ec->command_addr = ecdt_ptr->data.address; + ec->data_addr = ecdt_ptr->control.address; } else { - boot_ec->command_addr = ecdt_ptr->control.address; - boot_ec->data_addr = ecdt_ptr->data.address; + ec->command_addr = ecdt_ptr->control.address; + ec->data_addr = ecdt_ptr->data.address; } - boot_ec->gpe = ecdt_ptr->gpe; - boot_ec->handle = ACPI_ROOT_OBJECT; - ret = ec_install_handlers(boot_ec); - if (!ret) - first_ec = boot_ec; + ec->gpe = ecdt_ptr->gpe; + ec->handle = ACPI_ROOT_OBJECT; + ret = ec_install_handlers(ec); error: - if (ret) { - kfree(boot_ec); - boot_ec = NULL; - } + if (ret) + kfree(ec); + else + first_ec = boot_ec = ec; return ret; } diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 27cc7feabfe4..940218ff0193 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -87,6 +87,9 @@ bool acpi_queue_hotplug_work(struct work_struct *work); void acpi_device_hotplug(struct acpi_device *adev, u32 src); bool acpi_scan_is_offline(struct acpi_device *adev, bool uevent); +acpi_status acpi_sysfs_table_handler(u32 event, void *table, void *context); +void acpi_scan_table_handler(u32 event, void *table, void *context); + /* -------------------------------------------------------------------------- Device Node Initialization / Removal -------------------------------------------------------------------------- */ diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index 2215fc847fa9..1f0e06065ae6 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -928,7 +928,7 @@ static ssize_t format_show(struct device *dev, { struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); - return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->code)); + return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); } static DEVICE_ATTR_RO(format); @@ -961,8 +961,8 @@ static ssize_t format1_show(struct device *dev, continue; if (nfit_dcr->dcr->code == dcr->code) continue; - rc = sprintf(buf, "%#x\n", - be16_to_cpu(nfit_dcr->dcr->code)); + rc = sprintf(buf, "0x%04x\n", + le16_to_cpu(nfit_dcr->dcr->code)); break; } if (rc != ENXIO) @@ -1151,9 +1151,10 @@ static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, if (disable_vendor_specific) dsm_mask &= ~(1 << 8); } else { - dev_err(dev, "unknown dimm command family\n"); + dev_dbg(dev, "unknown dimm command family\n"); nfit_mem->family = -1; - return force_enable_dimms ? 0 : -ENODEV; + /* DSMs are optional, continue loading the driver... */ + return 0; } uuid = to_nfit_uuid(nfit_mem->family); diff --git a/drivers/acpi/nfit.h b/drivers/acpi/nfit.h index 11cb38348aef..02b9ea1e8d2e 100644 --- a/drivers/acpi/nfit.h +++ b/drivers/acpi/nfit.h @@ -53,12 +53,12 @@ enum nfit_uuids { }; /* - * Region format interface codes are stored as an array of bytes in the - * NFIT DIMM Control Region structure + * Region format interface codes are stored with the interface as the + * LSB and the function as the MSB. */ -#define NFIT_FIC_BYTE cpu_to_be16(0x101) /* byte-addressable energy backed */ -#define NFIT_FIC_BLK cpu_to_be16(0x201) /* block-addressable non-energy backed */ -#define NFIT_FIC_BYTEN cpu_to_be16(0x301) /* byte-addressable non-energy backed */ +#define NFIT_FIC_BYTE cpu_to_le16(0x101) /* byte-addressable energy backed */ +#define NFIT_FIC_BLK cpu_to_le16(0x201) /* block-addressable non-energy backed */ +#define NFIT_FIC_BYTEN cpu_to_le16(0x301) /* byte-addressable non-energy backed */ enum { NFIT_BLK_READ_FLUSH = 1, diff --git a/drivers/acpi/numa.c b/drivers/acpi/numa.c index d176e0ece470..ce3a7a16f03f 100644 --- a/drivers/acpi/numa.c +++ b/drivers/acpi/numa.c @@ -18,22 +18,21 @@ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * */ + +#define pr_fmt(fmt) "ACPI: " fmt + #include #include #include #include #include #include +#include +#include #include #include #include -#define PREFIX "ACPI: " - -#define ACPI_NUMA 0x80000000 -#define _COMPONENT ACPI_NUMA -ACPI_MODULE_NAME("numa"); - static nodemask_t nodes_found_map = NODE_MASK_NONE; /* maps to convert between proximity domain and logical node ID */ @@ -43,6 +42,7 @@ static int node_to_pxm_map[MAX_NUMNODES] = { [0 ... MAX_NUMNODES - 1] = PXM_INVAL }; unsigned char acpi_srat_revision __initdata; +int acpi_numa __initdata; int pxm_to_node(int pxm) { @@ -128,68 +128,63 @@ EXPORT_SYMBOL(acpi_map_pxm_to_online_node); static void __init acpi_table_print_srat_entry(struct acpi_subtable_header *header) { - - ACPI_FUNCTION_NAME("acpi_table_print_srat_entry"); - - if (!header) - return; - switch (header->type) { - case ACPI_SRAT_TYPE_CPU_AFFINITY: -#ifdef ACPI_DEBUG_OUTPUT { struct acpi_srat_cpu_affinity *p = (struct acpi_srat_cpu_affinity *)header; - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "SRAT Processor (id[0x%02x] eid[0x%02x]) in proximity domain %d %s\n", - p->apic_id, p->local_sapic_eid, - p->proximity_domain_lo, - (p->flags & ACPI_SRAT_CPU_ENABLED)? - "enabled" : "disabled")); + pr_debug("SRAT Processor (id[0x%02x] eid[0x%02x]) in proximity domain %d %s\n", + p->apic_id, p->local_sapic_eid, + p->proximity_domain_lo, + (p->flags & ACPI_SRAT_CPU_ENABLED) ? + "enabled" : "disabled"); } -#endif /* ACPI_DEBUG_OUTPUT */ break; case ACPI_SRAT_TYPE_MEMORY_AFFINITY: -#ifdef ACPI_DEBUG_OUTPUT { struct acpi_srat_mem_affinity *p = (struct acpi_srat_mem_affinity *)header; - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "SRAT Memory (0x%lx length 0x%lx) in proximity domain %d %s%s%s\n", - (unsigned long)p->base_address, - (unsigned long)p->length, - p->proximity_domain, - (p->flags & ACPI_SRAT_MEM_ENABLED)? - "enabled" : "disabled", - (p->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE)? - " hot-pluggable" : "", - (p->flags & ACPI_SRAT_MEM_NON_VOLATILE)? - " non-volatile" : "")); + pr_debug("SRAT Memory (0x%lx length 0x%lx) in proximity domain %d %s%s%s\n", + (unsigned long)p->base_address, + (unsigned long)p->length, + p->proximity_domain, + (p->flags & ACPI_SRAT_MEM_ENABLED) ? + "enabled" : "disabled", + (p->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) ? + " hot-pluggable" : "", + (p->flags & ACPI_SRAT_MEM_NON_VOLATILE) ? + " non-volatile" : ""); } -#endif /* ACPI_DEBUG_OUTPUT */ break; case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: -#ifdef ACPI_DEBUG_OUTPUT { struct acpi_srat_x2apic_cpu_affinity *p = (struct acpi_srat_x2apic_cpu_affinity *)header; - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "SRAT Processor (x2apicid[0x%08x]) in" - " proximity domain %d %s\n", - p->apic_id, - p->proximity_domain, - (p->flags & ACPI_SRAT_CPU_ENABLED) ? - "enabled" : "disabled")); + pr_debug("SRAT Processor (x2apicid[0x%08x]) in proximity domain %d %s\n", + p->apic_id, + p->proximity_domain, + (p->flags & ACPI_SRAT_CPU_ENABLED) ? + "enabled" : "disabled"); } -#endif /* ACPI_DEBUG_OUTPUT */ break; + + case ACPI_SRAT_TYPE_GICC_AFFINITY: + { + struct acpi_srat_gicc_affinity *p = + (struct acpi_srat_gicc_affinity *)header; + pr_debug("SRAT Processor (acpi id[0x%04x]) in proximity domain %d %s\n", + p->acpi_processor_uid, + p->proximity_domain, + (p->flags & ACPI_SRAT_GICC_ENABLED) ? + "enabled" : "disabled"); + } + break; + default: - printk(KERN_WARNING PREFIX - "Found unsupported SRAT entry (type = 0x%x)\n", - header->type); + pr_warn("Found unsupported SRAT entry (type = 0x%x)\n", + header->type); break; } } @@ -217,12 +212,117 @@ static int __init slit_valid(struct acpi_table_slit *slit) return 1; } +void __init bad_srat(void) +{ + pr_err("SRAT: SRAT not used.\n"); + acpi_numa = -1; +} + +int __init srat_disabled(void) +{ + return acpi_numa < 0; +} + +#if defined(CONFIG_X86) || defined(CONFIG_ARM64) +/* + * Callback for SLIT parsing. pxm_to_node() returns NUMA_NO_NODE for + * I/O localities since SRAT does not list them. I/O localities are + * not supported at this point. + */ +void __init acpi_numa_slit_init(struct acpi_table_slit *slit) +{ + int i, j; + + for (i = 0; i < slit->locality_count; i++) { + const int from_node = pxm_to_node(i); + + if (from_node == NUMA_NO_NODE) + continue; + + for (j = 0; j < slit->locality_count; j++) { + const int to_node = pxm_to_node(j); + + if (to_node == NUMA_NO_NODE) + continue; + + numa_set_distance(from_node, to_node, + slit->entry[slit->locality_count * i + j]); + } + } +} + +/* + * Default callback for parsing of the Proximity Domain <-> Memory + * Area mappings + */ +int __init +acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) +{ + u64 start, end; + u32 hotpluggable; + int node, pxm; + + if (srat_disabled()) + goto out_err; + if (ma->header.length < sizeof(struct acpi_srat_mem_affinity)) { + pr_err("SRAT: Unexpected header length: %d\n", + ma->header.length); + goto out_err_bad_srat; + } + if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0) + goto out_err; + hotpluggable = ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE; + if (hotpluggable && !IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) + goto out_err; + + start = ma->base_address; + end = start + ma->length; + pxm = ma->proximity_domain; + if (acpi_srat_revision <= 1) + pxm &= 0xff; + + node = acpi_map_pxm_to_node(pxm); + if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { + pr_err("SRAT: Too many proximity domains.\n"); + goto out_err_bad_srat; + } + + if (numa_add_memblk(node, start, end) < 0) { + pr_err("SRAT: Failed to add memblk to node %u [mem %#010Lx-%#010Lx]\n", + node, (unsigned long long) start, + (unsigned long long) end - 1); + goto out_err_bad_srat; + } + + node_set(node, numa_nodes_parsed); + + pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n", + node, pxm, + (unsigned long long) start, (unsigned long long) end - 1, + hotpluggable ? " hotplug" : "", + ma->flags & ACPI_SRAT_MEM_NON_VOLATILE ? " non-volatile" : ""); + + /* Mark hotplug range in memblock. */ + if (hotpluggable && memblock_mark_hotplug(start, ma->length)) + pr_warn("SRAT: Failed to mark hotplug range [mem %#010Lx-%#010Lx] in memblock\n", + (unsigned long long)start, (unsigned long long)end - 1); + + max_possible_pfn = max(max_possible_pfn, PFN_UP(end - 1)); + + return 0; +out_err_bad_srat: + bad_srat(); +out_err: + return -EINVAL; +} +#endif /* defined(CONFIG_X86) || defined (CONFIG_ARM64) */ + static int __init acpi_parse_slit(struct acpi_table_header *table) { struct acpi_table_slit *slit = (struct acpi_table_slit *)table; if (!slit_valid(slit)) { - printk(KERN_INFO "ACPI: SLIT table looks invalid. Not used.\n"); + pr_info("SLIT table looks invalid. Not used.\n"); return -EINVAL; } acpi_numa_slit_init(slit); @@ -233,12 +333,9 @@ static int __init acpi_parse_slit(struct acpi_table_header *table) void __init __weak acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) { - printk(KERN_WARNING PREFIX - "Found unsupported x2apic [0x%08x] SRAT entry\n", pa->apic_id); - return; + pr_warn("Found unsupported x2apic [0x%08x] SRAT entry\n", pa->apic_id); } - static int __init acpi_parse_x2apic_affinity(struct acpi_subtable_header *header, const unsigned long end) @@ -275,6 +372,24 @@ acpi_parse_processor_affinity(struct acpi_subtable_header *header, return 0; } +static int __init +acpi_parse_gicc_affinity(struct acpi_subtable_header *header, + const unsigned long end) +{ + struct acpi_srat_gicc_affinity *processor_affinity; + + processor_affinity = (struct acpi_srat_gicc_affinity *)header; + if (!processor_affinity) + return -EINVAL; + + acpi_table_print_srat_entry(header); + + /* let architecture-dependent part to do it */ + acpi_numa_gicc_affinity_init(processor_affinity); + + return 0; +} + static int __initdata parsed_numa_memblks; static int __init @@ -319,6 +434,9 @@ int __init acpi_numa_init(void) { int cnt = 0; + if (acpi_disabled) + return -EINVAL; + /* * Should not limit number with cpu num that is from NR_CPUS or nr_cpus= * SRAT cpu entries could have different order with that in MADT. @@ -327,13 +445,15 @@ int __init acpi_numa_init(void) /* SRAT: Static Resource Affinity Table */ if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) { - struct acpi_subtable_proc srat_proc[2]; + struct acpi_subtable_proc srat_proc[3]; memset(srat_proc, 0, sizeof(srat_proc)); srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY; srat_proc[0].handler = acpi_parse_processor_affinity; srat_proc[1].id = ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY; srat_proc[1].handler = acpi_parse_x2apic_affinity; + srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY; + srat_proc[2].handler = acpi_parse_gicc_affinity; acpi_table_parse_entries_array(ACPI_SIG_SRAT, sizeof(struct acpi_table_srat), @@ -347,8 +467,6 @@ int __init acpi_numa_init(void) /* SLIT: System Locality Information Table */ acpi_table_parse(ACPI_SIG_SLIT, acpi_parse_slit); - acpi_numa_arch_fixup(); - if (cnt < 0) return cnt; else if (!parsed_numa_memblks) diff --git a/drivers/acpi/pci_link.c b/drivers/acpi/pci_link.c index 8fc7323ed3e8..c983bf733ad3 100644 --- a/drivers/acpi/pci_link.c +++ b/drivers/acpi/pci_link.c @@ -470,6 +470,7 @@ static int acpi_irq_pci_sharing_penalty(int irq) { struct acpi_pci_link *link; int penalty = 0; + int i; list_for_each_entry(link, &acpi_link_list, list) { /* @@ -478,18 +479,14 @@ static int acpi_irq_pci_sharing_penalty(int irq) */ if (link->irq.active && link->irq.active == irq) penalty += PIRQ_PENALTY_PCI_USING; - else { - int i; - - /* - * If a link is inactive, penalize the IRQs it - * might use, but not as severely. - */ - for (i = 0; i < link->irq.possible_count; i++) - if (link->irq.possible[i] == irq) - penalty += PIRQ_PENALTY_PCI_POSSIBLE / - link->irq.possible_count; - } + + /* + * penalize the IRQs PCI might use, but not as severely. + */ + for (i = 0; i < link->irq.possible_count; i++) + if (link->irq.possible[i] == irq) + penalty += PIRQ_PENALTY_PCI_POSSIBLE / + link->irq.possible_count; } return penalty; @@ -499,9 +496,6 @@ static int acpi_irq_get_penalty(int irq) { int penalty = 0; - if (irq < ACPI_MAX_ISA_IRQS) - penalty += acpi_isa_irq_penalty[irq]; - /* * Penalize IRQ used by ACPI SCI. If ACPI SCI pin attributes conflict * with PCI IRQ attributes, mark ACPI SCI as ISA_ALWAYS so it won't be @@ -516,10 +510,49 @@ static int acpi_irq_get_penalty(int irq) penalty += PIRQ_PENALTY_PCI_USING; } + if (irq < ACPI_MAX_ISA_IRQS) + return penalty + acpi_isa_irq_penalty[irq]; + penalty += acpi_irq_pci_sharing_penalty(irq); return penalty; } +int __init acpi_irq_penalty_init(void) +{ + struct acpi_pci_link *link; + int i; + + /* + * Update penalties to facilitate IRQ balancing. + */ + list_for_each_entry(link, &acpi_link_list, list) { + + /* + * reflect the possible and active irqs in the penalty table -- + * useful for breaking ties. + */ + if (link->irq.possible_count) { + int penalty = + PIRQ_PENALTY_PCI_POSSIBLE / + link->irq.possible_count; + + for (i = 0; i < link->irq.possible_count; i++) { + if (link->irq.possible[i] < ACPI_MAX_ISA_IRQS) + acpi_isa_irq_penalty[link->irq. + possible[i]] += + penalty; + } + + } else if (link->irq.active && + (link->irq.active < ACPI_MAX_ISA_IRQS)) { + acpi_isa_irq_penalty[link->irq.active] += + PIRQ_PENALTY_PCI_POSSIBLE; + } + } + + return 0; +} + static int acpi_irq_balance = -1; /* 0: static, 1: balance */ static int acpi_pci_link_allocate(struct acpi_pci_link *link) @@ -839,7 +872,7 @@ void acpi_penalize_isa_irq(int irq, int active) { if ((irq >= 0) && (irq < ARRAY_SIZE(acpi_isa_irq_penalty))) acpi_isa_irq_penalty[irq] = acpi_irq_get_penalty(irq) + - active ? PIRQ_PENALTY_ISA_USED : PIRQ_PENALTY_PCI_USING; + (active ? PIRQ_PENALTY_ISA_USED : PIRQ_PENALTY_PCI_USING); } bool acpi_isa_irq_available(int irq) diff --git a/drivers/acpi/pci_slot.c b/drivers/acpi/pci_slot.c index 7188e53b6b7c..f62c68e24317 100644 --- a/drivers/acpi/pci_slot.c +++ b/drivers/acpi/pci_slot.c @@ -22,8 +22,9 @@ * General Public License for more details. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include -#include #include #include #include @@ -33,30 +34,11 @@ #include #include -static bool debug; static int check_sta_before_sun; -#define DRIVER_VERSION "0.1" -#define DRIVER_AUTHOR "Alex Chiang " -#define DRIVER_DESC "ACPI PCI Slot Detection Driver" -MODULE_AUTHOR(DRIVER_AUTHOR); -MODULE_DESCRIPTION(DRIVER_DESC); -MODULE_LICENSE("GPL"); -MODULE_PARM_DESC(debug, "Debugging mode enabled or not"); -module_param(debug, bool, 0644); - #define _COMPONENT ACPI_PCI_COMPONENT ACPI_MODULE_NAME("pci_slot"); -#define MY_NAME "pci_slot" -#define err(format, arg...) pr_err("%s: " format , MY_NAME , ## arg) -#define info(format, arg...) pr_info("%s: " format , MY_NAME , ## arg) -#define dbg(format, arg...) \ - do { \ - if (debug) \ - pr_debug("%s: " format, MY_NAME , ## arg); \ - } while (0) - #define SLOT_NAME_SIZE 21 /* Inspired by #define in acpiphp.h */ struct acpi_pci_slot { @@ -76,7 +58,7 @@ check_slot(acpi_handle handle, unsigned long long *sun) struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; acpi_get_name(handle, ACPI_FULL_PATHNAME, &buffer); - dbg("Checking slot on path: %s\n", (char *)buffer.pointer); + pr_debug("Checking slot on path: %s\n", (char *)buffer.pointer); if (check_sta_before_sun) { /* If SxFy doesn't have _STA, we just assume it's there */ @@ -87,14 +69,16 @@ check_slot(acpi_handle handle, unsigned long long *sun) status = acpi_evaluate_integer(handle, "_ADR", NULL, &adr); if (ACPI_FAILURE(status)) { - dbg("_ADR returned %d on %s\n", status, (char *)buffer.pointer); + pr_debug("_ADR returned %d on %s\n", + status, (char *)buffer.pointer); goto out; } /* No _SUN == not a slot == bail */ status = acpi_evaluate_integer(handle, "_SUN", NULL, sun); if (ACPI_FAILURE(status)) { - dbg("_SUN returned %d on %s\n", status, (char *)buffer.pointer); + pr_debug("_SUN returned %d on %s\n", + status, (char *)buffer.pointer); goto out; } @@ -132,15 +116,13 @@ register_slot(acpi_handle handle, u32 lvl, void *context, void **rv) } slot = kmalloc(sizeof(*slot), GFP_KERNEL); - if (!slot) { - err("%s: cannot allocate memory\n", __func__); + if (!slot) return AE_OK; - } snprintf(name, sizeof(name), "%llu", sun); pci_slot = pci_create_slot(pci_bus, device, name, NULL); if (IS_ERR(pci_slot)) { - err("pci_create_slot returned %ld\n", PTR_ERR(pci_slot)); + pr_err("pci_create_slot returned %ld\n", PTR_ERR(pci_slot)); kfree(slot); return AE_OK; } @@ -150,8 +132,8 @@ register_slot(acpi_handle handle, u32 lvl, void *context, void **rv) get_device(&pci_bus->dev); - dbg("pci_slot: %p, pci_bus: %x, device: %d, name: %s\n", - pci_slot, pci_bus->number, device, name); + pr_debug("%p, pci_bus: %x, device: %d, name: %s\n", + pci_slot, pci_bus->number, device, name); return AE_OK; } @@ -186,7 +168,8 @@ void acpi_pci_slot_remove(struct pci_bus *bus) static int do_sta_before_sun(const struct dmi_system_id *d) { - info("%s detected: will evaluate _STA before calling _SUN\n", d->ident); + pr_info("%s detected: will evaluate _STA before calling _SUN\n", + d->ident); check_sta_before_sun = 1; return 0; } diff --git a/drivers/acpi/pmic/intel_pmic.c b/drivers/acpi/pmic/intel_pmic.c index bd772cd56494..ca18e0d23df9 100644 --- a/drivers/acpi/pmic/intel_pmic.c +++ b/drivers/acpi/pmic/intel_pmic.c @@ -13,7 +13,7 @@ * GNU General Public License for more details. */ -#include +#include #include #include #include @@ -21,12 +21,19 @@ #define PMIC_POWER_OPREGION_ID 0x8d #define PMIC_THERMAL_OPREGION_ID 0x8c +#define PMIC_REGS_OPREGION_ID 0x8f + +struct intel_pmic_regs_handler_ctx { + unsigned int val; + u16 addr; +}; struct intel_pmic_opregion { struct mutex lock; struct acpi_lpat_conversion_table *lpat_table; struct regmap *regmap; struct intel_pmic_opregion_data *data; + struct intel_pmic_regs_handler_ctx ctx; }; static int pmic_get_reg_bit(int address, struct pmic_table *table, @@ -131,7 +138,7 @@ static int pmic_thermal_aux(struct intel_pmic_opregion *opregion, int reg, } static int pmic_thermal_pen(struct intel_pmic_opregion *opregion, int reg, - u32 function, u64 *value) + int bit, u32 function, u64 *value) { struct intel_pmic_opregion_data *d = opregion->data; struct regmap *regmap = opregion->regmap; @@ -140,12 +147,12 @@ static int pmic_thermal_pen(struct intel_pmic_opregion *opregion, int reg, return -ENXIO; if (function == ACPI_READ) - return d->get_policy(regmap, reg, value); + return d->get_policy(regmap, reg, bit, value); if (*value != 0 && *value != 1) return -EINVAL; - return d->update_policy(regmap, reg, *value); + return d->update_policy(regmap, reg, bit, *value); } static bool pmic_thermal_is_temp(int address) @@ -170,13 +177,13 @@ static acpi_status intel_pmic_thermal_handler(u32 function, { struct intel_pmic_opregion *opregion = region_context; struct intel_pmic_opregion_data *d = opregion->data; - int reg, result; + int reg, bit, result; if (bits != 32 || !value64) return AE_BAD_PARAMETER; result = pmic_get_reg_bit(address, d->thermal_table, - d->thermal_table_count, ®, NULL); + d->thermal_table_count, ®, &bit); if (result == -ENOENT) return AE_BAD_PARAMETER; @@ -187,7 +194,8 @@ static acpi_status intel_pmic_thermal_handler(u32 function, else if (pmic_thermal_is_aux(address)) result = pmic_thermal_aux(opregion, reg, function, value64); else if (pmic_thermal_is_pen(address)) - result = pmic_thermal_pen(opregion, reg, function, value64); + result = pmic_thermal_pen(opregion, reg, bit, + function, value64); else result = -EINVAL; @@ -203,6 +211,48 @@ static acpi_status intel_pmic_thermal_handler(u32 function, return AE_OK; } +static acpi_status intel_pmic_regs_handler(u32 function, + acpi_physical_address address, u32 bits, u64 *value64, + void *handler_context, void *region_context) +{ + struct intel_pmic_opregion *opregion = region_context; + int result = 0; + + switch (address) { + case 0: + return AE_OK; + case 1: + opregion->ctx.addr |= (*value64 & 0xff) << 8; + return AE_OK; + case 2: + opregion->ctx.addr |= *value64 & 0xff; + return AE_OK; + case 3: + opregion->ctx.val = *value64 & 0xff; + return AE_OK; + case 4: + if (*value64) { + result = regmap_write(opregion->regmap, opregion->ctx.addr, + opregion->ctx.val); + } else { + result = regmap_read(opregion->regmap, opregion->ctx.addr, + &opregion->ctx.val); + if (result == 0) + *value64 = opregion->ctx.val; + } + memset(&opregion->ctx, 0x00, sizeof(opregion->ctx)); + } + + if (result < 0) { + if (result == -EINVAL) + return AE_BAD_PARAMETER; + else + return AE_ERROR; + } + + return AE_OK; +} + int intel_pmic_install_opregion_handler(struct device *dev, acpi_handle handle, struct regmap *regmap, struct intel_pmic_opregion_data *d) @@ -242,16 +292,30 @@ int intel_pmic_install_opregion_handler(struct device *dev, acpi_handle handle, acpi_remove_address_space_handler(handle, PMIC_POWER_OPREGION_ID, intel_pmic_power_handler); ret = -ENODEV; - goto out_error; + goto out_remove_power_handler; + } + + status = acpi_install_address_space_handler(handle, + PMIC_REGS_OPREGION_ID, intel_pmic_regs_handler, NULL, + opregion); + if (ACPI_FAILURE(status)) { + ret = -ENODEV; + goto out_remove_thermal_handler; } opregion->data = d; return 0; +out_remove_thermal_handler: + acpi_remove_address_space_handler(handle, PMIC_THERMAL_OPREGION_ID, + intel_pmic_thermal_handler); + +out_remove_power_handler: + acpi_remove_address_space_handler(handle, PMIC_POWER_OPREGION_ID, + intel_pmic_power_handler); + out_error: acpi_lpat_free_conversion_table(opregion->lpat_table); return ret; } EXPORT_SYMBOL_GPL(intel_pmic_install_opregion_handler); - -MODULE_LICENSE("GPL"); diff --git a/drivers/acpi/pmic/intel_pmic.h b/drivers/acpi/pmic/intel_pmic.h index d4e90af8f0dd..e8bfa7b865a5 100644 --- a/drivers/acpi/pmic/intel_pmic.h +++ b/drivers/acpi/pmic/intel_pmic.h @@ -12,8 +12,8 @@ struct intel_pmic_opregion_data { int (*update_power)(struct regmap *r, int reg, int bit, bool on); int (*get_raw_temp)(struct regmap *r, int reg); int (*update_aux)(struct regmap *r, int reg, int raw_temp); - int (*get_policy)(struct regmap *r, int reg, u64 *value); - int (*update_policy)(struct regmap *r, int reg, int enable); + int (*get_policy)(struct regmap *r, int reg, int bit, u64 *value); + int (*update_policy)(struct regmap *r, int reg, int bit, int enable); struct pmic_table *power_table; int power_table_count; struct pmic_table *thermal_table; diff --git a/drivers/acpi/pmic/intel_pmic_bxtwc.c b/drivers/acpi/pmic/intel_pmic_bxtwc.c new file mode 100644 index 000000000000..90011aad4d20 --- /dev/null +++ b/drivers/acpi/pmic/intel_pmic_bxtwc.c @@ -0,0 +1,420 @@ +/* + * intel_pmic_bxtwc.c - Intel BXT WhiskeyCove PMIC operation region driver + * + * Copyright (C) 2015 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include "intel_pmic.h" + +#define WHISKEY_COVE_ALRT_HIGH_BIT_MASK 0x0F +#define WHISKEY_COVE_ADC_HIGH_BIT(x) (((x & 0x0F) << 8)) +#define WHISKEY_COVE_ADC_CURSRC(x) (((x & 0xF0) >> 4)) +#define VR_MODE_DISABLED 0 +#define VR_MODE_AUTO BIT(0) +#define VR_MODE_NORMAL BIT(1) +#define VR_MODE_SWITCH BIT(2) +#define VR_MODE_ECO (BIT(0)|BIT(1)) +#define VSWITCH2_OUTPUT BIT(5) +#define VSWITCH1_OUTPUT BIT(4) +#define VUSBPHY_CHARGE BIT(1) + +static struct pmic_table power_table[] = { + { + .address = 0x0, + .reg = 0x63, + .bit = VR_MODE_AUTO, + }, /* VDD1 -> VDD1CNT */ + { + .address = 0x04, + .reg = 0x65, + .bit = VR_MODE_AUTO, + }, /* VDD2 -> VDD2CNT */ + { + .address = 0x08, + .reg = 0x67, + .bit = VR_MODE_AUTO, + }, /* VDD3 -> VDD3CNT */ + { + .address = 0x0c, + .reg = 0x6d, + .bit = VR_MODE_AUTO, + }, /* VLFX -> VFLEXCNT */ + { + .address = 0x10, + .reg = 0x6f, + .bit = VR_MODE_NORMAL, + }, /* VP1A -> VPROG1ACNT */ + { + .address = 0x14, + .reg = 0x70, + .bit = VR_MODE_NORMAL, + }, /* VP1B -> VPROG1BCNT */ + { + .address = 0x18, + .reg = 0x71, + .bit = VR_MODE_NORMAL, + }, /* VP1C -> VPROG1CCNT */ + { + .address = 0x1c, + .reg = 0x72, + .bit = VR_MODE_NORMAL, + }, /* VP1D -> VPROG1DCNT */ + { + .address = 0x20, + .reg = 0x73, + .bit = VR_MODE_NORMAL, + }, /* VP2A -> VPROG2ACNT */ + { + .address = 0x24, + .reg = 0x74, + .bit = VR_MODE_NORMAL, + }, /* VP2B -> VPROG2BCNT */ + { + .address = 0x28, + .reg = 0x75, + .bit = VR_MODE_NORMAL, + }, /* VP2C -> VPROG2CCNT */ + { + .address = 0x2c, + .reg = 0x76, + .bit = VR_MODE_NORMAL, + }, /* VP3A -> VPROG3ACNT */ + { + .address = 0x30, + .reg = 0x77, + .bit = VR_MODE_NORMAL, + }, /* VP3B -> VPROG3BCNT */ + { + .address = 0x34, + .reg = 0x78, + .bit = VSWITCH2_OUTPUT, + }, /* VSW2 -> VLD0CNT Bit 5*/ + { + .address = 0x38, + .reg = 0x78, + .bit = VSWITCH1_OUTPUT, + }, /* VSW1 -> VLD0CNT Bit 4 */ + { + .address = 0x3c, + .reg = 0x78, + .bit = VUSBPHY_CHARGE, + }, /* VUPY -> VLDOCNT Bit 1 */ + { + .address = 0x40, + .reg = 0x7b, + .bit = VR_MODE_NORMAL, + }, /* VRSO -> VREFSOCCNT*/ + { + .address = 0x44, + .reg = 0xA0, + .bit = VR_MODE_NORMAL, + }, /* VP1E -> VPROG1ECNT */ + { + .address = 0x48, + .reg = 0xA1, + .bit = VR_MODE_NORMAL, + }, /* VP1F -> VPROG1FCNT */ + { + .address = 0x4c, + .reg = 0xA2, + .bit = VR_MODE_NORMAL, + }, /* VP2D -> VPROG2DCNT */ + { + .address = 0x50, + .reg = 0xA3, + .bit = VR_MODE_NORMAL, + }, /* VP4A -> VPROG4ACNT */ + { + .address = 0x54, + .reg = 0xA4, + .bit = VR_MODE_NORMAL, + }, /* VP4B -> VPROG4BCNT */ + { + .address = 0x58, + .reg = 0xA5, + .bit = VR_MODE_NORMAL, + }, /* VP4C -> VPROG4CCNT */ + { + .address = 0x5c, + .reg = 0xA6, + .bit = VR_MODE_NORMAL, + }, /* VP4D -> VPROG4DCNT */ + { + .address = 0x60, + .reg = 0xA7, + .bit = VR_MODE_NORMAL, + }, /* VP5A -> VPROG5ACNT */ + { + .address = 0x64, + .reg = 0xA8, + .bit = VR_MODE_NORMAL, + }, /* VP5B -> VPROG5BCNT */ + { + .address = 0x68, + .reg = 0xA9, + .bit = VR_MODE_NORMAL, + }, /* VP6A -> VPROG6ACNT */ + { + .address = 0x6c, + .reg = 0xAA, + .bit = VR_MODE_NORMAL, + }, /* VP6B -> VPROG6BCNT */ + { + .address = 0x70, + .reg = 0x36, + .bit = BIT(2), + }, /* SDWN_N -> MODEMCTRL Bit 2 */ + { + .address = 0x74, + .reg = 0x36, + .bit = BIT(0), + } /* MOFF -> MODEMCTRL Bit 0 */ +}; + +static struct pmic_table thermal_table[] = { + { + .address = 0x00, + .reg = 0x4F39 + }, + { + .address = 0x04, + .reg = 0x4F24 + }, + { + .address = 0x08, + .reg = 0x4F26 + }, + { + .address = 0x0c, + .reg = 0x4F3B + }, + { + .address = 0x10, + .reg = 0x4F28 + }, + { + .address = 0x14, + .reg = 0x4F2A + }, + { + .address = 0x18, + .reg = 0x4F3D + }, + { + .address = 0x1c, + .reg = 0x4F2C + }, + { + .address = 0x20, + .reg = 0x4F2E + }, + { + .address = 0x24, + .reg = 0x4F3F + }, + { + .address = 0x28, + .reg = 0x4F30 + }, + { + .address = 0x30, + .reg = 0x4F41 + }, + { + .address = 0x34, + .reg = 0x4F32 + }, + { + .address = 0x3c, + .reg = 0x4F43 + }, + { + .address = 0x40, + .reg = 0x4F34 + }, + { + .address = 0x48, + .reg = 0x4F6A, + .bit = 0, + }, + { + .address = 0x4C, + .reg = 0x4F6A, + .bit = 1 + }, + { + .address = 0x50, + .reg = 0x4F6A, + .bit = 2 + }, + { + .address = 0x54, + .reg = 0x4F6A, + .bit = 4 + }, + { + .address = 0x58, + .reg = 0x4F6A, + .bit = 5 + }, + { + .address = 0x5C, + .reg = 0x4F6A, + .bit = 3 + }, +}; + +static int intel_bxtwc_pmic_get_power(struct regmap *regmap, int reg, + int bit, u64 *value) +{ + int data; + + if (regmap_read(regmap, reg, &data)) + return -EIO; + + *value = (data & bit) ? 1 : 0; + return 0; +} + +static int intel_bxtwc_pmic_update_power(struct regmap *regmap, int reg, + int bit, bool on) +{ + u8 val, mask = bit; + + if (on) + val = 0xFF; + else + val = 0x0; + + return regmap_update_bits(regmap, reg, mask, val); +} + +static int intel_bxtwc_pmic_get_raw_temp(struct regmap *regmap, int reg) +{ + unsigned int val, adc_val, reg_val; + u8 temp_l, temp_h, cursrc; + unsigned long rlsb; + static const unsigned long rlsb_array[] = { + 0, 260420, 130210, 65100, 32550, 16280, + 8140, 4070, 2030, 0, 260420, 130210 }; + + if (regmap_read(regmap, reg, &val)) + return -EIO; + temp_l = (u8) val; + + if (regmap_read(regmap, (reg - 1), &val)) + return -EIO; + temp_h = (u8) val; + + reg_val = temp_l | WHISKEY_COVE_ADC_HIGH_BIT(temp_h); + cursrc = WHISKEY_COVE_ADC_CURSRC(temp_h); + rlsb = rlsb_array[cursrc]; + adc_val = reg_val * rlsb / 1000; + + return adc_val; +} + +static int +intel_bxtwc_pmic_update_aux(struct regmap *regmap, int reg, int raw) +{ + u32 bsr_num; + u16 resi_val, count = 0, thrsh = 0; + u8 alrt_h, alrt_l, cursel = 0; + + bsr_num = raw; + bsr_num /= (1 << 5); + + count = fls(bsr_num) - 1; + + cursel = clamp_t(s8, (count - 7), 0, 7); + thrsh = raw / (1 << (4 + cursel)); + + resi_val = (cursel << 9) | thrsh; + alrt_h = (resi_val >> 8) & WHISKEY_COVE_ALRT_HIGH_BIT_MASK; + if (regmap_update_bits(regmap, + reg - 1, + WHISKEY_COVE_ALRT_HIGH_BIT_MASK, + alrt_h)) + return -EIO; + + alrt_l = (u8)resi_val; + return regmap_write(regmap, reg, alrt_l); +} + +static int +intel_bxtwc_pmic_get_policy(struct regmap *regmap, int reg, int bit, u64 *value) +{ + u8 mask = BIT(bit); + unsigned int val; + + if (regmap_read(regmap, reg, &val)) + return -EIO; + + *value = (val & mask) >> bit; + return 0; +} + +static int +intel_bxtwc_pmic_update_policy(struct regmap *regmap, + int reg, int bit, int enable) +{ + u8 mask = BIT(bit), val = enable << bit; + + return regmap_update_bits(regmap, reg, mask, val); +} + +static struct intel_pmic_opregion_data intel_bxtwc_pmic_opregion_data = { + .get_power = intel_bxtwc_pmic_get_power, + .update_power = intel_bxtwc_pmic_update_power, + .get_raw_temp = intel_bxtwc_pmic_get_raw_temp, + .update_aux = intel_bxtwc_pmic_update_aux, + .get_policy = intel_bxtwc_pmic_get_policy, + .update_policy = intel_bxtwc_pmic_update_policy, + .power_table = power_table, + .power_table_count = ARRAY_SIZE(power_table), + .thermal_table = thermal_table, + .thermal_table_count = ARRAY_SIZE(thermal_table), +}; + +static int intel_bxtwc_pmic_opregion_probe(struct platform_device *pdev) +{ + struct intel_soc_pmic *pmic = dev_get_drvdata(pdev->dev.parent); + + return intel_pmic_install_opregion_handler(&pdev->dev, + ACPI_HANDLE(pdev->dev.parent), + pmic->regmap, + &intel_bxtwc_pmic_opregion_data); +} + +static struct platform_device_id bxt_wc_opregion_id_table[] = { + { .name = "bxt_wcove_region" }, + {}, +}; + +static struct platform_driver intel_bxtwc_pmic_opregion_driver = { + .probe = intel_bxtwc_pmic_opregion_probe, + .driver = { + .name = "bxt_whiskey_cove_pmic", + }, + .id_table = bxt_wc_opregion_id_table, +}; + +static int __init intel_bxtwc_pmic_opregion_driver_init(void) +{ + return platform_driver_register(&intel_bxtwc_pmic_opregion_driver); +} +device_initcall(intel_bxtwc_pmic_opregion_driver_init); diff --git a/drivers/acpi/pmic/intel_pmic_crc.c b/drivers/acpi/pmic/intel_pmic_crc.c index fcd1852dcdee..d7f1761ab1bc 100644 --- a/drivers/acpi/pmic/intel_pmic_crc.c +++ b/drivers/acpi/pmic/intel_pmic_crc.c @@ -141,7 +141,8 @@ static int intel_crc_pmic_update_aux(struct regmap *regmap, int reg, int raw) regmap_update_bits(regmap, reg - 1, 0x3, raw >> 8) ? -EIO : 0; } -static int intel_crc_pmic_get_policy(struct regmap *regmap, int reg, u64 *value) +static int intel_crc_pmic_get_policy(struct regmap *regmap, + int reg, int bit, u64 *value) { int pen; @@ -152,7 +153,7 @@ static int intel_crc_pmic_get_policy(struct regmap *regmap, int reg, u64 *value) } static int intel_crc_pmic_update_policy(struct regmap *regmap, - int reg, int enable) + int reg, int bit, int enable) { int alert0; diff --git a/drivers/acpi/pmic/intel_pmic_xpower.c b/drivers/acpi/pmic/intel_pmic_xpower.c index 6a082d4de12c..e6e991ac20f3 100644 --- a/drivers/acpi/pmic/intel_pmic_xpower.c +++ b/drivers/acpi/pmic/intel_pmic_xpower.c @@ -13,7 +13,7 @@ * GNU General Public License for more details. */ -#include +#include #include #include #include @@ -262,7 +262,4 @@ static int __init intel_xpower_pmic_opregion_driver_init(void) { return platform_driver_register(&intel_xpower_pmic_opregion_driver); } -module_init(intel_xpower_pmic_opregion_driver_init); - -MODULE_DESCRIPTION("XPower AXP288 ACPI operation region driver"); -MODULE_LICENSE("GPL"); +device_initcall(intel_xpower_pmic_opregion_driver_init); diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index 33a38d604630..9125d7d96372 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -108,13 +108,12 @@ static int map_gicc_mpidr(struct acpi_subtable_header *entry, return -EINVAL; } -static phys_cpuid_t map_madt_entry(int type, u32 acpi_id) +static phys_cpuid_t map_madt_entry(struct acpi_table_madt *madt, + int type, u32 acpi_id) { unsigned long madt_end, entry; phys_cpuid_t phys_id = PHYS_CPUID_INVALID; /* CPU hardware ID */ - struct acpi_table_madt *madt; - madt = get_madt_table(); if (!madt) return phys_id; @@ -145,6 +144,25 @@ static phys_cpuid_t map_madt_entry(int type, u32 acpi_id) return phys_id; } +phys_cpuid_t __init acpi_map_madt_entry(u32 acpi_id) +{ + struct acpi_table_madt *madt = NULL; + acpi_size tbl_size; + phys_cpuid_t rv; + + acpi_get_table_with_size(ACPI_SIG_MADT, 0, + (struct acpi_table_header **)&madt, + &tbl_size); + if (!madt) + return PHYS_CPUID_INVALID; + + rv = map_madt_entry(madt, 1, acpi_id); + + early_acpi_os_unmap_memory(madt, tbl_size); + + return rv; +} + static phys_cpuid_t map_mat_entry(acpi_handle handle, int type, u32 acpi_id) { struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; @@ -185,7 +203,7 @@ phys_cpuid_t acpi_get_phys_id(acpi_handle handle, int type, u32 acpi_id) phys_id = map_mat_entry(handle, type, acpi_id); if (invalid_phys_cpuid(phys_id)) - phys_id = map_madt_entry(type, acpi_id); + phys_id = map_madt_entry(get_madt_table(), type, acpi_id); return phys_id; } diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index d2fa8cb82d2b..0ca14ac7bb28 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -90,7 +90,7 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data) pr->performance_platform_limit); break; case ACPI_PROCESSOR_NOTIFY_POWER: - acpi_processor_cst_has_changed(pr); + acpi_processor_power_state_has_changed(pr); acpi_bus_generate_netlink_event(device->pnp.device_class, dev_name(&device->dev), event, 0); break; diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 444e3745c8b3..cea52528aa18 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -59,6 +59,12 @@ module_param(latency_factor, uint, 0644); static DEFINE_PER_CPU(struct cpuidle_device *, acpi_cpuidle_device); +struct cpuidle_driver acpi_idle_driver = { + .name = "acpi_idle", + .owner = THIS_MODULE, +}; + +#ifdef CONFIG_ACPI_PROCESSOR_CSTATE static DEFINE_PER_CPU(struct acpi_processor_cx * [CPUIDLE_STATE_MAX], acpi_cstate); @@ -297,7 +303,6 @@ static int acpi_processor_get_power_info_cst(struct acpi_processor *pr) struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; union acpi_object *cst; - if (nocst) return -ENODEV; @@ -570,7 +575,7 @@ static int acpi_processor_power_verify(struct acpi_processor *pr) return (working); } -static int acpi_processor_get_power_info(struct acpi_processor *pr) +static int acpi_processor_get_cstate_info(struct acpi_processor *pr) { unsigned int i; int result; @@ -804,36 +809,12 @@ static void acpi_idle_enter_freeze(struct cpuidle_device *dev, acpi_idle_do_entry(cx); } -struct cpuidle_driver acpi_idle_driver = { - .name = "acpi_idle", - .owner = THIS_MODULE, -}; - -/** - * acpi_processor_setup_cpuidle_cx - prepares and configures CPUIDLE - * device i.e. per-cpu data - * - * @pr: the ACPI processor - * @dev : the cpuidle device - */ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, struct cpuidle_device *dev) { int i, count = CPUIDLE_DRIVER_STATE_START; struct acpi_processor_cx *cx; - if (!pr->flags.power_setup_done) - return -EINVAL; - - if (pr->flags.power == 0) { - return -EINVAL; - } - - if (!dev) - return -EINVAL; - - dev->cpu = pr->id; - if (max_cstate == 0) max_cstate = 1; @@ -856,31 +837,13 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, return 0; } -/** - * acpi_processor_setup_cpuidle states- prepares and configures cpuidle - * global state data i.e. idle routines - * - * @pr: the ACPI processor - */ -static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) +static int acpi_processor_setup_cstates(struct acpi_processor *pr) { int i, count = CPUIDLE_DRIVER_STATE_START; struct acpi_processor_cx *cx; struct cpuidle_state *state; struct cpuidle_driver *drv = &acpi_idle_driver; - if (!pr->flags.power_setup_done) - return -EINVAL; - - if (pr->flags.power == 0) - return -EINVAL; - - drv->safe_state_index = -1; - for (i = CPUIDLE_DRIVER_STATE_START; i < CPUIDLE_STATE_MAX; i++) { - drv->states[i].name[0] = '\0'; - drv->states[i].desc[0] = '\0'; - } - if (max_cstate == 0) max_cstate = 1; @@ -892,7 +855,7 @@ static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) state = &drv->states[count]; snprintf(state->name, CPUIDLE_NAME_LEN, "C%d", i); - strncpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); + strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); state->exit_latency = cx->latency; state->target_residency = cx->latency * latency_factor; state->enter = acpi_idle_enter; @@ -925,6 +888,450 @@ static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) return 0; } +static inline void acpi_processor_cstate_first_run_checks(void) +{ + acpi_status status; + static int first_run; + + if (first_run) + return; + dmi_check_system(processor_power_dmi_table); + max_cstate = acpi_processor_cstate_check(max_cstate); + if (max_cstate < ACPI_C_STATES_MAX) + pr_notice("ACPI: processor limited to max C-state %d\n", + max_cstate); + first_run++; + + if (acpi_gbl_FADT.cst_control && !nocst) { + status = acpi_os_write_port(acpi_gbl_FADT.smi_command, + acpi_gbl_FADT.cst_control, 8); + if (ACPI_FAILURE(status)) + ACPI_EXCEPTION((AE_INFO, status, + "Notifying BIOS of _CST ability failed")); + } +} +#else + +static inline int disabled_by_idle_boot_param(void) { return 0; } +static inline void acpi_processor_cstate_first_run_checks(void) { } +static int acpi_processor_get_cstate_info(struct acpi_processor *pr) +{ + return -ENODEV; +} + +static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, + struct cpuidle_device *dev) +{ + return -EINVAL; +} + +static int acpi_processor_setup_cstates(struct acpi_processor *pr) +{ + return -EINVAL; +} + +#endif /* CONFIG_ACPI_PROCESSOR_CSTATE */ + +struct acpi_lpi_states_array { + unsigned int size; + unsigned int composite_states_size; + struct acpi_lpi_state *entries; + struct acpi_lpi_state *composite_states[ACPI_PROCESSOR_MAX_POWER]; +}; + +static int obj_get_integer(union acpi_object *obj, u32 *value) +{ + if (obj->type != ACPI_TYPE_INTEGER) + return -EINVAL; + + *value = obj->integer.value; + return 0; +} + +static int acpi_processor_evaluate_lpi(acpi_handle handle, + struct acpi_lpi_states_array *info) +{ + acpi_status status; + int ret = 0; + int pkg_count, state_idx = 1, loop; + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *lpi_data; + struct acpi_lpi_state *lpi_state; + + status = acpi_evaluate_object(handle, "_LPI", NULL, &buffer); + if (ACPI_FAILURE(status)) { + ACPI_DEBUG_PRINT((ACPI_DB_INFO, "No _LPI, giving up\n")); + return -ENODEV; + } + + lpi_data = buffer.pointer; + + /* There must be at least 4 elements = 3 elements + 1 package */ + if (!lpi_data || lpi_data->type != ACPI_TYPE_PACKAGE || + lpi_data->package.count < 4) { + pr_debug("not enough elements in _LPI\n"); + ret = -ENODATA; + goto end; + } + + pkg_count = lpi_data->package.elements[2].integer.value; + + /* Validate number of power states. */ + if (pkg_count < 1 || pkg_count != lpi_data->package.count - 3) { + pr_debug("count given by _LPI is not valid\n"); + ret = -ENODATA; + goto end; + } + + lpi_state = kcalloc(pkg_count, sizeof(*lpi_state), GFP_KERNEL); + if (!lpi_state) { + ret = -ENOMEM; + goto end; + } + + info->size = pkg_count; + info->entries = lpi_state; + + /* LPI States start at index 3 */ + for (loop = 3; state_idx <= pkg_count; loop++, state_idx++, lpi_state++) { + union acpi_object *element, *pkg_elem, *obj; + + element = &lpi_data->package.elements[loop]; + if (element->type != ACPI_TYPE_PACKAGE || element->package.count < 7) + continue; + + pkg_elem = element->package.elements; + + obj = pkg_elem + 6; + if (obj->type == ACPI_TYPE_BUFFER) { + struct acpi_power_register *reg; + + reg = (struct acpi_power_register *)obj->buffer.pointer; + if (reg->space_id != ACPI_ADR_SPACE_SYSTEM_IO && + reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) + continue; + + lpi_state->address = reg->address; + lpi_state->entry_method = + reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE ? + ACPI_CSTATE_FFH : ACPI_CSTATE_SYSTEMIO; + } else if (obj->type == ACPI_TYPE_INTEGER) { + lpi_state->entry_method = ACPI_CSTATE_INTEGER; + lpi_state->address = obj->integer.value; + } else { + continue; + } + + /* elements[7,8] skipped for now i.e. Residency/Usage counter*/ + + obj = pkg_elem + 9; + if (obj->type == ACPI_TYPE_STRING) + strlcpy(lpi_state->desc, obj->string.pointer, + ACPI_CX_DESC_LEN); + + lpi_state->index = state_idx; + if (obj_get_integer(pkg_elem + 0, &lpi_state->min_residency)) { + pr_debug("No min. residency found, assuming 10 us\n"); + lpi_state->min_residency = 10; + } + + if (obj_get_integer(pkg_elem + 1, &lpi_state->wake_latency)) { + pr_debug("No wakeup residency found, assuming 10 us\n"); + lpi_state->wake_latency = 10; + } + + if (obj_get_integer(pkg_elem + 2, &lpi_state->flags)) + lpi_state->flags = 0; + + if (obj_get_integer(pkg_elem + 3, &lpi_state->arch_flags)) + lpi_state->arch_flags = 0; + + if (obj_get_integer(pkg_elem + 4, &lpi_state->res_cnt_freq)) + lpi_state->res_cnt_freq = 1; + + if (obj_get_integer(pkg_elem + 5, &lpi_state->enable_parent_state)) + lpi_state->enable_parent_state = 0; + } + + acpi_handle_debug(handle, "Found %d power states\n", state_idx); +end: + kfree(buffer.pointer); + return ret; +} + +/* + * flat_state_cnt - the number of composite LPI states after the process of flattening + */ +static int flat_state_cnt; + +/** + * combine_lpi_states - combine local and parent LPI states to form a composite LPI state + * + * @local: local LPI state + * @parent: parent LPI state + * @result: composite LPI state + */ +static bool combine_lpi_states(struct acpi_lpi_state *local, + struct acpi_lpi_state *parent, + struct acpi_lpi_state *result) +{ + if (parent->entry_method == ACPI_CSTATE_INTEGER) { + if (!parent->address) /* 0 means autopromotable */ + return false; + result->address = local->address + parent->address; + } else { + result->address = parent->address; + } + + result->min_residency = max(local->min_residency, parent->min_residency); + result->wake_latency = local->wake_latency + parent->wake_latency; + result->enable_parent_state = parent->enable_parent_state; + result->entry_method = local->entry_method; + + result->flags = parent->flags; + result->arch_flags = parent->arch_flags; + result->index = parent->index; + + strlcpy(result->desc, local->desc, ACPI_CX_DESC_LEN); + strlcat(result->desc, "+", ACPI_CX_DESC_LEN); + strlcat(result->desc, parent->desc, ACPI_CX_DESC_LEN); + return true; +} + +#define ACPI_LPI_STATE_FLAGS_ENABLED BIT(0) + +static void stash_composite_state(struct acpi_lpi_states_array *curr_level, + struct acpi_lpi_state *t) +{ + curr_level->composite_states[curr_level->composite_states_size++] = t; +} + +static int flatten_lpi_states(struct acpi_processor *pr, + struct acpi_lpi_states_array *curr_level, + struct acpi_lpi_states_array *prev_level) +{ + int i, j, state_count = curr_level->size; + struct acpi_lpi_state *p, *t = curr_level->entries; + + curr_level->composite_states_size = 0; + for (j = 0; j < state_count; j++, t++) { + struct acpi_lpi_state *flpi; + + if (!(t->flags & ACPI_LPI_STATE_FLAGS_ENABLED)) + continue; + + if (flat_state_cnt >= ACPI_PROCESSOR_MAX_POWER) { + pr_warn("Limiting number of LPI states to max (%d)\n", + ACPI_PROCESSOR_MAX_POWER); + pr_warn("Please increase ACPI_PROCESSOR_MAX_POWER if needed.\n"); + break; + } + + flpi = &pr->power.lpi_states[flat_state_cnt]; + + if (!prev_level) { /* leaf/processor node */ + memcpy(flpi, t, sizeof(*t)); + stash_composite_state(curr_level, flpi); + flat_state_cnt++; + continue; + } + + for (i = 0; i < prev_level->composite_states_size; i++) { + p = prev_level->composite_states[i]; + if (t->index <= p->enable_parent_state && + combine_lpi_states(p, t, flpi)) { + stash_composite_state(curr_level, flpi); + flat_state_cnt++; + flpi++; + } + } + } + + kfree(curr_level->entries); + return 0; +} + +static int acpi_processor_get_lpi_info(struct acpi_processor *pr) +{ + int ret, i; + acpi_status status; + acpi_handle handle = pr->handle, pr_ahandle; + struct acpi_device *d = NULL; + struct acpi_lpi_states_array info[2], *tmp, *prev, *curr; + + if (!osc_pc_lpi_support_confirmed) + return -EOPNOTSUPP; + + if (!acpi_has_method(handle, "_LPI")) + return -EINVAL; + + flat_state_cnt = 0; + prev = &info[0]; + curr = &info[1]; + handle = pr->handle; + ret = acpi_processor_evaluate_lpi(handle, prev); + if (ret) + return ret; + flatten_lpi_states(pr, prev, NULL); + + status = acpi_get_parent(handle, &pr_ahandle); + while (ACPI_SUCCESS(status)) { + acpi_bus_get_device(pr_ahandle, &d); + handle = pr_ahandle; + + if (strcmp(acpi_device_hid(d), ACPI_PROCESSOR_CONTAINER_HID)) + break; + + /* can be optional ? */ + if (!acpi_has_method(handle, "_LPI")) + break; + + ret = acpi_processor_evaluate_lpi(handle, curr); + if (ret) + break; + + /* flatten all the LPI states in this level of hierarchy */ + flatten_lpi_states(pr, curr, prev); + + tmp = prev, prev = curr, curr = tmp; + + status = acpi_get_parent(handle, &pr_ahandle); + } + + pr->power.count = flat_state_cnt; + /* reset the index after flattening */ + for (i = 0; i < pr->power.count; i++) + pr->power.lpi_states[i].index = i; + + /* Tell driver that _LPI is supported. */ + pr->flags.has_lpi = 1; + pr->flags.power = 1; + + return 0; +} + +int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu) +{ + return -ENODEV; +} + +int __weak acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi) +{ + return -ENODEV; +} + +/** + * acpi_idle_lpi_enter - enters an ACPI any LPI state + * @dev: the target CPU + * @drv: cpuidle driver containing cpuidle state info + * @index: index of target state + * + * Return: 0 for success or negative value for error + */ +static int acpi_idle_lpi_enter(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) +{ + struct acpi_processor *pr; + struct acpi_lpi_state *lpi; + + pr = __this_cpu_read(processors); + + if (unlikely(!pr)) + return -EINVAL; + + lpi = &pr->power.lpi_states[index]; + if (lpi->entry_method == ACPI_CSTATE_FFH) + return acpi_processor_ffh_lpi_enter(lpi); + + return -EINVAL; +} + +static int acpi_processor_setup_lpi_states(struct acpi_processor *pr) +{ + int i; + struct acpi_lpi_state *lpi; + struct cpuidle_state *state; + struct cpuidle_driver *drv = &acpi_idle_driver; + + if (!pr->flags.has_lpi) + return -EOPNOTSUPP; + + for (i = 0; i < pr->power.count && i < CPUIDLE_STATE_MAX; i++) { + lpi = &pr->power.lpi_states[i]; + + state = &drv->states[i]; + snprintf(state->name, CPUIDLE_NAME_LEN, "LPI-%d", i); + strlcpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN); + state->exit_latency = lpi->wake_latency; + state->target_residency = lpi->min_residency; + if (lpi->arch_flags) + state->flags |= CPUIDLE_FLAG_TIMER_STOP; + state->enter = acpi_idle_lpi_enter; + drv->safe_state_index = i; + } + + drv->state_count = i; + + return 0; +} + +/** + * acpi_processor_setup_cpuidle_states- prepares and configures cpuidle + * global state data i.e. idle routines + * + * @pr: the ACPI processor + */ +static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) +{ + int i; + struct cpuidle_driver *drv = &acpi_idle_driver; + + if (!pr->flags.power_setup_done || !pr->flags.power) + return -EINVAL; + + drv->safe_state_index = -1; + for (i = CPUIDLE_DRIVER_STATE_START; i < CPUIDLE_STATE_MAX; i++) { + drv->states[i].name[0] = '\0'; + drv->states[i].desc[0] = '\0'; + } + + if (pr->flags.has_lpi) + return acpi_processor_setup_lpi_states(pr); + + return acpi_processor_setup_cstates(pr); +} + +/** + * acpi_processor_setup_cpuidle_dev - prepares and configures CPUIDLE + * device i.e. per-cpu data + * + * @pr: the ACPI processor + * @dev : the cpuidle device + */ +static int acpi_processor_setup_cpuidle_dev(struct acpi_processor *pr, + struct cpuidle_device *dev) +{ + if (!pr->flags.power_setup_done || !pr->flags.power || !dev) + return -EINVAL; + + dev->cpu = pr->id; + if (pr->flags.has_lpi) + return acpi_processor_ffh_lpi_probe(pr->id); + + return acpi_processor_setup_cpuidle_cx(pr, dev); +} + +static int acpi_processor_get_power_info(struct acpi_processor *pr) +{ + int ret; + + ret = acpi_processor_get_lpi_info(pr); + if (ret) + ret = acpi_processor_get_cstate_info(pr); + + return ret; +} + int acpi_processor_hotplug(struct acpi_processor *pr) { int ret = 0; @@ -933,18 +1340,15 @@ int acpi_processor_hotplug(struct acpi_processor *pr) if (disabled_by_idle_boot_param()) return 0; - if (nocst) - return -ENODEV; - if (!pr->flags.power_setup_done) return -ENODEV; dev = per_cpu(acpi_cpuidle_device, pr->id); cpuidle_pause_and_lock(); cpuidle_disable_device(dev); - acpi_processor_get_power_info(pr); - if (pr->flags.power) { - acpi_processor_setup_cpuidle_cx(pr, dev); + ret = acpi_processor_get_power_info(pr); + if (!ret && pr->flags.power) { + acpi_processor_setup_cpuidle_dev(pr, dev); ret = cpuidle_enable_device(dev); } cpuidle_resume_and_unlock(); @@ -952,7 +1356,7 @@ int acpi_processor_hotplug(struct acpi_processor *pr) return ret; } -int acpi_processor_cst_has_changed(struct acpi_processor *pr) +int acpi_processor_power_state_has_changed(struct acpi_processor *pr) { int cpu; struct acpi_processor *_pr; @@ -961,9 +1365,6 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr) if (disabled_by_idle_boot_param()) return 0; - if (nocst) - return -ENODEV; - if (!pr->flags.power_setup_done) return -ENODEV; @@ -1000,7 +1401,7 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr) acpi_processor_get_power_info(_pr); if (_pr->flags.power) { dev = per_cpu(acpi_cpuidle_device, cpu); - acpi_processor_setup_cpuidle_cx(_pr, dev); + acpi_processor_setup_cpuidle_dev(_pr, dev); cpuidle_enable_device(dev); } } @@ -1015,35 +1416,16 @@ static int acpi_processor_registered; int acpi_processor_power_init(struct acpi_processor *pr) { - acpi_status status; int retval; struct cpuidle_device *dev; - static int first_run; if (disabled_by_idle_boot_param()) return 0; - if (!first_run) { - dmi_check_system(processor_power_dmi_table); - max_cstate = acpi_processor_cstate_check(max_cstate); - if (max_cstate < ACPI_C_STATES_MAX) - printk(KERN_NOTICE - "ACPI: processor limited to max C-state %d\n", - max_cstate); - first_run++; - } - - if (acpi_gbl_FADT.cst_control && !nocst) { - status = - acpi_os_write_port(acpi_gbl_FADT.smi_command, acpi_gbl_FADT.cst_control, 8); - if (ACPI_FAILURE(status)) { - ACPI_EXCEPTION((AE_INFO, status, - "Notifying BIOS of _CST ability failed")); - } - } + acpi_processor_cstate_first_run_checks(); - acpi_processor_get_power_info(pr); - pr->flags.power_setup_done = 1; + if (!acpi_processor_get_power_info(pr)) + pr->flags.power_setup_done = 1; /* * Install the idle handler if processor power management is supported. @@ -1066,7 +1448,7 @@ int acpi_processor_power_init(struct acpi_processor *pr) return -ENOMEM; per_cpu(acpi_cpuidle_device, pr->id) = dev; - acpi_processor_setup_cpuidle_cx(pr, dev); + acpi_processor_setup_cpuidle_dev(pr, dev); /* Register per-cpu cpuidle_device. Cpuidle driver * must already be registered before registering device diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 5f28cf778349..ad9fc84a8601 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -46,6 +46,13 @@ DEFINE_MUTEX(acpi_device_lock); LIST_HEAD(acpi_wakeup_device_list); static DEFINE_MUTEX(acpi_hp_context_lock); +/* + * The UART device described by the SPCR table is the only object which needs + * special-casing. Everything else is covered by ACPI namespace paths in STAO + * table. + */ +static u64 spcr_uart_addr; + struct acpi_dep_data { struct list_head node; acpi_handle master; @@ -494,6 +501,8 @@ static void acpi_device_del(struct acpi_device *device) device_del(&device->dev); } +static BLOCKING_NOTIFIER_HEAD(acpi_reconfig_chain); + static LIST_HEAD(acpi_device_del_list); static DEFINE_MUTEX(acpi_device_del_lock); @@ -514,6 +523,9 @@ static void acpi_device_del_work_fn(struct work_struct *work_not_used) mutex_unlock(&acpi_device_del_lock); + blocking_notifier_call_chain(&acpi_reconfig_chain, + ACPI_RECONFIG_DEVICE_REMOVE, adev); + acpi_device_del(adev); /* * Drop references to all power resources that might have been @@ -1406,7 +1418,7 @@ void acpi_init_device_object(struct acpi_device *device, acpi_handle handle, acpi_bus_get_flags(device); device->flags.match_driver = false; device->flags.initialized = true; - device->flags.visited = false; + acpi_device_clear_enumerated(device); device_initialize(&device->dev); dev_set_uevent_suppress(&device->dev, true); acpi_init_coherency(device); @@ -1453,6 +1465,41 @@ static int acpi_add_single_object(struct acpi_device **child, return 0; } +static acpi_status acpi_get_resource_memory(struct acpi_resource *ares, + void *context) +{ + struct resource *res = context; + + if (acpi_dev_resource_memory(ares, res)) + return AE_CTRL_TERMINATE; + + return AE_OK; +} + +static bool acpi_device_should_be_hidden(acpi_handle handle) +{ + acpi_status status; + struct resource res; + + /* Check if it should ignore the UART device */ + if (!(spcr_uart_addr && acpi_has_method(handle, METHOD_NAME__CRS))) + return false; + + /* + * The UART device described in SPCR table is assumed to have only one + * memory resource present. So we only look for the first one here. + */ + status = acpi_walk_resources(handle, METHOD_NAME__CRS, + acpi_get_resource_memory, &res); + if (ACPI_FAILURE(status) || res.start != spcr_uart_addr) + return false; + + acpi_handle_info(handle, "The UART device @%pa in SPCR table will be hidden\n", + &res.start); + + return true; +} + static int acpi_bus_type_and_status(acpi_handle handle, int *type, unsigned long long *sta) { @@ -1466,6 +1513,9 @@ static int acpi_bus_type_and_status(acpi_handle handle, int *type, switch (acpi_type) { case ACPI_TYPE_ANY: /* for ACPI_ROOT_OBJECT */ case ACPI_TYPE_DEVICE: + if (acpi_device_should_be_hidden(handle)) + return -ENODEV; + *type = ACPI_BUS_TYPE_DEVICE; status = acpi_bus_get_status_handle(handle, sta); if (ACPI_FAILURE(status)) @@ -1676,15 +1726,20 @@ static void acpi_default_enumeration(struct acpi_device *device) bool is_spi_i2c_slave = false; /* - * Do not enemerate SPI/I2C slaves as they will be enuerated by their + * Do not enumerate SPI/I2C slaves as they will be enumerated by their * respective parents. */ INIT_LIST_HEAD(&resource_list); acpi_dev_get_resources(device, &resource_list, acpi_check_spi_i2c_slave, &is_spi_i2c_slave); acpi_dev_free_resource_list(&resource_list); - if (!is_spi_i2c_slave) + if (!is_spi_i2c_slave) { acpi_create_platform_device(device); + acpi_device_set_enumerated(device); + } else { + blocking_notifier_call_chain(&acpi_reconfig_chain, + ACPI_RECONFIG_DEVICE_ADD, device); + } } static const struct acpi_device_id generic_device_ids[] = { @@ -1751,7 +1806,7 @@ static void acpi_bus_attach(struct acpi_device *device) acpi_bus_get_status(device); /* Skip devices that are not present. */ if (!acpi_device_is_present(device)) { - device->flags.visited = false; + acpi_device_clear_enumerated(device); device->flags.power_manageable = 0; return; } @@ -1766,7 +1821,7 @@ static void acpi_bus_attach(struct acpi_device *device) device->flags.initialized = true; } - device->flags.visited = false; + ret = acpi_scan_attach_handler(device); if (ret < 0) return; @@ -1780,7 +1835,6 @@ static void acpi_bus_attach(struct acpi_device *device) if (!ret && device->pnp.type.platform_id) acpi_default_enumeration(device); } - device->flags.visited = true; ok: list_for_each_entry(child, &device->children, node) @@ -1872,7 +1926,7 @@ void acpi_bus_trim(struct acpi_device *adev) */ acpi_device_set_power(adev, ACPI_STATE_D3_COLD); adev->flags.initialized = false; - adev->flags.visited = false; + acpi_device_clear_enumerated(adev); } EXPORT_SYMBOL_GPL(acpi_bus_trim); @@ -1916,9 +1970,26 @@ static int acpi_bus_scan_fixed(void) return result < 0 ? result : 0; } +static void __init acpi_get_spcr_uart_addr(void) +{ + acpi_status status; + struct acpi_table_spcr *spcr_ptr; + + status = acpi_get_table(ACPI_SIG_SPCR, 0, + (struct acpi_table_header **)&spcr_ptr); + if (ACPI_SUCCESS(status)) + spcr_uart_addr = spcr_ptr->serial_port.address; + else + printk(KERN_WARNING PREFIX "STAO table present, but SPCR is missing\n"); +} + +static bool acpi_scan_initialized; + int __init acpi_scan_init(void) { int result; + acpi_status status; + struct acpi_table_stao *stao_ptr; acpi_pci_root_init(); acpi_pci_link_init(); @@ -1934,6 +2005,20 @@ int __init acpi_scan_init(void) acpi_scan_add_handler(&generic_device_handler); + /* + * If there is STAO table, check whether it needs to ignore the UART + * device in SPCR table. + */ + status = acpi_get_table(ACPI_SIG_STAO, 0, + (struct acpi_table_header **)&stao_ptr); + if (ACPI_SUCCESS(status)) { + if (stao_ptr->header.length > sizeof(struct acpi_table_stao)) + printk(KERN_INFO PREFIX "STAO Name List not yet supported."); + + if (stao_ptr->ignore_uart) + acpi_get_spcr_uart_addr(); + } + mutex_lock(&acpi_scan_lock); /* * Enumerate devices in the ACPI namespace. @@ -1960,6 +2045,8 @@ int __init acpi_scan_init(void) acpi_update_all_gpes(); + acpi_scan_initialized = true; + out: mutex_unlock(&acpi_scan_lock); return result; @@ -2003,3 +2090,57 @@ int __init __acpi_probe_device_table(struct acpi_probe_entry *ap_head, int nr) return count; } + +struct acpi_table_events_work { + struct work_struct work; + void *table; + u32 event; +}; + +static void acpi_table_events_fn(struct work_struct *work) +{ + struct acpi_table_events_work *tew; + + tew = container_of(work, struct acpi_table_events_work, work); + + if (tew->event == ACPI_TABLE_EVENT_LOAD) { + acpi_scan_lock_acquire(); + acpi_bus_scan(ACPI_ROOT_OBJECT); + acpi_scan_lock_release(); + } + + kfree(tew); +} + +void acpi_scan_table_handler(u32 event, void *table, void *context) +{ + struct acpi_table_events_work *tew; + + if (!acpi_scan_initialized) + return; + + if (event != ACPI_TABLE_EVENT_LOAD) + return; + + tew = kmalloc(sizeof(*tew), GFP_KERNEL); + if (!tew) + return; + + INIT_WORK(&tew->work, acpi_table_events_fn); + tew->table = table; + tew->event = event; + + schedule_work(&tew->work); +} + +int acpi_reconfig_notifier_register(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&acpi_reconfig_chain, nb); +} +EXPORT_SYMBOL(acpi_reconfig_notifier_register); + +int acpi_reconfig_notifier_unregister(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&acpi_reconfig_chain, nb); +} +EXPORT_SYMBOL(acpi_reconfig_notifier_unregister); diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index 7a2e4d45b266..2b38c1bb0446 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -47,15 +47,32 @@ static void acpi_sleep_tts_switch(u32 acpi_state) } } -static int tts_notify_reboot(struct notifier_block *this, +static void acpi_sleep_pts_switch(u32 acpi_state) +{ + acpi_status status; + + status = acpi_execute_simple_method(NULL, "\\_PTS", acpi_state); + if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) { + /* + * OS can't evaluate the _PTS object correctly. Some warning + * message will be printed. But it won't break anything. + */ + printk(KERN_NOTICE "Failure in evaluating _PTS object\n"); + } +} + +static int sleep_notify_reboot(struct notifier_block *this, unsigned long code, void *x) { acpi_sleep_tts_switch(ACPI_STATE_S5); + + acpi_sleep_pts_switch(ACPI_STATE_S5); + return NOTIFY_DONE; } -static struct notifier_block tts_notifier = { - .notifier_call = tts_notify_reboot, +static struct notifier_block sleep_notifier = { + .notifier_call = sleep_notify_reboot, .next = NULL, .priority = 0, }; @@ -899,9 +916,9 @@ int __init acpi_sleep_init(void) pr_info(PREFIX "(supports%s)\n", supported); /* - * Register the tts_notifier to reboot notifier list so that the _TTS - * object can also be evaluated when the system enters S5. + * Register the sleep_notifier to reboot notifier list so that the _TTS + * and _PTS object can also be evaluated when the system enters S5. */ - register_reboot_notifier(&tts_notifier); + register_reboot_notifier(&sleep_notifier); return 0; } diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c index 4b3a9e27f1b6..358165e9f5b8 100644 --- a/drivers/acpi/sysfs.c +++ b/drivers/acpi/sysfs.c @@ -378,8 +378,7 @@ static void acpi_table_attr_init(struct acpi_table_attr *table_attr, return; } -static acpi_status -acpi_sysfs_table_handler(u32 event, void *table, void *context) +acpi_status acpi_sysfs_table_handler(u32 event, void *table, void *context) { struct acpi_table_attr *table_attr; @@ -452,9 +451,8 @@ static int acpi_tables_sysfs_init(void) kobject_uevent(tables_kobj, KOBJ_ADD); kobject_uevent(dynamic_tables_kobj, KOBJ_ADD); - status = acpi_install_table_handler(acpi_sysfs_table_handler, NULL); - return ACPI_FAILURE(status) ? -EINVAL : 0; + return 0; err_dynamic_tables: kobject_put(tables_kobj); err: diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index a372f9eaa15d..9f0ad6ebb368 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -34,6 +34,8 @@ #include #include #include +#include +#include #include "internal.h" #ifdef CONFIG_ACPI_CUSTOM_DSDT @@ -481,8 +483,10 @@ static DECLARE_BITMAP(acpi_initrd_installed, NR_ACPI_INITRD_TABLES); #define MAP_CHUNK_SIZE (NR_FIX_BTMAPS << PAGE_SHIFT) -static void __init acpi_table_initrd_init(void *data, size_t size) +void __init acpi_table_upgrade(void) { + void *data = (void *)initrd_start; + size_t size = initrd_end - initrd_start; int sig, no, table_nr = 0, total_offset = 0; long offset = 0; struct acpi_table_header *table; @@ -540,7 +544,7 @@ static void __init acpi_table_initrd_init(void *data, size_t size) return; acpi_tables_addr = - memblock_find_in_range(0, max_low_pfn_mapped << PAGE_SHIFT, + memblock_find_in_range(0, ACPI_TABLE_UPGRADE_MAX_PHYS, all_tables_size, PAGE_SIZE); if (!acpi_tables_addr) { WARN_ON(1); @@ -578,10 +582,10 @@ static void __init acpi_table_initrd_init(void *data, size_t size) clen = size; if (clen > MAP_CHUNK_SIZE - slop) clen = MAP_CHUNK_SIZE - slop; - dest_p = early_ioremap(dest_addr & PAGE_MASK, - clen + slop); + dest_p = early_memremap(dest_addr & PAGE_MASK, + clen + slop); memcpy(dest_p + slop, src_p, clen); - early_iounmap(dest_p, clen + slop); + early_memunmap(dest_p, clen + slop); src_p += clen; dest_addr += clen; size -= clen; @@ -696,10 +700,6 @@ next_table: } } #else -static void __init acpi_table_initrd_init(void *data, size_t size) -{ -} - static acpi_status acpi_table_initrd_override(struct acpi_table_header *existing_table, acpi_physical_address *address, @@ -742,11 +742,6 @@ acpi_os_table_override(struct acpi_table_header *existing_table, return AE_OK; } -void __init early_acpi_table_init(void *data, size_t size) -{ - acpi_table_initrd_init(data, size); -} - /* * acpi_table_init() * diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 82707f9824ca..f4ebe39539af 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -1259,7 +1259,8 @@ static int __init acpi_thermal_init(void) return -ENODEV; } - acpi_thermal_pm_queue = create_workqueue("acpi_thermal_pm"); + acpi_thermal_pm_queue = alloc_workqueue("acpi_thermal_pm", + WQ_HIGHPRI | WQ_MEM_RECLAIM, 0); if (!acpi_thermal_pm_queue) return -ENODEV; diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 3d1327615f72..a6b36fc53aec 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -167,6 +167,14 @@ static const struct dmi_system_id video_detect_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad X201s"), }, }, + { + .callback = video_detect_force_video, + .ident = "ThinkPad X201T", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad X201T"), + }, + }, /* The native backlight controls do not work on some older machines */ { diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index e2dc4c045146..2c8be74f401d 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -98,12 +98,12 @@ config SATA_AHCI_PLATFORM If unsure, say N. -config AHCI_BRCMSTB - tristate "Broadcom STB AHCI SATA support" - depends on ARCH_BRCMSTB || BMIPS_GENERIC +config AHCI_BRCM + tristate "Broadcom AHCI SATA support" + depends on ARCH_BRCMSTB || BMIPS_GENERIC || ARCH_BCM_NSP help This option enables support for the AHCI SATA3 controller found on - STB SoC's. + Broadcom SoC's. If unsure, say N. diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile index 0b2afb7e5f35..a46e6b784bda 100644 --- a/drivers/ata/Makefile +++ b/drivers/ata/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_SATA_INIC162X) += sata_inic162x.o obj-$(CONFIG_SATA_SIL24) += sata_sil24.o obj-$(CONFIG_SATA_DWC) += sata_dwc_460ex.o obj-$(CONFIG_SATA_HIGHBANK) += sata_highbank.o libahci.o -obj-$(CONFIG_AHCI_BRCMSTB) += ahci_brcmstb.o libahci.o libahci_platform.o +obj-$(CONFIG_AHCI_BRCM) += ahci_brcm.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_CEVA) += ahci_ceva.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_DA850) += ahci_da850.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_IMX) += ahci_imx.o libahci.o libahci_platform.o diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index a83bbcc58b4c..90eabaf81215 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -580,7 +580,7 @@ static struct pci_driver ahci_pci_driver = { }, }; -#if defined(CONFIG_PATA_MARVELL) || defined(CONFIG_PATA_MARVELL_MODULE) +#if IS_ENABLED(CONFIG_PATA_MARVELL) static int marvell_enable; #else static int marvell_enable = 1; diff --git a/drivers/ata/ahci_brcmstb.c b/drivers/ata/ahci_brcm.c similarity index 91% rename from drivers/ata/ahci_brcmstb.c rename to drivers/ata/ahci_brcm.c index e87bcec0fd7c..6f8a7341fa08 100644 --- a/drivers/ata/ahci_brcmstb.c +++ b/drivers/ata/ahci_brcm.c @@ -71,6 +71,12 @@ (DATA_ENDIAN << DMADESC_ENDIAN_SHIFT) | \ (MMIO_ENDIAN << MMIO_ENDIAN_SHIFT)) +enum brcm_ahci_version { + BRCM_SATA_BCM7425 = 1, + BRCM_SATA_BCM7445, + BRCM_SATA_NSP, +}; + enum brcm_ahci_quirks { BRCM_AHCI_QUIRK_NO_NCQ = BIT(0), BRCM_AHCI_QUIRK_SKIP_PHY_ENABLE = BIT(1), @@ -81,6 +87,7 @@ struct brcm_ahci_priv { void __iomem *top_ctrl; u32 port_mask; u32 quirks; + enum brcm_ahci_version version; }; static const struct ata_port_info ahci_brcm_port_info = { @@ -247,9 +254,19 @@ static u32 brcm_ahci_get_portmask(struct platform_device *pdev, static void brcm_sata_init(struct brcm_ahci_priv *priv) { + void __iomem *ctrl = priv->top_ctrl + SATA_TOP_CTRL_BUS_CTRL; + /* Configure endianness */ - brcm_sata_writereg(BUS_CTRL_ENDIAN_CONF, - priv->top_ctrl + SATA_TOP_CTRL_BUS_CTRL); + if (priv->version == BRCM_SATA_NSP) { + u32 data = brcm_sata_readreg(ctrl); + + data &= ~((0x03 << DMADATA_ENDIAN_SHIFT) | + (0x03 << DMADESC_ENDIAN_SHIFT)); + data |= (0x02 << DMADATA_ENDIAN_SHIFT) | + (0x02 << DMADESC_ENDIAN_SHIFT); + brcm_sata_writereg(data, ctrl); + } else + brcm_sata_writereg(BUS_CTRL_ENDIAN_CONF, ctrl); } #ifdef CONFIG_PM_SLEEP @@ -282,8 +299,17 @@ static struct scsi_host_template ahci_platform_sht = { AHCI_SHT(DRV_NAME), }; +static const struct of_device_id ahci_of_match[] = { + {.compatible = "brcm,bcm7425-ahci", .data = (void *)BRCM_SATA_BCM7425}, + {.compatible = "brcm,bcm7445-ahci", .data = (void *)BRCM_SATA_BCM7445}, + {.compatible = "brcm,bcm-nsp-ahci", .data = (void *)BRCM_SATA_NSP}, + {}, +}; +MODULE_DEVICE_TABLE(of, ahci_of_match); + static int brcm_ahci_probe(struct platform_device *pdev) { + const struct of_device_id *of_id; struct device *dev = &pdev->dev; struct brcm_ahci_priv *priv; struct ahci_host_priv *hpriv; @@ -293,6 +319,12 @@ static int brcm_ahci_probe(struct platform_device *pdev) priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; + + of_id = of_match_node(ahci_of_match, pdev->dev.of_node); + if (!of_id) + return -ENODEV; + + priv->version = (enum brcm_ahci_version)of_id->data; priv->dev = dev; res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "top-ctrl"); @@ -300,7 +332,8 @@ static int brcm_ahci_probe(struct platform_device *pdev) if (IS_ERR(priv->top_ctrl)) return PTR_ERR(priv->top_ctrl); - if (of_device_is_compatible(dev->of_node, "brcm,bcm7425-ahci")) { + if ((priv->version == BRCM_SATA_BCM7425) || + (priv->version == BRCM_SATA_NSP)) { priv->quirks |= BRCM_AHCI_QUIRK_NO_NCQ; priv->quirks |= BRCM_AHCI_QUIRK_SKIP_PHY_ENABLE; } @@ -354,13 +387,6 @@ static int brcm_ahci_remove(struct platform_device *pdev) return 0; } -static const struct of_device_id ahci_of_match[] = { - {.compatible = "brcm,bcm7425-ahci"}, - {.compatible = "brcm,bcm7445-ahci"}, - {}, -}; -MODULE_DEVICE_TABLE(of, ahci_of_match); - static SIMPLE_DEV_PM_OPS(ahci_brcm_pm_ops, brcm_ahci_suspend, brcm_ahci_resume); static struct platform_driver brcm_ahci_driver = { diff --git a/drivers/ata/ahci_seattle.c b/drivers/ata/ahci_seattle.c index 6e702ab57220..1d31c0c0fc20 100644 --- a/drivers/ata/ahci_seattle.c +++ b/drivers/ata/ahci_seattle.c @@ -137,7 +137,7 @@ static const struct ata_port_info *ahci_seattle_get_port_info( u32 val; plat_data = devm_kzalloc(dev, sizeof(*plat_data), GFP_KERNEL); - if (IS_ERR(plat_data)) + if (!plat_data) return &ahci_port_info; plat_data->sgpio_ctrl = devm_ioremap_resource(dev, diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index 71b07198e207..7461a587b39b 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -1975,7 +1975,7 @@ unsigned int ahci_qc_issue(struct ata_queued_cmd *qc) */ pp->active_link = qc->dev->link; - if (qc->tf.protocol == ATA_PROT_NCQ) + if (ata_is_ncq(qc->tf.protocol)) writel(1 << qc->tag, port_mmio + PORT_SCR_ACT); if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) { @@ -2392,12 +2392,20 @@ static int ahci_port_start(struct ata_port *ap) static void ahci_port_stop(struct ata_port *ap) { const char *emsg = NULL; + struct ahci_host_priv *hpriv = ap->host->private_data; + void __iomem *host_mmio = hpriv->mmio; int rc; /* de-initialize port */ rc = ahci_deinit_port(ap, &emsg); if (rc) ata_port_warn(ap, "%s (%d)\n", emsg, rc); + + /* + * Clear GHC.IS to prevent stuck INTx after disabling MSI and + * re-enabling INTx. + */ + writel(1 << ap->port_no, host_mmio + HOST_IRQ_STAT); } void ahci_print_info(struct ata_host *host, const char *scc_s) diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 2eca572f4df6..223a770f78f3 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -1239,7 +1239,7 @@ static int ata_read_native_max_address(struct ata_device *dev, u64 *max_sectors) } else tf.command = ATA_CMD_READ_NATIVE_MAX; - tf.protocol |= ATA_PROT_NODATA; + tf.protocol = ATA_PROT_NODATA; tf.device |= ATA_LBA; err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0); @@ -1298,7 +1298,7 @@ static int ata_set_max_sectors(struct ata_device *dev, u64 new_sectors) tf.device |= (new_sectors >> 24) & 0xf; } - tf.protocol |= ATA_PROT_NODATA; + tf.protocol = ATA_PROT_NODATA; tf.device |= ATA_LBA; tf.lbal = (new_sectors >> 0) & 0xff; @@ -4315,6 +4315,12 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = { */ { "ST380013AS", "3.20", ATA_HORKAGE_MAX_SEC_1024 }, + /* + * Device times out with higher max sects. + * https://bugzilla.kernel.org/show_bug.cgi?id=121671 + */ + { "LITEON CX1-JB256-HP", NULL, ATA_HORKAGE_MAX_SEC_1024 }, + /* Devices we expect to fail diagnostics */ /* Devices where NCQ should be avoided */ @@ -4843,7 +4849,7 @@ int ata_std_qc_defer(struct ata_queued_cmd *qc) { struct ata_link *link = qc->dev->link; - if (qc->tf.protocol == ATA_PROT_NCQ) { + if (ata_is_ncq(qc->tf.protocol)) { if (!ata_tag_valid(link->active_tag)) return 0; } else { @@ -5008,7 +5014,7 @@ void __ata_qc_complete(struct ata_queued_cmd *qc) ata_sg_clean(qc); /* command should be marked inactive atomically with qc completion */ - if (qc->tf.protocol == ATA_PROT_NCQ) { + if (ata_is_ncq(qc->tf.protocol)) { link->sactive &= ~(1 << qc->tag); if (!link->sactive) ap->nr_active_links--; @@ -5045,7 +5051,7 @@ static void ata_verify_xfer(struct ata_queued_cmd *qc) { struct ata_device *dev = qc->dev; - if (ata_is_nodata(qc->tf.protocol)) + if (!ata_is_data(qc->tf.protocol)) return; if ((dev->mwdma_mask || dev->udma_mask) && ata_is_pio(qc->tf.protocol)) @@ -5131,7 +5137,9 @@ void ata_qc_complete(struct ata_queued_cmd *qc) switch (qc->tf.command) { case ATA_CMD_SET_FEATURES: if (qc->tf.feature != SETFEATURES_WC_ON && - qc->tf.feature != SETFEATURES_WC_OFF) + qc->tf.feature != SETFEATURES_WC_OFF && + qc->tf.feature != SETFEATURES_RA_ON && + qc->tf.feature != SETFEATURES_RA_OFF) break; /* fall through */ case ATA_CMD_INIT_DEV_PARAMS: /* CHS translation changed */ diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index 61dc7a99e89a..0e1ec37070d1 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -606,7 +606,7 @@ void ata_scsi_error(struct Scsi_Host *host) ata_scsi_port_error_handler(host, ap); /* finish or retry handled scmd's and clean up */ - WARN_ON(host->host_failed || !list_empty(&eh_work_q)); + WARN_ON(!list_empty(&eh_work_q)); DPRINTK("EXIT\n"); } @@ -2607,9 +2607,13 @@ static void ata_eh_link_report(struct ata_link *link) [DMA_FROM_DEVICE] = "in", }; static const char *prot_str[] = { + [ATA_PROT_UNKNOWN] = "unknown", + [ATA_PROT_NODATA] = "nodata", [ATA_PROT_PIO] = "pio", [ATA_PROT_DMA] = "dma", - [ATA_PROT_NCQ] = "ncq", + [ATA_PROT_NCQ] = "ncq dma", + [ATA_PROT_NCQ_NODATA] = "ncq nodata", + [ATAPI_PROT_NODATA] = "nodata", [ATAPI_PROT_PIO] = "pio", [ATAPI_PROT_DMA] = "dma", }; @@ -3177,7 +3181,7 @@ static void ata_eh_park_issue_cmd(struct ata_device *dev, int park) } tf.flags |= ATA_TFLAG_DEVICE | ATA_TFLAG_ISADDR; - tf.protocol |= ATA_PROT_NODATA; + tf.protocol = ATA_PROT_NODATA; err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0); if (park && (err_mask || tf.lbal != 0xc4)) { ata_dev_err(dev, "head unload failed!\n"); diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index bfec66fb26e2..e207b33e4ce9 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -304,7 +304,7 @@ static void ata_scsi_set_invalid_field(struct ata_device *dev, struct scsi_cmnd *cmd, u16 field, u8 bit) { ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, 0x24, 0x0); - /* "Invalid field in cbd" */ + /* "Invalid field in CDB" */ scsi_set_sense_field_pointer(cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE, field, bit, 1); } @@ -1190,7 +1190,7 @@ static int atapi_drain_needed(struct request *rq) if (likely(rq->cmd_type != REQ_TYPE_BLOCK_PC)) return 0; - if (!blk_rq_bytes(rq) || (rq->cmd_flags & REQ_WRITE)) + if (!blk_rq_bytes(rq) || op_is_write(req_op(rq))) return 0; return atapi_cmd_type(rq->cmd[0]) == ATAPI_MISC; @@ -2075,8 +2075,8 @@ static unsigned int ata_scsiop_inq_std(struct ata_scsi_args *args, u8 *rbuf) 0x03, 0x20, /* SBC-2 (no version claimed) */ - 0x02, - 0x60 /* SPC-3 (no version claimed) */ + 0x03, + 0x00 /* SPC-3 (no version claimed) */ }; const u8 versions_zbc[] = { 0x00, @@ -2097,7 +2097,10 @@ static unsigned int ata_scsiop_inq_std(struct ata_scsi_args *args, u8 *rbuf) 0, 0x5, /* claim SPC-3 version compatibility */ 2, - 95 - 4 + 95 - 4, + 0, + 0, + 2 }; VPRINTK("ENTER\n"); @@ -2109,8 +2112,10 @@ static unsigned int ata_scsiop_inq_std(struct ata_scsi_args *args, u8 *rbuf) (args->dev->link->ap->pflags & ATA_PFLAG_EXTERNAL)) hdr[1] |= (1 << 7); - if (args->dev->class == ATA_DEV_ZAC) + if (args->dev->class == ATA_DEV_ZAC) { hdr[0] = TYPE_ZBC; + hdr[2] = 0x7; /* claim SPC-5 version compatibility */ + } memcpy(rbuf, hdr, sizeof(hdr)); memcpy(&rbuf[8], "ATA ", 8); @@ -2314,7 +2319,7 @@ static unsigned int ata_scsiop_inq_b0(struct ata_scsi_args *args, u8 *rbuf) * with the unmap bit set. */ if (ata_id_has_trim(args->id)) { - put_unaligned_be64(65535 * 512 / 8, &rbuf[36]); + put_unaligned_be64(65535 * ATA_MAX_TRIM_RNUM, &rbuf[36]); put_unaligned_be32(1, &rbuf[28]); } @@ -2424,15 +2429,17 @@ static void modecpy(u8 *dest, const u8 *src, int n, bool changeable) static unsigned int ata_msense_caching(u16 *id, u8 *buf, bool changeable) { modecpy(buf, def_cache_mpage, sizeof(def_cache_mpage), changeable); - if (changeable || ata_id_wcache_enabled(id)) - buf[2] |= (1 << 2); /* write cache enable */ - if (!changeable && !ata_id_rahead_enabled(id)) - buf[12] |= (1 << 5); /* disable read ahead */ + if (changeable) { + buf[2] |= (1 << 2); /* ata_mselect_caching() */ + } else { + buf[2] |= (ata_id_wcache_enabled(id) << 2); /* write cache enable */ + buf[12] |= (!ata_id_rahead_enabled(id) << 5); /* disable read ahead */ + } return sizeof(def_cache_mpage); } /** - * ata_msense_ctl_mode - Simulate MODE SENSE control mode page + * ata_msense_control - Simulate MODE SENSE control mode page * @dev: ATA device of interest * @buf: output buffer * @changeable: whether changeable parameters are requested @@ -2442,12 +2449,17 @@ static unsigned int ata_msense_caching(u16 *id, u8 *buf, bool changeable) * LOCKING: * None. */ -static unsigned int ata_msense_ctl_mode(struct ata_device *dev, u8 *buf, +static unsigned int ata_msense_control(struct ata_device *dev, u8 *buf, bool changeable) { modecpy(buf, def_control_mpage, sizeof(def_control_mpage), changeable); - if (changeable && (dev->flags & ATA_DFLAG_D_SENSE)) - buf[2] |= (1 << 2); /* Descriptor sense requested */ + if (changeable) { + buf[2] |= (1 << 2); /* ata_mselect_control() */ + } else { + bool d_sense = (dev->flags & ATA_DFLAG_D_SENSE); + + buf[2] |= (d_sense << 2); /* descriptor format sense data */ + } return sizeof(def_control_mpage); } @@ -2566,13 +2578,13 @@ static unsigned int ata_scsiop_mode_sense(struct ata_scsi_args *args, u8 *rbuf) break; case CONTROL_MPAGE: - p += ata_msense_ctl_mode(args->dev, p, page_control == 1); + p += ata_msense_control(args->dev, p, page_control == 1); break; case ALL_MPAGES: p += ata_msense_rw_recovery(p, page_control == 1); p += ata_msense_caching(args->id, p, page_control == 1); - p += ata_msense_ctl_mode(args->dev, p, page_control == 1); + p += ata_msense_control(args->dev, p, page_control == 1); break; default: /* invalid page code */ @@ -3077,6 +3089,9 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc) goto invalid_fld; } + if (ata_is_ncq(tf->protocol) && (cdb[2] & 0x3) == 0) + tf->protocol = ATA_PROT_NCQ_NODATA; + /* enable LBA */ tf->flags |= ATA_TFLAG_LBA; @@ -3125,8 +3140,8 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc) tf->command = cdb[9]; } - /* For NCQ commands with FPDMA protocol, copy the tag value */ - if (tf->protocol == ATA_PROT_NCQ) + /* For NCQ commands copy the tag value */ + if (ata_is_ncq(tf->protocol)) tf->nsect = qc->tag << 3; /* enforce correct master/slave bit */ @@ -3305,7 +3320,13 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc) goto invalid_param_len; buf = page_address(sg_page(scsi_sglist(scmd))); - size = ata_set_lba_range_entries(buf, 512, block, n_block); + + if (n_block <= 65535 * ATA_MAX_TRIM_RNUM) { + size = ata_set_lba_range_entries(buf, ATA_MAX_TRIM_RNUM, block, n_block); + } else { + fp = 2; + goto invalid_fld; + } if (ata_ncq_enabled(dev) && ata_fpdma_dsm_supported(dev)) { /* Newer devices support queued TRIM commands */ @@ -3454,7 +3475,7 @@ static unsigned int ata_scsi_zbc_in_xlat(struct ata_queued_cmd *qc) goto invalid_param_len; } sect = n_block / 512; - options = cdb[14]; + options = cdb[14] & 0xbf; if (ata_ncq_enabled(qc->dev) && ata_fpdma_zac_mgmt_in_supported(qc->dev)) { @@ -3464,7 +3485,7 @@ static unsigned int ata_scsi_zbc_in_xlat(struct ata_queued_cmd *qc) tf->nsect = qc->tag << 3; tf->feature = sect & 0xff; tf->hob_feature = (sect >> 8) & 0xff; - tf->auxiliary = ATA_SUBCMD_ZAC_MGMT_IN_REPORT_ZONES; + tf->auxiliary = ATA_SUBCMD_ZAC_MGMT_IN_REPORT_ZONES | (options << 8); } else { tf->command = ATA_CMD_ZAC_MGMT_IN; tf->feature = ATA_SUBCMD_ZAC_MGMT_IN_REPORT_ZONES; @@ -3506,7 +3527,7 @@ static unsigned int ata_scsi_zbc_out_xlat(struct ata_queued_cmd *qc) struct scsi_cmnd *scmd = qc->scsicmd; struct ata_device *dev = qc->dev; const u8 *cdb = scmd->cmnd; - u8 reset_all, sa; + u8 all, sa; u64 block; u32 n_block; u16 fp = (u16)-1; @@ -3533,20 +3554,20 @@ static unsigned int ata_scsi_zbc_out_xlat(struct ata_queued_cmd *qc) if (block > dev->n_sectors) goto out_of_range; - reset_all = cdb[14] & 0x1; + all = cdb[14] & 0x1; if (ata_ncq_enabled(qc->dev) && ata_fpdma_zac_mgmt_out_supported(qc->dev)) { - tf->protocol = ATA_PROT_NCQ; + tf->protocol = ATA_PROT_NCQ_NODATA; tf->command = ATA_CMD_NCQ_NON_DATA; - tf->hob_nsect = ATA_SUBCMD_NCQ_NON_DATA_ZAC_MGMT_OUT; + tf->feature = ATA_SUBCMD_NCQ_NON_DATA_ZAC_MGMT_OUT; tf->nsect = qc->tag << 3; - tf->auxiliary = sa | (reset_all & 0x1) << 8; + tf->auxiliary = sa | ((u16)all << 8); } else { tf->protocol = ATA_PROT_NODATA; tf->command = ATA_CMD_ZAC_MGMT_OUT; tf->feature = sa; - tf->hob_feature = reset_all & 0x1; + tf->hob_feature = all; } tf->lbah = (block >> 16) & 0xff; tf->lbam = (block >> 8) & 0xff; @@ -3667,7 +3688,7 @@ static int ata_mselect_control(struct ata_queued_cmd *qc, /* * Check that read-only bits are not modified. */ - ata_msense_ctl_mode(dev, mpage, false); + ata_msense_control(dev, mpage, false); for (i = 0; i < CONTROL_MPAGE_LEN - 2; i++) { if (i == 0) continue; @@ -4039,11 +4060,6 @@ void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd) args.done = cmd->scsi_done; switch(scsicmd[0]) { - /* TODO: worth improving? */ - case FORMAT_UNIT: - ata_scsi_invalid_field(dev, cmd, 0); - break; - case INQUIRY: if (scsicmd[1] & 2) /* is CmdDt set? */ ata_scsi_invalid_field(dev, cmd, 1); diff --git a/drivers/ata/libata-transport.c b/drivers/ata/libata-transport.c index e2d94972962d..7ef16c085058 100644 --- a/drivers/ata/libata-transport.c +++ b/drivers/ata/libata-transport.c @@ -495,12 +495,13 @@ struct ata_show_ering_arg { static int ata_show_ering(struct ata_ering_entry *ent, void *void_arg) { struct ata_show_ering_arg* arg = void_arg; - struct timespec time; + u64 seconds; + u32 rem; - jiffies_to_timespec(ent->timestamp,&time); + seconds = div_u64_rem(ent->timestamp, HZ, &rem); arg->written += sprintf(arg->buf + arg->written, - "[%5lu.%06lu]", - time.tv_sec, time.tv_nsec); + "[%5llu.%09lu]", seconds, + rem * NSEC_PER_SEC / HZ); arg->written += get_ata_err_names(ent->err_mask, arg->buf + arg->written); return 0; diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c index 80fe0f6fed29..b4d54771c9fe 100644 --- a/drivers/ata/pata_arasan_cf.c +++ b/drivers/ata/pata_arasan_cf.c @@ -565,7 +565,7 @@ chan_request_fail: qc->ap->hsm_task_state = HSM_ST_ERR; cf_ctrl_reset(acdev); - spin_unlock_irqrestore(qc->ap->lock, flags); + spin_unlock_irqrestore(&acdev->host->lock, flags); sff_intr: dma_complete(acdev); } diff --git a/drivers/ata/pata_atiixp.c b/drivers/ata/pata_atiixp.c index 970f7767e5fd..49d705c9f0f7 100644 --- a/drivers/ata/pata_atiixp.c +++ b/drivers/ata/pata_atiixp.c @@ -183,8 +183,8 @@ static void atiixp_set_dmamode(struct ata_port *ap, struct ata_device *adev) * We must now look at the PIO mode situation. We may need to * adjust the PIO mode to keep the timings acceptable */ - if (adev->dma_mode >= XFER_MW_DMA_2) - wanted_pio = 4; + if (adev->dma_mode >= XFER_MW_DMA_2) + wanted_pio = 4; else if (adev->dma_mode == XFER_MW_DMA_1) wanted_pio = 3; else if (adev->dma_mode == XFER_MW_DMA_0) diff --git a/drivers/ata/pata_hpt366.c b/drivers/ata/pata_hpt366.c index e5fb7525a5df..a219a503c229 100644 --- a/drivers/ata/pata_hpt366.c +++ b/drivers/ata/pata_hpt366.c @@ -368,7 +368,7 @@ static int hpt36x_init_one(struct pci_dev *dev, const struct pci_device_id *id) /* PCI clocking determines the ATA timing values to use */ /* info_hpt366 is safe against re-entry so we can scribble on it */ - switch ((reg1 & 0x700) >> 8) { + switch ((reg1 & 0xf00) >> 8) { case 9: hpriv = &hpt366_40; break; diff --git a/drivers/ata/pata_marvell.c b/drivers/ata/pata_marvell.c index ae9feb1ba8f7..ff468a6fd8dd 100644 --- a/drivers/ata/pata_marvell.c +++ b/drivers/ata/pata_marvell.c @@ -146,7 +146,7 @@ static int marvell_init_one (struct pci_dev *pdev, const struct pci_device_id *i if (pdev->device == 0x6101) ppi[1] = &ata_dummy_port_info; -#if defined(CONFIG_SATA_AHCI) || defined(CONFIG_SATA_AHCI_MODULE) +#if IS_ENABLED(CONFIG_SATA_AHCI) if (!marvell_pata_active(pdev)) { printk(KERN_INFO DRV_NAME ": PATA port not active, deferring to AHCI driver.\n"); return -ENODEV; diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c index 00c2af1d211b..e0939bd5ea73 100644 --- a/drivers/ata/sata_dwc_460ex.c +++ b/drivers/ata/sata_dwc_460ex.c @@ -259,11 +259,8 @@ static int sata_dwc_dma_init_old(struct platform_device *pdev, /* Get physical SATA DMA register base address */ res = platform_get_resource(pdev, IORESOURCE_MEM, 1); hsdev->dma->regs = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(hsdev->dma->regs)) { - dev_err(&pdev->dev, - "ioremap failed for AHBDMA register address\n"); + if (IS_ERR(hsdev->dma->regs)) return PTR_ERR(hsdev->dma->regs); - } /* Initialize AHB DMAC */ return dw_dma_probe(hsdev->dma); @@ -281,7 +278,7 @@ static void sata_dwc_dma_exit_old(struct sata_dwc_device *hsdev) static const char *get_prot_descript(u8 protocol) { - switch ((enum ata_tf_protocols)protocol) { + switch (protocol) { case ATA_PROT_NODATA: return "ATA no data"; case ATA_PROT_PIO: @@ -290,6 +287,8 @@ static const char *get_prot_descript(u8 protocol) return "ATA DMA"; case ATA_PROT_NCQ: return "ATA NCQ"; + case ATA_PROT_NCQ_NODATA: + return "ATA NCQ no data"; case ATAPI_PROT_NODATA: return "ATAPI no data"; case ATAPI_PROT_PIO: @@ -1225,11 +1224,8 @@ static int sata_dwc_probe(struct platform_device *ofdev) /* Ioremap SATA registers */ res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); base = devm_ioremap_resource(&ofdev->dev, res); - if (IS_ERR(base)) { - dev_err(&ofdev->dev, - "ioremap failed for SATA register address\n"); + if (IS_ERR(base)) return PTR_ERR(base); - } dev_dbg(&ofdev->dev, "ioremap done for SATA register address\n"); /* Synopsys DWC SATA specific Registers */ diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c index bd74ee555278..745489a1c86a 100644 --- a/drivers/ata/sata_mv.c +++ b/drivers/ata/sata_mv.c @@ -986,7 +986,7 @@ static inline void mv_write_cached_reg(void __iomem *addr, u32 *old, u32 new) * Looks like a lot of fuss, but it avoids an unnecessary * +1 usec read-after-write delay for unaffected registers. */ - laddr = (long)addr & 0xffff; + laddr = (unsigned long)addr & 0xffff; if (laddr >= 0x300 && laddr <= 0x33c) { laddr &= 0x000f; if (laddr == 0x4 || laddr == 0xc) { diff --git a/drivers/atm/horizon.c b/drivers/atm/horizon.c index 527bbd595e37..5fc81e240c24 100644 --- a/drivers/atm/horizon.c +++ b/drivers/atm/horizon.c @@ -2795,9 +2795,7 @@ static int hrz_probe(struct pci_dev *pci_dev, dev->atm_dev->ci_range.vpi_bits = vpi_bits; dev->atm_dev->ci_range.vci_bits = 10-vpi_bits; - init_timer(&dev->housekeeping); - dev->housekeeping.function = do_housekeeping; - dev->housekeeping.data = (unsigned long) dev; + setup_timer(&dev->housekeeping, do_housekeeping, (unsigned long) dev); mod_timer(&dev->housekeeping, jiffies); out: diff --git a/drivers/atm/nicstar.c b/drivers/atm/nicstar.c index ddc4ceb85fc5..700ed15c2362 100644 --- a/drivers/atm/nicstar.c +++ b/drivers/atm/nicstar.c @@ -874,7 +874,8 @@ static scq_info *get_scq(ns_dev *card, int size, u32 scd) scq->skb = kmalloc(sizeof(struct sk_buff *) * (size / NS_SCQE_SIZE), GFP_KERNEL); if (!scq->skb) { - kfree(scq->org); + dma_free_coherent(&card->pcidev->dev, + 2 * size, scq->org, scq->dma); kfree(scq); return NULL; } diff --git a/drivers/base/memory.c b/drivers/base/memory.c index f46dba8b7092..dc75de9059cd 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -391,6 +391,7 @@ static ssize_t show_valid_zones(struct device *dev, unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; struct page *first_page; struct zone *zone; + int zone_shift = 0; start_pfn = section_nr_to_pfn(mem->start_section_nr); end_pfn = start_pfn + nr_pages; @@ -402,21 +403,26 @@ static ssize_t show_valid_zones(struct device *dev, zone = page_zone(first_page); - if (zone_idx(zone) == ZONE_MOVABLE - 1) { - /*The mem block is the last memoryblock of this zone.*/ - if (end_pfn == zone_end_pfn(zone)) - return sprintf(buf, "%s %s\n", - zone->name, (zone + 1)->name); + /* MMOP_ONLINE_KEEP */ + sprintf(buf, "%s", zone->name); + + /* MMOP_ONLINE_KERNEL */ + zone_shift = zone_can_shift(start_pfn, nr_pages, ZONE_NORMAL); + if (zone_shift) { + strcat(buf, " "); + strcat(buf, (zone + zone_shift)->name); } - if (zone_idx(zone) == ZONE_MOVABLE) { - /*The mem block is the first memoryblock of ZONE_MOVABLE.*/ - if (start_pfn == zone->zone_start_pfn) - return sprintf(buf, "%s %s\n", - zone->name, (zone - 1)->name); + /* MMOP_ONLINE_MOVABLE */ + zone_shift = zone_can_shift(start_pfn, nr_pages, ZONE_MOVABLE); + if (zone_shift) { + strcat(buf, " "); + strcat(buf, (zone + zone_shift)->name); } - return sprintf(buf, "%s\n", zone->name); + strcat(buf, "\n"); + + return strlen(buf); } static DEVICE_ATTR(valid_zones, 0444, show_valid_zones, NULL); #endif diff --git a/drivers/base/node.c b/drivers/base/node.c index 560751bad294..51c7db2c4ee2 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -113,6 +113,8 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d SUnreclaim: %8lu kB\n" #ifdef CONFIG_TRANSPARENT_HUGEPAGE "Node %d AnonHugePages: %8lu kB\n" + "Node %d ShmemHugePages: %8lu kB\n" + "Node %d ShmemPmdMapped: %8lu kB\n" #endif , nid, K(node_page_state(nid, NR_FILE_DIRTY)), @@ -131,10 +133,13 @@ static ssize_t node_read_meminfo(struct device *dev, node_page_state(nid, NR_SLAB_UNRECLAIMABLE)), nid, K(node_page_state(nid, NR_SLAB_RECLAIMABLE)), #ifdef CONFIG_TRANSPARENT_HUGEPAGE - nid, K(node_page_state(nid, NR_SLAB_UNRECLAIMABLE)) - , nid, - K(node_page_state(nid, NR_ANON_TRANSPARENT_HUGEPAGES) * - HPAGE_PMD_NR)); + nid, K(node_page_state(nid, NR_SLAB_UNRECLAIMABLE)), + nid, K(node_page_state(nid, NR_ANON_THPS) * + HPAGE_PMD_NR), + nid, K(node_page_state(nid, NR_SHMEM_THPS) * + HPAGE_PMD_NR), + nid, K(node_page_state(nid, NR_SHMEM_PMDMAPPED) * + HPAGE_PMD_NR)); #else nid, K(node_page_state(nid, NR_SLAB_UNRECLAIMABLE))); #endif diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c index 3657ac1cb801..8e2e4757adcb 100644 --- a/drivers/base/power/clock_ops.c +++ b/drivers/base/power/clock_ops.c @@ -121,6 +121,7 @@ int pm_clk_add(struct device *dev, const char *con_id) { return __pm_clk_add(dev, con_id, NULL); } +EXPORT_SYMBOL_GPL(pm_clk_add); /** * pm_clk_add_clk - Start using a device clock for power management. @@ -136,8 +137,41 @@ int pm_clk_add_clk(struct device *dev, struct clk *clk) { return __pm_clk_add(dev, NULL, clk); } +EXPORT_SYMBOL_GPL(pm_clk_add_clk); +/** + * of_pm_clk_add_clk - Start using a device clock for power management. + * @dev: Device whose clock is going to be used for power management. + * @name: Name of clock that is going to be used for power management. + * + * Add the clock described in the 'clocks' device-tree node that matches + * with the 'name' provided, to the list of clocks used for the power + * management of @dev. On success, returns 0. Returns a negative error + * code if the clock is not found or cannot be added. + */ +int of_pm_clk_add_clk(struct device *dev, const char *name) +{ + struct clk *clk; + int ret; + + if (!dev || !dev->of_node || !name) + return -EINVAL; + + clk = of_clk_get_by_name(dev->of_node, name); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + ret = pm_clk_add_clk(dev, clk); + if (ret) { + clk_put(clk); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(of_pm_clk_add_clk); + /** * of_pm_clk_add_clks - Start using device clock(s) for power management. * @dev: Device whose clock(s) is going to be used for power management. @@ -192,6 +226,7 @@ error: return ret; } +EXPORT_SYMBOL_GPL(of_pm_clk_add_clks); /** * __pm_clk_remove - Destroy PM clock entry. @@ -252,6 +287,7 @@ void pm_clk_remove(struct device *dev, const char *con_id) __pm_clk_remove(ce); } +EXPORT_SYMBOL_GPL(pm_clk_remove); /** * pm_clk_remove_clk - Stop using a device clock for power management. @@ -285,6 +321,7 @@ void pm_clk_remove_clk(struct device *dev, struct clk *clk) __pm_clk_remove(ce); } +EXPORT_SYMBOL_GPL(pm_clk_remove_clk); /** * pm_clk_init - Initialize a device's list of power management clocks. @@ -299,6 +336,7 @@ void pm_clk_init(struct device *dev) if (psd) INIT_LIST_HEAD(&psd->clock_list); } +EXPORT_SYMBOL_GPL(pm_clk_init); /** * pm_clk_create - Create and initialize a device's list of PM clocks. @@ -311,6 +349,7 @@ int pm_clk_create(struct device *dev) { return dev_pm_get_subsys_data(dev); } +EXPORT_SYMBOL_GPL(pm_clk_create); /** * pm_clk_destroy - Destroy a device's list of power management clocks. @@ -345,6 +384,7 @@ void pm_clk_destroy(struct device *dev) __pm_clk_remove(ce); } } +EXPORT_SYMBOL_GPL(pm_clk_destroy); /** * pm_clk_suspend - Disable clocks in a device's PM clock list. @@ -375,6 +415,7 @@ int pm_clk_suspend(struct device *dev) return 0; } +EXPORT_SYMBOL_GPL(pm_clk_suspend); /** * pm_clk_resume - Enable clocks in a device's PM clock list. @@ -400,6 +441,7 @@ int pm_clk_resume(struct device *dev) return 0; } +EXPORT_SYMBOL_GPL(pm_clk_resume); /** * pm_clk_notify - Notify routine for device addition and removal. @@ -480,6 +522,7 @@ int pm_clk_runtime_suspend(struct device *dev) return 0; } +EXPORT_SYMBOL_GPL(pm_clk_runtime_suspend); int pm_clk_runtime_resume(struct device *dev) { @@ -495,6 +538,7 @@ int pm_clk_runtime_resume(struct device *dev) return pm_generic_runtime_resume(dev); } +EXPORT_SYMBOL_GPL(pm_clk_runtime_resume); #else /* !CONFIG_PM_CLK */ @@ -598,3 +642,4 @@ void pm_clk_add_notifier(struct bus_type *bus, clknb->nb.notifier_call = pm_clk_notify; bus_register_notifier(bus, &clknb->nb); } +EXPORT_SYMBOL_GPL(pm_clk_add_notifier); diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index de23b648fce3..a1f2aff33997 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -187,8 +187,7 @@ static int genpd_poweron(struct generic_pm_domain *genpd, unsigned int depth) struct gpd_link *link; int ret = 0; - if (genpd->status == GPD_STATE_ACTIVE - || (genpd->prepared_count > 0 && genpd->suspend_power_off)) + if (genpd->status == GPD_STATE_ACTIVE) return 0; /* @@ -735,81 +734,23 @@ static int pm_genpd_prepare(struct device *dev) mutex_lock(&genpd->lock); - if (genpd->prepared_count++ == 0) { + if (genpd->prepared_count++ == 0) genpd->suspended_count = 0; - genpd->suspend_power_off = genpd->status == GPD_STATE_POWER_OFF; - } mutex_unlock(&genpd->lock); - if (genpd->suspend_power_off) - return 0; - - /* - * The PM domain must be in the GPD_STATE_ACTIVE state at this point, - * so genpd_poweron() will return immediately, but if the device - * is suspended (e.g. it's been stopped by genpd_stop_dev()), we need - * to make it operational. - */ - pm_runtime_resume(dev); - __pm_runtime_disable(dev, false); - ret = pm_generic_prepare(dev); if (ret) { mutex_lock(&genpd->lock); - if (--genpd->prepared_count == 0) - genpd->suspend_power_off = false; + genpd->prepared_count--; mutex_unlock(&genpd->lock); - pm_runtime_enable(dev); } return ret; } -/** - * pm_genpd_suspend - Suspend a device belonging to an I/O PM domain. - * @dev: Device to suspend. - * - * Suspend a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a PM domain consisting of I/O devices. - */ -static int pm_genpd_suspend(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_suspend(dev); -} - -/** - * pm_genpd_suspend_late - Late suspend of a device from an I/O PM domain. - * @dev: Device to suspend. - * - * Carry out a late suspend of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a PM domain consisting of I/O devices. - */ -static int pm_genpd_suspend_late(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_suspend_late(dev); -} - /** * pm_genpd_suspend_noirq - Completion of suspend of device in an I/O PM domain. * @dev: Device to suspend. @@ -820,6 +761,7 @@ static int pm_genpd_suspend_late(struct device *dev) static int pm_genpd_suspend_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret; dev_dbg(dev, "%s()\n", __func__); @@ -827,11 +769,14 @@ static int pm_genpd_suspend_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - if (genpd->suspend_power_off - || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev))) + if (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)) return 0; - genpd_stop_dev(genpd, dev); + if (genpd->dev_ops.stop && genpd->dev_ops.start) { + ret = pm_runtime_force_suspend(dev); + if (ret) + return ret; + } /* * Since all of the "noirq" callbacks are executed sequentially, it is @@ -853,6 +798,7 @@ static int pm_genpd_suspend_noirq(struct device *dev) static int pm_genpd_resume_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -860,8 +806,7 @@ static int pm_genpd_resume_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - if (genpd->suspend_power_off - || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev))) + if (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)) return 0; /* @@ -872,93 +817,10 @@ static int pm_genpd_resume_noirq(struct device *dev) pm_genpd_sync_poweron(genpd, true); genpd->suspended_count--; - return genpd_start_dev(genpd, dev); -} - -/** - * pm_genpd_resume_early - Early resume of a device in an I/O PM domain. - * @dev: Device to resume. - * - * Carry out an early resume of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_resume_early(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_resume_early(dev); -} - -/** - * pm_genpd_resume - Resume of device in an I/O PM domain. - * @dev: Device to resume. - * - * Resume a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_resume(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_resume(dev); -} - -/** - * pm_genpd_freeze - Freezing a device in an I/O PM domain. - * @dev: Device to freeze. - * - * Freeze a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_freeze(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_freeze(dev); -} + if (genpd->dev_ops.stop && genpd->dev_ops.start) + ret = pm_runtime_force_resume(dev); -/** - * pm_genpd_freeze_late - Late freeze of a device in an I/O PM domain. - * @dev: Device to freeze. - * - * Carry out a late freeze of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_freeze_late(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_freeze_late(dev); + return ret; } /** @@ -973,6 +835,7 @@ static int pm_genpd_freeze_late(struct device *dev) static int pm_genpd_freeze_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -980,7 +843,10 @@ static int pm_genpd_freeze_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - return genpd->suspend_power_off ? 0 : genpd_stop_dev(genpd, dev); + if (genpd->dev_ops.stop && genpd->dev_ops.start) + ret = pm_runtime_force_suspend(dev); + + return ret; } /** @@ -993,6 +859,7 @@ static int pm_genpd_freeze_noirq(struct device *dev) static int pm_genpd_thaw_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -1000,51 +867,10 @@ static int pm_genpd_thaw_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - return genpd->suspend_power_off ? - 0 : genpd_start_dev(genpd, dev); -} - -/** - * pm_genpd_thaw_early - Early thaw of device in an I/O PM domain. - * @dev: Device to thaw. - * - * Carry out an early thaw of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_thaw_early(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_thaw_early(dev); -} - -/** - * pm_genpd_thaw - Thaw a device belonging to an I/O power domain. - * @dev: Device to thaw. - * - * Thaw a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_thaw(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); + if (genpd->dev_ops.stop && genpd->dev_ops.start) + ret = pm_runtime_force_resume(dev); - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_thaw(dev); + return ret; } /** @@ -1057,6 +883,7 @@ static int pm_genpd_thaw(struct device *dev) static int pm_genpd_restore_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -1072,30 +899,20 @@ static int pm_genpd_restore_noirq(struct device *dev) * At this point suspended_count == 0 means we are being run for the * first time for the given domain in the present cycle. */ - if (genpd->suspended_count++ == 0) { + if (genpd->suspended_count++ == 0) /* * The boot kernel might put the domain into arbitrary state, * so make it appear as powered off to pm_genpd_sync_poweron(), * so that it tries to power it on in case it was really off. */ genpd->status = GPD_STATE_POWER_OFF; - if (genpd->suspend_power_off) { - /* - * If the domain was off before the hibernation, make - * sure it will be off going forward. - */ - genpd_power_off(genpd, true); - - return 0; - } - } - - if (genpd->suspend_power_off) - return 0; pm_genpd_sync_poweron(genpd, true); - return genpd_start_dev(genpd, dev); + if (genpd->dev_ops.stop && genpd->dev_ops.start) + ret = pm_runtime_force_resume(dev); + + return ret; } /** @@ -1110,7 +927,6 @@ static int pm_genpd_restore_noirq(struct device *dev) static void pm_genpd_complete(struct device *dev) { struct generic_pm_domain *genpd; - bool run_complete; dev_dbg(dev, "%s()\n", __func__); @@ -1118,20 +934,15 @@ static void pm_genpd_complete(struct device *dev) if (IS_ERR(genpd)) return; + pm_generic_complete(dev); + mutex_lock(&genpd->lock); - run_complete = !genpd->suspend_power_off; - if (--genpd->prepared_count == 0) - genpd->suspend_power_off = false; + genpd->prepared_count--; + if (!genpd->prepared_count) + genpd_queue_power_off_work(genpd); mutex_unlock(&genpd->lock); - - if (run_complete) { - pm_generic_complete(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - pm_request_idle(dev); - } } /** @@ -1173,18 +984,10 @@ EXPORT_SYMBOL_GPL(pm_genpd_syscore_poweron); #else /* !CONFIG_PM_SLEEP */ #define pm_genpd_prepare NULL -#define pm_genpd_suspend NULL -#define pm_genpd_suspend_late NULL #define pm_genpd_suspend_noirq NULL -#define pm_genpd_resume_early NULL #define pm_genpd_resume_noirq NULL -#define pm_genpd_resume NULL -#define pm_genpd_freeze NULL -#define pm_genpd_freeze_late NULL #define pm_genpd_freeze_noirq NULL -#define pm_genpd_thaw_early NULL #define pm_genpd_thaw_noirq NULL -#define pm_genpd_thaw NULL #define pm_genpd_restore_noirq NULL #define pm_genpd_complete NULL @@ -1455,12 +1258,14 @@ EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain); * @genpd: PM domain object to initialize. * @gov: PM domain governor to associate with the domain (may be NULL). * @is_off: Initial value of the domain's power_is_off field. + * + * Returns 0 on successful initialization, else a negative error code. */ -void pm_genpd_init(struct generic_pm_domain *genpd, - struct dev_power_governor *gov, bool is_off) +int pm_genpd_init(struct generic_pm_domain *genpd, + struct dev_power_governor *gov, bool is_off) { if (IS_ERR_OR_NULL(genpd)) - return; + return -EINVAL; INIT_LIST_HEAD(&genpd->master_links); INIT_LIST_HEAD(&genpd->slave_links); @@ -1476,24 +1281,24 @@ void pm_genpd_init(struct generic_pm_domain *genpd, genpd->domain.ops.runtime_suspend = genpd_runtime_suspend; genpd->domain.ops.runtime_resume = genpd_runtime_resume; genpd->domain.ops.prepare = pm_genpd_prepare; - genpd->domain.ops.suspend = pm_genpd_suspend; - genpd->domain.ops.suspend_late = pm_genpd_suspend_late; + genpd->domain.ops.suspend = pm_generic_suspend; + genpd->domain.ops.suspend_late = pm_generic_suspend_late; genpd->domain.ops.suspend_noirq = pm_genpd_suspend_noirq; genpd->domain.ops.resume_noirq = pm_genpd_resume_noirq; - genpd->domain.ops.resume_early = pm_genpd_resume_early; - genpd->domain.ops.resume = pm_genpd_resume; - genpd->domain.ops.freeze = pm_genpd_freeze; - genpd->domain.ops.freeze_late = pm_genpd_freeze_late; + genpd->domain.ops.resume_early = pm_generic_resume_early; + genpd->domain.ops.resume = pm_generic_resume; + genpd->domain.ops.freeze = pm_generic_freeze; + genpd->domain.ops.freeze_late = pm_generic_freeze_late; genpd->domain.ops.freeze_noirq = pm_genpd_freeze_noirq; genpd->domain.ops.thaw_noirq = pm_genpd_thaw_noirq; - genpd->domain.ops.thaw_early = pm_genpd_thaw_early; - genpd->domain.ops.thaw = pm_genpd_thaw; - genpd->domain.ops.poweroff = pm_genpd_suspend; - genpd->domain.ops.poweroff_late = pm_genpd_suspend_late; + genpd->domain.ops.thaw_early = pm_generic_thaw_early; + genpd->domain.ops.thaw = pm_generic_thaw; + genpd->domain.ops.poweroff = pm_generic_poweroff; + genpd->domain.ops.poweroff_late = pm_generic_poweroff_late; genpd->domain.ops.poweroff_noirq = pm_genpd_suspend_noirq; genpd->domain.ops.restore_noirq = pm_genpd_restore_noirq; - genpd->domain.ops.restore_early = pm_genpd_resume_early; - genpd->domain.ops.restore = pm_genpd_resume; + genpd->domain.ops.restore_early = pm_generic_restore_early; + genpd->domain.ops.restore = pm_generic_restore; genpd->domain.ops.complete = pm_genpd_complete; if (genpd->flags & GENPD_FLAG_PM_CLK) { @@ -1518,6 +1323,8 @@ void pm_genpd_init(struct generic_pm_domain *genpd, mutex_lock(&gpd_list_lock); list_add(&genpd->gpd_list_node, &gpd_list); mutex_unlock(&gpd_list_lock); + + return 0; } EXPORT_SYMBOL_GPL(pm_genpd_init); diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index b74690418504..e097d355cc04 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1045,10 +1045,14 @@ int __pm_runtime_set_status(struct device *dev, unsigned int status) */ if (!parent->power.disable_depth && !parent->power.ignore_children - && parent->power.runtime_status != RPM_ACTIVE) + && parent->power.runtime_status != RPM_ACTIVE) { + dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n", + dev_name(dev), + dev_name(parent)); error = -EBUSY; - else if (dev->power.runtime_status == RPM_SUSPENDED) + } else if (dev->power.runtime_status == RPM_SUSPENDED) { atomic_inc(&parent->power.child_count); + } spin_unlock(&parent->power.lock); @@ -1256,7 +1260,7 @@ void pm_runtime_allow(struct device *dev) dev->power.runtime_auto = true; if (atomic_dec_and_test(&dev->power.usage_count)) - rpm_idle(dev, RPM_AUTO); + rpm_idle(dev, RPM_AUTO | RPM_ASYNC); out: spin_unlock_irq(&dev->power.lock); @@ -1506,6 +1510,9 @@ int pm_runtime_force_resume(struct device *dev) goto out; } + if (!pm_runtime_status_suspended(dev)) + goto out; + ret = pm_runtime_set_active(dev); if (ret) goto out; diff --git a/drivers/base/regmap/regmap-i2c.c b/drivers/base/regmap/regmap-i2c.c index 1a8ec3b2b601..4735318f4268 100644 --- a/drivers/base/regmap/regmap-i2c.c +++ b/drivers/base/regmap/regmap-i2c.c @@ -259,7 +259,7 @@ static const struct regmap_bus *regmap_get_i2c_bus(struct i2c_client *i2c, { if (i2c_check_functionality(i2c->adapter, I2C_FUNC_I2C)) return ®map_i2c; - else if (config->reg_bits == 8 && + else if (config->val_bits == 8 && config->reg_bits == 8 && i2c_check_functionality(i2c->adapter, I2C_FUNC_SMBUS_I2C_BLOCK)) return ®map_i2c_smbus_i2c_block; diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index 26f799e71c82..ec262476d043 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -268,13 +268,16 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) bool handled = false; u32 reg; + if (chip->handle_pre_irq) + chip->handle_pre_irq(chip->irq_drv_data); + if (chip->runtime_pm) { ret = pm_runtime_get_sync(map->dev); if (ret < 0) { dev_err(map->dev, "IRQ thread failed to resume: %d\n", ret); pm_runtime_put(map->dev); - return IRQ_NONE; + goto exit; } } @@ -296,7 +299,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) if (ret != 0) { dev_err(map->dev, "Failed to read IRQ status: %d\n", ret); - return IRQ_NONE; + goto exit; } for (i = 0; i < data->chip->num_regs; i++) { @@ -312,7 +315,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) break; default: BUG(); - return IRQ_NONE; + goto exit; } } @@ -329,7 +332,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) ret); if (chip->runtime_pm) pm_runtime_put(map->dev); - return IRQ_NONE; + goto exit; } } } @@ -365,6 +368,10 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) if (chip->runtime_pm) pm_runtime_put(map->dev); +exit: + if (chip->handle_post_irq) + chip->handle_post_irq(chip->irq_drv_data); + if (handled) return IRQ_HANDLED; else diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index df2d2ef5d6b3..51fa7d66a393 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -1777,8 +1777,6 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, size_t val_bytes = map->format.val_bytes; size_t total_size = val_bytes * val_count; - if (map->bus && !map->format.parse_inplace) - return -EINVAL; if (!IS_ALIGNED(reg, map->reg_stride)) return -EINVAL; @@ -1789,7 +1787,8 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, * * The first if block is used for memory mapped io. It does not allow * val_bytes of 3 for example. - * The second one is used for busses which do not have this limitation + * The second one is for busses that do not provide raw I/O. + * The third one is used for busses which do not have these limitations * and can write arbitrary value lengths. */ if (!map->bus) { @@ -1825,6 +1824,32 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, } out: map->unlock(map->lock_arg); + } else if (map->bus && !map->format.parse_inplace) { + const u8 *u8 = val; + const u16 *u16 = val; + const u32 *u32 = val; + unsigned int ival; + + for (i = 0; i < val_count; i++) { + switch (map->format.val_bytes) { + case 4: + ival = u32[i]; + break; + case 2: + ival = u16[i]; + break; + case 1: + ival = u8[i]; + break; + default: + return -EINVAL; + } + + ret = regmap_write(map, reg + (i * map->reg_stride), + ival); + if (ret) + return ret; + } } else if (map->use_single_write || (map->max_raw_write && map->max_raw_write < total_size)) { int chunk_stride = map->reg_stride; diff --git a/drivers/base/topology.c b/drivers/base/topology.c index 8b7d7f8e5851..df3c97cb4c99 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -77,6 +77,14 @@ static DEVICE_ATTR_RO(book_siblings); static DEVICE_ATTR_RO(book_siblings_list); #endif +#ifdef CONFIG_SCHED_DRAWER +define_id_show_func(drawer_id); +static DEVICE_ATTR_RO(drawer_id); +define_siblings_show_func(drawer_siblings, drawer_cpumask); +static DEVICE_ATTR_RO(drawer_siblings); +static DEVICE_ATTR_RO(drawer_siblings_list); +#endif + static struct attribute *default_attrs[] = { &dev_attr_physical_package_id.attr, &dev_attr_core_id.attr, @@ -88,6 +96,11 @@ static struct attribute *default_attrs[] = { &dev_attr_book_id.attr, &dev_attr_book_siblings.attr, &dev_attr_book_siblings_list.attr, +#endif +#ifdef CONFIG_SCHED_DRAWER + &dev_attr_drawer_id.attr, + &dev_attr_drawer_siblings.attr, + &dev_attr_drawer_siblings_list.attr, #endif NULL }; diff --git a/drivers/bcma/Kconfig b/drivers/bcma/Kconfig index efdc2ae8441a..b5c48a8d485f 100644 --- a/drivers/bcma/Kconfig +++ b/drivers/bcma/Kconfig @@ -76,9 +76,16 @@ config BCMA_PFLASH default y config BCMA_SFLASH - bool - depends on BCMA_DRIVER_MIPS + bool "ChipCommon-attached serial flash support" + depends on BCMA_HOST_SOC default y + help + Some cheap devices have serial flash connected to the ChipCommon + instead of independent SPI controller. It requires using a separated + driver that implements ChipCommon specific interface communication. + + Enabling this symbol will let bcma recognize serial flash and register + it as platform device. config BCMA_NFLASH bool diff --git a/drivers/bcma/bcma_private.h b/drivers/bcma/bcma_private.h index eda09090cb52..f642c4264c27 100644 --- a/drivers/bcma/bcma_private.h +++ b/drivers/bcma/bcma_private.h @@ -8,8 +8,6 @@ #include #include -#define BCMA_CORE_SIZE 0x1000 - #define bcma_err(bus, fmt, ...) \ pr_err("bus%d: " fmt, (bus)->num, ##__VA_ARGS__) #define bcma_warn(bus, fmt, ...) \ diff --git a/drivers/bcma/driver_chipcommon_b.c b/drivers/bcma/driver_chipcommon_b.c index c20b5f4ff290..57f10b58b47c 100644 --- a/drivers/bcma/driver_chipcommon_b.c +++ b/drivers/bcma/driver_chipcommon_b.c @@ -33,11 +33,12 @@ static bool bcma_wait_reg(struct bcma_bus *bus, void __iomem *addr, u32 mask, void bcma_chipco_b_mii_write(struct bcma_drv_cc_b *ccb, u32 offset, u32 value) { struct bcma_bus *bus = ccb->core->bus; + void __iomem *mii = ccb->mii; - writel(offset, ccb->mii + 0x00); - bcma_wait_reg(bus, ccb->mii + 0x00, 0x0100, 0x0000, 100); - writel(value, ccb->mii + 0x04); - bcma_wait_reg(bus, ccb->mii + 0x00, 0x0100, 0x0000, 100); + writel(offset, mii + BCMA_CCB_MII_MNG_CTL); + bcma_wait_reg(bus, mii + BCMA_CCB_MII_MNG_CTL, 0x0100, 0x0000, 100); + writel(value, mii + BCMA_CCB_MII_MNG_CMD_DATA); + bcma_wait_reg(bus, mii + BCMA_CCB_MII_MNG_CTL, 0x0100, 0x0000, 100); } EXPORT_SYMBOL_GPL(bcma_chipco_b_mii_write); diff --git a/drivers/bcma/host_pci.c b/drivers/bcma/host_pci.c index cae5385cf499..bd46569e0e52 100644 --- a/drivers/bcma/host_pci.c +++ b/drivers/bcma/host_pci.c @@ -295,6 +295,7 @@ static const struct pci_device_id bcma_pci_bridge_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, 0x4359) }, { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, 0x4360) }, { PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, 0x4365, PCI_VENDOR_ID_DELL, 0x0016) }, + { PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, 0x4365, PCI_VENDOR_ID_FOXCONN, 0xe092) }, { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, 0x43a0) }, { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, 0x43a9) }, { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, 0x43aa) }, diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c index d597e432e195..ab19adb07a12 100644 --- a/drivers/block/aoe/aoecmd.c +++ b/drivers/block/aoe/aoecmd.c @@ -1750,7 +1750,7 @@ aoecmd_init(void) int ret; /* get_zeroed_page returns page with ref count 1 */ - p = (void *) get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); + p = (void *) get_zeroed_page(GFP_KERNEL); if (!p) return -ENOMEM; empty_page = virt_to_page(p); diff --git a/drivers/block/brd.c b/drivers/block/brd.c index c04bd9bc39fd..ba5145d384d8 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -339,7 +339,7 @@ static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio) if (bio_end_sector(bio) > get_capacity(bdev->bd_disk)) goto io_error; - if (unlikely(bio->bi_rw & REQ_DISCARD)) { + if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { if (sector & ((PAGE_SIZE >> SECTOR_SHIFT) - 1) || bio->bi_iter.bi_size & ~PAGE_MASK) goto io_error; @@ -347,9 +347,7 @@ static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio) goto out; } - rw = bio_rw(bio); - if (rw == READA) - rw = READ; + rw = bio_data_dir(bio); bio_for_each_segment(bvec, bio, iter) { unsigned int len = bvec.bv_len; @@ -509,7 +507,9 @@ static struct brd_device *brd_alloc(int i) blk_queue_max_discard_sectors(brd->brd_queue, UINT_MAX); brd->brd_queue->limits.discard_zeroes_data = 1; queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue); - +#ifdef CONFIG_BLK_DEV_RAM_DAX + queue_flag_set_unlocked(QUEUE_FLAG_DAX, brd->brd_queue); +#endif disk = brd->brd_disk = alloc_disk(max_part); if (!disk) goto out_free_queue; diff --git a/drivers/block/cciss.c b/drivers/block/cciss.c index 63c2064689f8..db9d6bb6352d 100644 --- a/drivers/block/cciss.c +++ b/drivers/block/cciss.c @@ -1951,7 +1951,6 @@ static int cciss_add_disk(ctlr_info_t *h, struct gendisk *disk, if (cciss_create_ld_sysfs_entry(h, drv_index)) goto cleanup_queue; disk->private_data = h->drv[drv_index]; - disk->driverfs_dev = &h->drv[drv_index]->dev; /* Set up queue information */ blk_queue_bounce_limit(disk->queue, h->pdev->dma_mask); @@ -1973,7 +1972,7 @@ static int cciss_add_disk(ctlr_info_t *h, struct gendisk *disk, /* allows the interrupt handler to start the queue */ wmb(); h->drv[drv_index]->queue = disk->queue; - add_disk(disk); + device_add_disk(&h->drv[drv_index]->dev, disk); return 0; cleanup_queue: diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c index 10459a145062..0a1aaf8c24c4 100644 --- a/drivers/block/drbd/drbd_actlog.c +++ b/drivers/block/drbd/drbd_actlog.c @@ -137,19 +137,19 @@ void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_b static int _drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bdev, - sector_t sector, int rw) + sector_t sector, int op) { struct bio *bio; /* we do all our meta data IO in aligned 4k blocks. */ const int size = 4096; - int err; + int err, op_flags = 0; device->md_io.done = 0; device->md_io.error = -ENODEV; - if ((rw & WRITE) && !test_bit(MD_NO_FUA, &device->flags)) - rw |= REQ_FUA | REQ_FLUSH; - rw |= REQ_SYNC | REQ_NOIDLE; + if ((op == REQ_OP_WRITE) && !test_bit(MD_NO_FUA, &device->flags)) + op_flags |= REQ_FUA | REQ_PREFLUSH; + op_flags |= REQ_SYNC | REQ_NOIDLE; bio = bio_alloc_drbd(GFP_NOIO); bio->bi_bdev = bdev->md_bdev; @@ -159,9 +159,9 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, goto out; bio->bi_private = device; bio->bi_end_io = drbd_md_endio; - bio->bi_rw = rw; + bio_set_op_attrs(bio, op, op_flags); - if (!(rw & WRITE) && device->state.disk == D_DISKLESS && device->ldev == NULL) + if (op != REQ_OP_WRITE && device->state.disk == D_DISKLESS && device->ldev == NULL) /* special case, drbd_md_read() during drbd_adm_attach(): no get_ldev */ ; else if (!get_ldev_if_state(device, D_ATTACHING)) { @@ -174,10 +174,10 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, bio_get(bio); /* one bio_put() is in the completion handler */ atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */ device->md_io.submit_jif = jiffies; - if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) + if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) bio_io_error(bio); else - submit_bio(rw, bio); + submit_bio(bio); wait_until_done_or_force_detached(device, bdev, &device->md_io.done); if (!bio->bi_error) err = device->md_io.error; @@ -188,7 +188,7 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, } int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bdev, - sector_t sector, int rw) + sector_t sector, int op) { int err; D_ASSERT(device, atomic_read(&device->md_io.in_use) == 1); @@ -197,19 +197,21 @@ int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bd dynamic_drbd_dbg(device, "meta_data io: %s [%d]:%s(,%llus,%s) %pS\n", current->comm, current->pid, __func__, - (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ", + (unsigned long long)sector, (op == REQ_OP_WRITE) ? "WRITE" : "READ", (void*)_RET_IP_ ); if (sector < drbd_md_first_sector(bdev) || sector + 7 > drbd_md_last_sector(bdev)) drbd_alert(device, "%s [%d]:%s(,%llus,%s) out of range md access!\n", current->comm, current->pid, __func__, - (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ"); + (unsigned long long)sector, + (op == REQ_OP_WRITE) ? "WRITE" : "READ"); - err = _drbd_md_sync_page_io(device, bdev, sector, rw); + err = _drbd_md_sync_page_io(device, bdev, sector, op); if (err) { drbd_err(device, "drbd_md_sync_page_io(,%llus,%s) failed with error %d\n", - (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ", err); + (unsigned long long)sector, + (op == REQ_OP_WRITE) ? "WRITE" : "READ", err); } return err; } @@ -256,7 +258,7 @@ bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); - D_ASSERT(device, (unsigned)(last - first) <= 1); + D_ASSERT(device, first <= last); D_ASSERT(device, atomic_read(&device->local_cnt) > 0); /* FIXME figure out a fast path for bios crossing AL extent boundaries */ @@ -339,6 +341,8 @@ static int __al_write_transaction(struct drbd_device *device, struct al_transact i = 0; + drbd_bm_reset_al_hints(device); + /* Even though no one can start to change this list * once we set the LC_LOCKED -- from drbd_al_begin_io(), * lc_try_lock_for_transaction() --, someone may still @@ -768,10 +772,18 @@ static bool lazy_bitmap_update_due(struct drbd_device *device) static void maybe_schedule_on_disk_bitmap_update(struct drbd_device *device, bool rs_done) { - if (rs_done) - set_bit(RS_DONE, &device->flags); - /* and also set RS_PROGRESS below */ - else if (!lazy_bitmap_update_due(device)) + if (rs_done) { + struct drbd_connection *connection = first_peer_device(device)->connection; + if (connection->agreed_pro_version <= 95 || + is_sync_target_state(device->state.conn)) + set_bit(RS_DONE, &device->flags); + /* and also set RS_PROGRESS below */ + + /* Else: rather wait for explicit notification via receive_state, + * to avoid uuids-rotated-too-fast causing full resync + * in next handshake, in case the replication link breaks + * at the most unfortunate time... */ + } else if (!lazy_bitmap_update_due(device)) return; drbd_device_post_work(device, RS_PROGRESS); @@ -830,6 +842,13 @@ static int update_sync_bits(struct drbd_device *device, return count; } +static bool plausible_request_size(int size) +{ + return size > 0 + && size <= DRBD_MAX_BATCH_BIO_SIZE + && IS_ALIGNED(size, 512); +} + /* clear the bit corresponding to the piece of storage in question: * size byte of data starting from sector. Only clear a bits of the affected * one ore more _aligned_ BM_BLOCK_SIZE blocks. @@ -845,11 +864,11 @@ int __drbd_change_sync(struct drbd_device *device, sector_t sector, int size, unsigned long count = 0; sector_t esector, nr_sectors; - /* This would be an empty REQ_FLUSH, be silent. */ + /* This would be an empty REQ_PREFLUSH, be silent. */ if ((mode == SET_OUT_OF_SYNC) && size == 0) return 0; - if (size <= 0 || !IS_ALIGNED(size, 512) || size > DRBD_MAX_DISCARD_SIZE) { + if (!plausible_request_size(size)) { drbd_err(device, "%s: sector=%llus size=%d nonsense!\n", drbd_change_sync_fname[mode], (unsigned long long)sector, size); diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c index 92d6fc020a65..ab62b81c2ca7 100644 --- a/drivers/block/drbd/drbd_bitmap.c +++ b/drivers/block/drbd/drbd_bitmap.c @@ -96,6 +96,13 @@ struct drbd_bitmap { struct page **bm_pages; spinlock_t bm_lock; + /* exclusively to be used by __al_write_transaction(), + * drbd_bm_mark_for_writeout() and + * and drbd_bm_write_hinted() -> bm_rw() called from there. + */ + unsigned int n_bitmap_hints; + unsigned int al_bitmap_hints[AL_UPDATES_PER_TRANSACTION]; + /* see LIMITATIONS: above */ unsigned long bm_set; /* nr of set bits; THINK maybe atomic_t? */ @@ -242,6 +249,11 @@ static void bm_set_page_need_writeout(struct page *page) set_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page)); } +void drbd_bm_reset_al_hints(struct drbd_device *device) +{ + device->bitmap->n_bitmap_hints = 0; +} + /** * drbd_bm_mark_for_writeout() - mark a page with a "hint" to be considered for writeout * @device: DRBD device. @@ -253,6 +265,7 @@ static void bm_set_page_need_writeout(struct page *page) */ void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr) { + struct drbd_bitmap *b = device->bitmap; struct page *page; if (page_nr >= device->bitmap->bm_number_of_pages) { drbd_warn(device, "BAD: page_nr: %u, number_of_pages: %u\n", @@ -260,7 +273,9 @@ void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr) return; } page = device->bitmap->bm_pages[page_nr]; - set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page)); + BUG_ON(b->n_bitmap_hints >= ARRAY_SIZE(b->al_bitmap_hints)); + if (!test_and_set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page))) + b->al_bitmap_hints[b->n_bitmap_hints++] = page_nr; } static int bm_test_page_unchanged(struct page *page) @@ -427,8 +442,7 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want) } /* - * called on driver init only. TODO call when a device is created. - * allocates the drbd_bitmap, and stores it in device->bitmap. + * allocates the drbd_bitmap and stores it in device->bitmap. */ int drbd_bm_init(struct drbd_device *device) { @@ -633,7 +647,8 @@ int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bi unsigned long bits, words, owords, obits; unsigned long want, have, onpages; /* number of pages */ struct page **npages, **opages = NULL; - int err = 0, growing; + int err = 0; + bool growing; if (!expect(b)) return -ENOMEM; @@ -980,7 +995,7 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho struct drbd_bitmap *b = device->bitmap; struct page *page; unsigned int len; - unsigned int rw = (ctx->flags & BM_AIO_READ) ? READ : WRITE; + unsigned int op = (ctx->flags & BM_AIO_READ) ? REQ_OP_READ : REQ_OP_WRITE; sector_t on_disk_sector = device->ldev->md.md_offset + device->ldev->md.bm_offset; @@ -1011,12 +1026,12 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho bio_add_page(bio, page, len, 0); bio->bi_private = ctx; bio->bi_end_io = drbd_bm_endio; + bio_set_op_attrs(bio, op, 0); - if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { - bio->bi_rw |= rw; + if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { bio_io_error(bio); } else { - submit_bio(rw, bio); + submit_bio(bio); /* this should not count as user activity and cause the * resync to throttle -- see drbd_rs_should_slow_down(). */ atomic_add(len >> 9, &device->rs_sect_ev); @@ -1030,7 +1045,7 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned { struct drbd_bm_aio_ctx *ctx; struct drbd_bitmap *b = device->bitmap; - int num_pages, i, count = 0; + unsigned int num_pages, i, count = 0; unsigned long now; char ppb[10]; int err = 0; @@ -1078,16 +1093,37 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned now = jiffies; /* let the layers below us try to merge these bios... */ - for (i = 0; i < num_pages; i++) { - /* ignore completely unchanged pages */ - if (lazy_writeout_upper_idx && i == lazy_writeout_upper_idx) - break; - if (!(flags & BM_AIO_READ)) { - if ((flags & BM_AIO_WRITE_HINTED) && - !test_and_clear_bit(BM_PAGE_HINT_WRITEOUT, - &page_private(b->bm_pages[i]))) - continue; + if (flags & BM_AIO_READ) { + for (i = 0; i < num_pages; i++) { + atomic_inc(&ctx->in_flight); + bm_page_io_async(ctx, i); + ++count; + cond_resched(); + } + } else if (flags & BM_AIO_WRITE_HINTED) { + /* ASSERT: BM_AIO_WRITE_ALL_PAGES is not set. */ + unsigned int hint; + for (hint = 0; hint < b->n_bitmap_hints; hint++) { + i = b->al_bitmap_hints[hint]; + if (i >= num_pages) /* == -1U: no hint here. */ + continue; + /* Several AL-extents may point to the same page. */ + if (!test_and_clear_bit(BM_PAGE_HINT_WRITEOUT, + &page_private(b->bm_pages[i]))) + continue; + /* Has it even changed? */ + if (bm_test_page_unchanged(b->bm_pages[i])) + continue; + atomic_inc(&ctx->in_flight); + bm_page_io_async(ctx, i); + ++count; + } + } else { + for (i = 0; i < num_pages; i++) { + /* ignore completely unchanged pages */ + if (lazy_writeout_upper_idx && i == lazy_writeout_upper_idx) + break; if (!(flags & BM_AIO_WRITE_ALL_PAGES) && bm_test_page_unchanged(b->bm_pages[i])) { dynamic_drbd_dbg(device, "skipped bm write for idx %u\n", i); @@ -1100,11 +1136,11 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned dynamic_drbd_dbg(device, "skipped bm lazy write for idx %u\n", i); continue; } + atomic_inc(&ctx->in_flight); + bm_page_io_async(ctx, i); + ++count; + cond_resched(); } - atomic_inc(&ctx->in_flight); - bm_page_io_async(ctx, i); - ++count; - cond_resched(); } /* @@ -1121,10 +1157,14 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy); /* summary for global bitmap IO */ - if (flags == 0) - drbd_info(device, "bitmap %s of %u pages took %lu jiffies\n", - (flags & BM_AIO_READ) ? "READ" : "WRITE", - count, jiffies - now); + if (flags == 0) { + unsigned int ms = jiffies_to_msecs(jiffies - now); + if (ms > 5) { + drbd_info(device, "bitmap %s of %u pages took %u ms\n", + (flags & BM_AIO_READ) ? "READ" : "WRITE", + count, ms); + } + } if (ctx->error) { drbd_alert(device, "we had at least one MD IO ERROR during bitmap IO\n"); diff --git a/drivers/block/drbd/drbd_debugfs.c b/drivers/block/drbd/drbd_debugfs.c index 4de95bbff486..be91a8d7c22a 100644 --- a/drivers/block/drbd/drbd_debugfs.c +++ b/drivers/block/drbd/drbd_debugfs.c @@ -237,14 +237,9 @@ static void seq_print_peer_request_flags(struct seq_file *m, struct drbd_peer_re seq_print_rq_state_bit(m, f & EE_SEND_WRITE_ACK, &sep, "C"); seq_print_rq_state_bit(m, f & EE_MAY_SET_IN_SYNC, &sep, "set-in-sync"); - if (f & EE_IS_TRIM) { - seq_putc(m, sep); - sep = '|'; - if (f & EE_IS_TRIM_USE_ZEROOUT) - seq_puts(m, "zero-out"); - else - seq_puts(m, "trim"); - } + if (f & EE_IS_TRIM) + __seq_print_rq_state_bit(m, f & EE_IS_TRIM_USE_ZEROOUT, &sep, "zero-out", "trim"); + seq_print_rq_state_bit(m, f & EE_WRITE_SAME, &sep, "write-same"); seq_putc(m, '\n'); } @@ -908,7 +903,7 @@ static int drbd_version_open(struct inode *inode, struct file *file) return single_open(file, drbd_version_show, NULL); } -static struct file_operations drbd_version_fops = { +static const struct file_operations drbd_version_fops = { .owner = THIS_MODULE, .open = drbd_version_open, .llseek = seq_lseek, diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index 7a1cf7eaa71d..7b54354976a5 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -468,9 +468,15 @@ enum { /* this is/was a write request */ __EE_WRITE, + /* this is/was a write same request */ + __EE_WRITE_SAME, + /* this originates from application on peer * (not some resync or verify or other DRBD internal request) */ __EE_APPLICATION, + + /* If it contains only 0 bytes, send back P_RS_DEALLOCATED */ + __EE_RS_THIN_REQ, }; #define EE_CALL_AL_COMPLETE_IO (1<<__EE_CALL_AL_COMPLETE_IO) #define EE_MAY_SET_IN_SYNC (1<<__EE_MAY_SET_IN_SYNC) @@ -484,7 +490,9 @@ enum { #define EE_IN_INTERVAL_TREE (1<<__EE_IN_INTERVAL_TREE) #define EE_SUBMITTED (1<<__EE_SUBMITTED) #define EE_WRITE (1<<__EE_WRITE) +#define EE_WRITE_SAME (1<<__EE_WRITE_SAME) #define EE_APPLICATION (1<<__EE_APPLICATION) +#define EE_RS_THIN_REQ (1<<__EE_RS_THIN_REQ) /* flag bits per device */ enum { @@ -1123,6 +1131,7 @@ extern int drbd_send_ov_request(struct drbd_peer_device *, sector_t sector, int extern int drbd_send_bitmap(struct drbd_device *device); extern void drbd_send_sr_reply(struct drbd_peer_device *, enum drbd_state_rv retcode); extern void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode); +extern int drbd_send_rs_deallocated(struct drbd_peer_device *, struct drbd_peer_request *); extern void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev); extern void drbd_device_cleanup(struct drbd_device *device); void drbd_print_uuids(struct drbd_device *device, const char *text); @@ -1327,14 +1336,14 @@ struct bm_extent { #endif #endif -/* BIO_MAX_SIZE is 256 * PAGE_SIZE, +/* Estimate max bio size as 256 * PAGE_SIZE, * so for typical PAGE_SIZE of 4k, that is (1<<20) Byte. * Since we may live in a mixed-platform cluster, * we limit us to a platform agnostic constant here for now. * A followup commit may allow even bigger BIO sizes, * once we thought that through. */ #define DRBD_MAX_BIO_SIZE (1U << 20) -#if DRBD_MAX_BIO_SIZE > BIO_MAX_SIZE +#if DRBD_MAX_BIO_SIZE > (BIO_MAX_PAGES << PAGE_SHIFT) #error Architecture not supported: DRBD_MAX_BIO_SIZE > BIO_MAX_SIZE #endif #define DRBD_MAX_BIO_SIZE_SAFE (1U << 12) /* Works always = 4k */ @@ -1342,11 +1351,11 @@ struct bm_extent { #define DRBD_MAX_SIZE_H80_PACKET (1U << 15) /* Header 80 only allows packets up to 32KiB data */ #define DRBD_MAX_BIO_SIZE_P95 (1U << 17) /* Protocol 95 to 99 allows bios up to 128KiB */ -/* For now, don't allow more than one activity log extent worth of data - * to be discarded in one go. We may need to rework drbd_al_begin_io() - * to allow for even larger discard ranges */ -#define DRBD_MAX_DISCARD_SIZE AL_EXTENT_SIZE -#define DRBD_MAX_DISCARD_SECTORS (DRBD_MAX_DISCARD_SIZE >> 9) +/* For now, don't allow more than half of what we can "activate" in one + * activity log transaction to be discarded in one go. We may need to rework + * drbd_al_begin_io() to allow for even larger discard ranges */ +#define DRBD_MAX_BATCH_BIO_SIZE (AL_UPDATES_PER_TRANSACTION/2*AL_EXTENT_SIZE) +#define DRBD_MAX_BBIO_SECTORS (DRBD_MAX_BATCH_BIO_SIZE >> 9) extern int drbd_bm_init(struct drbd_device *device); extern int drbd_bm_resize(struct drbd_device *device, sector_t sectors, int set_new_bits); @@ -1369,6 +1378,7 @@ extern int drbd_bm_e_weight(struct drbd_device *device, unsigned long enr); extern int drbd_bm_read(struct drbd_device *device) __must_hold(local); extern void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr); extern int drbd_bm_write(struct drbd_device *device) __must_hold(local); +extern void drbd_bm_reset_al_hints(struct drbd_device *device) __must_hold(local); extern int drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local); extern int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_hold(local); extern int drbd_bm_write_all(struct drbd_device *device) __must_hold(local); @@ -1483,12 +1493,14 @@ enum determine_dev_size { extern enum determine_dev_size drbd_determine_dev_size(struct drbd_device *, enum dds_flags, struct resize_parms *) __must_hold(local); extern void resync_after_online_grow(struct drbd_device *); -extern void drbd_reconsider_max_bio_size(struct drbd_device *device, struct drbd_backing_dev *bdev); +extern void drbd_reconsider_queue_parameters(struct drbd_device *device, + struct drbd_backing_dev *bdev, struct o_qlim *o); extern enum drbd_state_rv drbd_set_role(struct drbd_device *device, enum drbd_role new_role, int force); extern bool conn_try_outdate_peer(struct drbd_connection *connection); extern void conn_try_outdate_peer_async(struct drbd_connection *connection); +extern enum drbd_peer_state conn_khelper(struct drbd_connection *connection, char *cmd); extern int drbd_khelper(struct drbd_device *device, char *cmd); /* drbd_worker.c */ @@ -1507,7 +1519,7 @@ extern int drbd_resync_finished(struct drbd_device *device); extern void *drbd_md_get_buffer(struct drbd_device *device, const char *intent); extern void drbd_md_put_buffer(struct drbd_device *device); extern int drbd_md_sync_page_io(struct drbd_device *device, - struct drbd_backing_dev *bdev, sector_t sector, int rw); + struct drbd_backing_dev *bdev, sector_t sector, int op); extern void drbd_ov_out_of_sync_found(struct drbd_device *, sector_t, int); extern void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_backing_dev *bdev, unsigned int *done); @@ -1548,6 +1560,8 @@ extern void start_resync_timer_fn(unsigned long data); extern void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req); /* drbd_receiver.c */ +extern int drbd_issue_discard_or_zero_out(struct drbd_device *device, + sector_t start, unsigned int nr_sectors, bool discard); extern int drbd_receiver(struct drbd_thread *thi); extern int drbd_ack_receiver(struct drbd_thread *thi); extern void drbd_send_ping_wf(struct work_struct *ws); @@ -1557,11 +1571,11 @@ extern bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector bool throttle_if_app_is_waiting); extern int drbd_submit_peer_request(struct drbd_device *, struct drbd_peer_request *, const unsigned, - const int); + const unsigned, const int); extern int drbd_free_peer_reqs(struct drbd_device *, struct list_head *); extern struct drbd_peer_request *drbd_alloc_peer_req(struct drbd_peer_device *, u64, sector_t, unsigned int, - bool, + unsigned int, gfp_t) __must_hold(local); extern void __drbd_free_peer_req(struct drbd_device *, struct drbd_peer_request *, int); @@ -1635,8 +1649,6 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin /* drbd_proc.c */ extern struct proc_dir_entry *drbd_proc; extern const struct file_operations drbd_proc_fops; -extern const char *drbd_conn_str(enum drbd_conns s); -extern const char *drbd_role_str(enum drbd_role s); /* drbd_actlog.c */ extern bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i); @@ -2095,13 +2107,22 @@ static inline void _sub_unacked(struct drbd_device *device, int n, const char *f ERR_IF_CNT_IS_NEGATIVE(unacked_cnt, func, line); } +static inline bool is_sync_target_state(enum drbd_conns connection_state) +{ + return connection_state == C_SYNC_TARGET || + connection_state == C_PAUSED_SYNC_T; +} + +static inline bool is_sync_source_state(enum drbd_conns connection_state) +{ + return connection_state == C_SYNC_SOURCE || + connection_state == C_PAUSED_SYNC_S; +} + static inline bool is_sync_state(enum drbd_conns connection_state) { - return - (connection_state == C_SYNC_SOURCE - || connection_state == C_SYNC_TARGET - || connection_state == C_PAUSED_SYNC_S - || connection_state == C_PAUSED_SYNC_T); + return is_sync_source_state(connection_state) || + is_sync_target_state(connection_state); } /** diff --git a/drivers/block/drbd/drbd_interval.h b/drivers/block/drbd/drbd_interval.h index f210543f05f4..23c5a94428d2 100644 --- a/drivers/block/drbd/drbd_interval.h +++ b/drivers/block/drbd/drbd_interval.h @@ -6,13 +6,13 @@ struct drbd_interval { struct rb_node rb; - sector_t sector; /* start sector of the interval */ - unsigned int size; /* size in bytes */ - sector_t end; /* highest interval end in subtree */ - int local:1 /* local or remote request? */; - int waiting:1; /* someone is waiting for this to complete */ - int completed:1; /* this has been completed already; - * ignore for conflict detection */ + sector_t sector; /* start sector of the interval */ + unsigned int size; /* size in bytes */ + sector_t end; /* highest interval end in subtree */ + unsigned int local:1 /* local or remote request? */; + unsigned int waiting:1; /* someone is waiting for completion */ + unsigned int completed:1; /* this has been completed already; + * ignore for conflict detection */ }; static inline void drbd_clear_interval(struct drbd_interval *i) diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 2ba1494b2799..0501ae0c517b 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include #include @@ -920,6 +920,31 @@ void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device) } } +/* communicated if (agreed_features & DRBD_FF_WSAME) */ +void assign_p_sizes_qlim(struct drbd_device *device, struct p_sizes *p, struct request_queue *q) +{ + if (q) { + p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q)); + p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q)); + p->qlim->alignment_offset = cpu_to_be32(queue_alignment_offset(q)); + p->qlim->io_min = cpu_to_be32(queue_io_min(q)); + p->qlim->io_opt = cpu_to_be32(queue_io_opt(q)); + p->qlim->discard_enabled = blk_queue_discard(q); + p->qlim->discard_zeroes_data = queue_discard_zeroes_data(q); + p->qlim->write_same_capable = !!q->limits.max_write_same_sectors; + } else { + q = device->rq_queue; + p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q)); + p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q)); + p->qlim->alignment_offset = 0; + p->qlim->io_min = cpu_to_be32(queue_io_min(q)); + p->qlim->io_opt = cpu_to_be32(queue_io_opt(q)); + p->qlim->discard_enabled = 0; + p->qlim->discard_zeroes_data = 0; + p->qlim->write_same_capable = 0; + } +} + int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags) { struct drbd_device *device = peer_device->device; @@ -928,29 +953,37 @@ int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enu sector_t d_size, u_size; int q_order_type; unsigned int max_bio_size; + unsigned int packet_size; + + sock = &peer_device->connection->data; + p = drbd_prepare_command(peer_device, sock); + if (!p) + return -EIO; + packet_size = sizeof(*p); + if (peer_device->connection->agreed_features & DRBD_FF_WSAME) + packet_size += sizeof(p->qlim[0]); + + memset(p, 0, packet_size); if (get_ldev_if_state(device, D_NEGOTIATING)) { - D_ASSERT(device, device->ldev->backing_bdev); + struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev); d_size = drbd_get_max_capacity(device->ldev); rcu_read_lock(); u_size = rcu_dereference(device->ldev->disk_conf)->disk_size; rcu_read_unlock(); q_order_type = drbd_queue_order_type(device); - max_bio_size = queue_max_hw_sectors(device->ldev->backing_bdev->bd_disk->queue) << 9; + max_bio_size = queue_max_hw_sectors(q) << 9; max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE); + assign_p_sizes_qlim(device, p, q); put_ldev(device); } else { d_size = 0; u_size = 0; q_order_type = QUEUE_ORDERED_NONE; max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */ + assign_p_sizes_qlim(device, p, NULL); } - sock = &peer_device->connection->data; - p = drbd_prepare_command(peer_device, sock); - if (!p) - return -EIO; - if (peer_device->connection->agreed_pro_version <= 94) max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET); else if (peer_device->connection->agreed_pro_version < 100) @@ -962,7 +995,8 @@ int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enu p->max_bio_size = cpu_to_be32(max_bio_size); p->queue_order_type = cpu_to_be16(q_order_type); p->dds_flags = cpu_to_be16(flags); - return drbd_send_command(peer_device, sock, P_SIZES, sizeof(*p), NULL, 0); + + return drbd_send_command(peer_device, sock, P_SIZES, packet_size, NULL, 0); } /** @@ -1377,6 +1411,22 @@ int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd, cpu_to_be64(block_id)); } +int drbd_send_rs_deallocated(struct drbd_peer_device *peer_device, + struct drbd_peer_request *peer_req) +{ + struct drbd_socket *sock; + struct p_block_desc *p; + + sock = &peer_device->connection->data; + p = drbd_prepare_command(peer_device, sock); + if (!p) + return -EIO; + p->sector = cpu_to_be64(peer_req->i.sector); + p->blksize = cpu_to_be32(peer_req->i.size); + p->pad = 0; + return drbd_send_command(peer_device, sock, P_RS_DEALLOCATED, sizeof(*p), NULL, 0); +} + int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd, sector_t sector, int size, u64 block_id) { @@ -1561,6 +1611,9 @@ static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio) ? 0 : MSG_MORE); if (err) return err; + /* REQ_OP_WRITE_SAME has only one segment */ + if (bio_op(bio) == REQ_OP_WRITE_SAME) + break; } return 0; } @@ -1579,6 +1632,9 @@ static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *b bio_iter_last(bvec, iter) ? 0 : MSG_MORE); if (err) return err; + /* REQ_OP_WRITE_SAME has only one segment */ + if (bio_op(bio) == REQ_OP_WRITE_SAME) + break; } return 0; } @@ -1603,15 +1659,17 @@ static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device, return 0; } -static u32 bio_flags_to_wire(struct drbd_connection *connection, unsigned long bi_rw) +static u32 bio_flags_to_wire(struct drbd_connection *connection, + struct bio *bio) { if (connection->agreed_pro_version >= 95) - return (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) | - (bi_rw & REQ_FUA ? DP_FUA : 0) | - (bi_rw & REQ_FLUSH ? DP_FLUSH : 0) | - (bi_rw & REQ_DISCARD ? DP_DISCARD : 0); + return (bio->bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) | + (bio->bi_rw & REQ_FUA ? DP_FUA : 0) | + (bio->bi_rw & REQ_PREFLUSH ? DP_FLUSH : 0) | + (bio_op(bio) == REQ_OP_WRITE_SAME ? DP_WSAME : 0) | + (bio_op(bio) == REQ_OP_DISCARD ? DP_DISCARD : 0); else - return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0; + return bio->bi_rw & REQ_SYNC ? DP_RW_SYNC : 0; } /* Used to send write or TRIM aka REQ_DISCARD requests @@ -1622,6 +1680,8 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request * struct drbd_device *device = peer_device->device; struct drbd_socket *sock; struct p_data *p; + struct p_wsame *wsame = NULL; + void *digest_out; unsigned int dp_flags = 0; int digest_size; int err; @@ -1636,7 +1696,7 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request * p->sector = cpu_to_be64(req->i.sector); p->block_id = (unsigned long)req; p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq)); - dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio->bi_rw); + dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio); if (device->state.conn >= C_SYNC_SOURCE && device->state.conn <= C_PAUSED_SYNC_T) dp_flags |= DP_MAY_SET_IN_SYNC; @@ -1657,12 +1717,29 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request * err = __send_command(peer_device->connection, device->vnr, sock, P_TRIM, sizeof(*t), NULL, 0); goto out; } + if (dp_flags & DP_WSAME) { + /* this will only work if DRBD_FF_WSAME is set AND the + * handshake agreed that all nodes and backend devices are + * WRITE_SAME capable and agree on logical_block_size */ + wsame = (struct p_wsame*)p; + digest_out = wsame + 1; + wsame->size = cpu_to_be32(req->i.size); + } else + digest_out = p + 1; /* our digest is still only over the payload. * TRIM does not carry any payload. */ if (digest_size) - drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, p + 1); - err = __send_command(peer_device->connection, device->vnr, sock, P_DATA, sizeof(*p) + digest_size, NULL, req->i.size); + drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest_out); + if (wsame) { + err = + __send_command(peer_device->connection, device->vnr, sock, P_WSAME, + sizeof(*wsame) + digest_size, NULL, + bio_iovec(req->master_bio).bv_len); + } else + err = + __send_command(peer_device->connection, device->vnr, sock, P_DATA, + sizeof(*p) + digest_size, NULL, req->i.size); if (!err) { /* For protocol A, we have to memcpy the payload into * socket buffers, as we may complete right away @@ -3061,7 +3138,7 @@ void drbd_md_write(struct drbd_device *device, void *b) D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset); sector = device->ldev->md.md_offset; - if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) { + if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) { /* this was a try anyways ... */ drbd_err(device, "meta data update failed!\n"); drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR); @@ -3263,7 +3340,8 @@ int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev) * Affects the paranoia out-of-range access check in drbd_md_sync_page_io(). */ bdev->md.md_size_sect = 8; - if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, READ)) { + if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, + REQ_OP_READ)) { /* NOTE: can't do normal error processing here as this is called BEFORE disk is attached */ drbd_err(device, "Error while reading metadata.\n"); @@ -3505,7 +3583,12 @@ static int w_bitmap_io(struct drbd_work *w, int unused) struct bm_io_work *work = &device->bm_io_work; int rv = -EIO; - D_ASSERT(device, atomic_read(&device->ap_bio_cnt) == 0); + if (work->flags != BM_LOCKED_CHANGE_ALLOWED) { + int cnt = atomic_read(&device->ap_bio_cnt); + if (cnt) + drbd_err(device, "FIXME: ap_bio_cnt %d, expected 0; queued for '%s'\n", + cnt, work->why); + } if (get_ldev(device)) { drbd_bm_lock(device, work->why, work->flags); @@ -3585,18 +3668,20 @@ void drbd_queue_bitmap_io(struct drbd_device *device, int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *), char *why, enum bm_flag flags) { + /* Only suspend io, if some operation is supposed to be locked out */ + const bool do_suspend_io = flags & (BM_DONT_CLEAR|BM_DONT_SET|BM_DONT_TEST); int rv; D_ASSERT(device, current != first_peer_device(device)->connection->worker.task); - if ((flags & BM_LOCKED_SET_ALLOWED) == 0) + if (do_suspend_io) drbd_suspend_io(device); drbd_bm_lock(device, why, flags); rv = io_fn(device); drbd_bm_unlock(device); - if ((flags & BM_LOCKED_SET_ALLOWED) == 0) + if (do_suspend_io) drbd_resume_io(device); return rv; @@ -3635,6 +3720,8 @@ const char *cmdname(enum drbd_packet cmd) * one PRO_VERSION */ static const char *cmdnames[] = { [P_DATA] = "Data", + [P_WSAME] = "WriteSame", + [P_TRIM] = "Trim", [P_DATA_REPLY] = "DataReply", [P_RS_DATA_REPLY] = "RSDataReply", [P_BARRIER] = "Barrier", @@ -3679,6 +3766,8 @@ const char *cmdname(enum drbd_packet cmd) [P_CONN_ST_CHG_REPLY] = "conn_st_chg_reply", [P_RETRY_WRITE] = "retry_write", [P_PROTOCOL_UPDATE] = "protocol_update", + [P_RS_THIN_REQ] = "rs_thin_req", + [P_RS_DEALLOCATED] = "rs_deallocated", /* enum drbd_packet, but not commands - obsoleted flags: * P_MAY_IGNORE diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 0bac9c8246bc..f35db29cac76 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -343,7 +343,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd) (char[20]) { }, /* address family */ (char[60]) { }, /* address */ NULL }; - char mb[12]; + char mb[14]; char *argv[] = {usermode_helper, cmd, mb, NULL }; struct drbd_connection *connection = first_peer_device(device)->connection; struct sib_info sib; @@ -352,7 +352,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd) if (current == connection->worker.task) set_bit(CALLBACK_PENDING, &connection->flags); - snprintf(mb, 12, "minor-%d", device_to_minor(device)); + snprintf(mb, 14, "minor-%d", device_to_minor(device)); setup_khelper_env(connection, envp); /* The helper may take some time. @@ -387,7 +387,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd) return ret; } -static int conn_khelper(struct drbd_connection *connection, char *cmd) +enum drbd_peer_state conn_khelper(struct drbd_connection *connection, char *cmd) { char *envp[] = { "HOME=/", "TERM=linux", @@ -442,19 +442,17 @@ static enum drbd_fencing_p highest_fencing_policy(struct drbd_connection *connec } rcu_read_unlock(); - if (fp == FP_NOT_AVAIL) { - /* IO Suspending works on the whole resource. - Do it only for one device. */ - vnr = 0; - peer_device = idr_get_next(&connection->peer_devices, &vnr); - drbd_change_state(peer_device->device, CS_VERBOSE | CS_HARD, NS(susp_fen, 0)); - } - return fp; } +static bool resource_is_supended(struct drbd_resource *resource) +{ + return resource->susp || resource->susp_fen || resource->susp_nod; +} + bool conn_try_outdate_peer(struct drbd_connection *connection) { + struct drbd_resource * const resource = connection->resource; unsigned int connect_cnt; union drbd_state mask = { }; union drbd_state val = { }; @@ -462,21 +460,41 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) char *ex_to_string; int r; - spin_lock_irq(&connection->resource->req_lock); + spin_lock_irq(&resource->req_lock); if (connection->cstate >= C_WF_REPORT_PARAMS) { drbd_err(connection, "Expected cstate < C_WF_REPORT_PARAMS\n"); - spin_unlock_irq(&connection->resource->req_lock); + spin_unlock_irq(&resource->req_lock); return false; } connect_cnt = connection->connect_cnt; - spin_unlock_irq(&connection->resource->req_lock); + spin_unlock_irq(&resource->req_lock); fp = highest_fencing_policy(connection); switch (fp) { case FP_NOT_AVAIL: drbd_warn(connection, "Not fencing peer, I'm not even Consistent myself.\n"); - goto out; + spin_lock_irq(&resource->req_lock); + if (connection->cstate < C_WF_REPORT_PARAMS) { + _conn_request_state(connection, + (union drbd_state) { { .susp_fen = 1 } }, + (union drbd_state) { { .susp_fen = 0 } }, + CS_VERBOSE | CS_HARD | CS_DC_SUSP); + /* We are no longer suspended due to the fencing policy. + * We may still be suspended due to the on-no-data-accessible policy. + * If that was OND_IO_ERROR, fail pending requests. */ + if (!resource_is_supended(resource)) + _tl_restart(connection, CONNECTION_LOST_WHILE_PENDING); + } + /* Else: in case we raced with a connection handshake, + * let the handshake figure out if we maybe can RESEND, + * and do not resume/fail pending requests here. + * Worst case is we stay suspended for now, which may be + * resolved by either re-establishing the replication link, or + * the next link failure, or eventually the administrator. */ + spin_unlock_irq(&resource->req_lock); + return false; + case FP_DONT_CARE: return true; default: ; @@ -485,17 +503,17 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) r = conn_khelper(connection, "fence-peer"); switch ((r>>8) & 0xff) { - case 3: /* peer is inconsistent */ + case P_INCONSISTENT: /* peer is inconsistent */ ex_to_string = "peer is inconsistent or worse"; mask.pdsk = D_MASK; val.pdsk = D_INCONSISTENT; break; - case 4: /* peer got outdated, or was already outdated */ + case P_OUTDATED: /* peer got outdated, or was already outdated */ ex_to_string = "peer was fenced"; mask.pdsk = D_MASK; val.pdsk = D_OUTDATED; break; - case 5: /* peer was down */ + case P_DOWN: /* peer was down */ if (conn_highest_disk(connection) == D_UP_TO_DATE) { /* we will(have) create(d) a new UUID anyways... */ ex_to_string = "peer is unreachable, assumed to be dead"; @@ -505,7 +523,7 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) ex_to_string = "peer unreachable, doing nothing since disk != UpToDate"; } break; - case 6: /* Peer is primary, voluntarily outdate myself. + case P_PRIMARY: /* Peer is primary, voluntarily outdate myself. * This is useful when an unconnected R_SECONDARY is asked to * become R_PRIMARY, but finds the other peer being active. */ ex_to_string = "peer is active"; @@ -513,7 +531,9 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) mask.disk = D_MASK; val.disk = D_OUTDATED; break; - case 7: + case P_FENCING: + /* THINK: do we need to handle this + * like case 4, or more like case 5? */ if (fp != FP_STONITH) drbd_err(connection, "fence-peer() = 7 && fencing != Stonith !!!\n"); ex_to_string = "peer was stonithed"; @@ -529,13 +549,11 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) drbd_info(connection, "fence-peer helper returned %d (%s)\n", (r>>8) & 0xff, ex_to_string); - out: - /* Not using conn_request_state(connection, mask, val, CS_VERBOSE); here, because we might were able to re-establish the connection in the meantime. */ - spin_lock_irq(&connection->resource->req_lock); + spin_lock_irq(&resource->req_lock); if (connection->cstate < C_WF_REPORT_PARAMS && !test_bit(STATE_SENT, &connection->flags)) { if (connection->connect_cnt != connect_cnt) /* In case the connection was established and droped @@ -544,7 +562,7 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) else _conn_request_state(connection, mask, val, CS_VERBOSE); } - spin_unlock_irq(&connection->resource->req_lock); + spin_unlock_irq(&resource->req_lock); return conn_highest_pdsk(connection) <= D_OUTDATED; } @@ -1154,51 +1172,160 @@ static int drbd_check_al_size(struct drbd_device *device, struct disk_conf *dc) return 0; } +static void blk_queue_discard_granularity(struct request_queue *q, unsigned int granularity) +{ + q->limits.discard_granularity = granularity; +} + +static unsigned int drbd_max_discard_sectors(struct drbd_connection *connection) +{ + /* when we introduced REQ_WRITE_SAME support, we also bumped + * our maximum supported batch bio size used for discards. */ + if (connection->agreed_features & DRBD_FF_WSAME) + return DRBD_MAX_BBIO_SECTORS; + /* before, with DRBD <= 8.4.6, we only allowed up to one AL_EXTENT_SIZE. */ + return AL_EXTENT_SIZE >> 9; +} + +static void decide_on_discard_support(struct drbd_device *device, + struct request_queue *q, + struct request_queue *b, + bool discard_zeroes_if_aligned) +{ + /* q = drbd device queue (device->rq_queue) + * b = backing device queue (device->ldev->backing_bdev->bd_disk->queue), + * or NULL if diskless + */ + struct drbd_connection *connection = first_peer_device(device)->connection; + bool can_do = b ? blk_queue_discard(b) : true; + + if (can_do && b && !b->limits.discard_zeroes_data && !discard_zeroes_if_aligned) { + can_do = false; + drbd_info(device, "discard_zeroes_data=0 and discard_zeroes_if_aligned=no: disabling discards\n"); + } + if (can_do && connection->cstate >= C_CONNECTED && !(connection->agreed_features & DRBD_FF_TRIM)) { + can_do = false; + drbd_info(connection, "peer DRBD too old, does not support TRIM: disabling discards\n"); + } + if (can_do) { + /* We don't care for the granularity, really. + * Stacking limits below should fix it for the local + * device. Whether or not it is a suitable granularity + * on the remote device is not our problem, really. If + * you care, you need to use devices with similar + * topology on all peers. */ + blk_queue_discard_granularity(q, 512); + q->limits.max_discard_sectors = drbd_max_discard_sectors(connection); + queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); + } else { + queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); + blk_queue_discard_granularity(q, 0); + q->limits.max_discard_sectors = 0; + } +} + +static void fixup_discard_if_not_supported(struct request_queue *q) +{ + /* To avoid confusion, if this queue does not support discard, clear + * max_discard_sectors, which is what lsblk -D reports to the user. + * Older kernels got this wrong in "stack limits". + * */ + if (!blk_queue_discard(q)) { + blk_queue_max_discard_sectors(q, 0); + blk_queue_discard_granularity(q, 0); + } +} + +static void decide_on_write_same_support(struct drbd_device *device, + struct request_queue *q, + struct request_queue *b, struct o_qlim *o) +{ + struct drbd_peer_device *peer_device = first_peer_device(device); + struct drbd_connection *connection = peer_device->connection; + bool can_do = b ? b->limits.max_write_same_sectors : true; + + if (can_do && connection->cstate >= C_CONNECTED && !(connection->agreed_features & DRBD_FF_WSAME)) { + can_do = false; + drbd_info(peer_device, "peer does not support WRITE_SAME\n"); + } + + if (o) { + /* logical block size; queue_logical_block_size(NULL) is 512 */ + unsigned int peer_lbs = be32_to_cpu(o->logical_block_size); + unsigned int me_lbs_b = queue_logical_block_size(b); + unsigned int me_lbs = queue_logical_block_size(q); + + if (me_lbs_b != me_lbs) { + drbd_warn(device, + "logical block size of local backend does not match (drbd:%u, backend:%u); was this a late attach?\n", + me_lbs, me_lbs_b); + /* rather disable write same than trigger some BUG_ON later in the scsi layer. */ + can_do = false; + } + if (me_lbs_b != peer_lbs) { + drbd_warn(peer_device, "logical block sizes do not match (me:%u, peer:%u); this may cause problems.\n", + me_lbs, peer_lbs); + if (can_do) { + drbd_dbg(peer_device, "logical block size mismatch: WRITE_SAME disabled.\n"); + can_do = false; + } + me_lbs = max(me_lbs, me_lbs_b); + /* We cannot change the logical block size of an in-use queue. + * We can only hope that access happens to be properly aligned. + * If not, the peer will likely produce an IO error, and detach. */ + if (peer_lbs > me_lbs) { + if (device->state.role != R_PRIMARY) { + blk_queue_logical_block_size(q, peer_lbs); + drbd_warn(peer_device, "logical block size set to %u\n", peer_lbs); + } else { + drbd_warn(peer_device, + "current Primary must NOT adjust logical block size (%u -> %u); hope for the best.\n", + me_lbs, peer_lbs); + } + } + } + if (can_do && !o->write_same_capable) { + /* If we introduce an open-coded write-same loop on the receiving side, + * the peer would present itself as "capable". */ + drbd_dbg(peer_device, "WRITE_SAME disabled (peer device not capable)\n"); + can_do = false; + } + } + + blk_queue_max_write_same_sectors(q, can_do ? DRBD_MAX_BBIO_SECTORS : 0); +} + static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backing_dev *bdev, - unsigned int max_bio_size) + unsigned int max_bio_size, struct o_qlim *o) { struct request_queue * const q = device->rq_queue; unsigned int max_hw_sectors = max_bio_size >> 9; unsigned int max_segments = 0; struct request_queue *b = NULL; + struct disk_conf *dc; + bool discard_zeroes_if_aligned = true; if (bdev) { b = bdev->backing_bdev->bd_disk->queue; max_hw_sectors = min(queue_max_hw_sectors(b), max_bio_size >> 9); rcu_read_lock(); - max_segments = rcu_dereference(device->ldev->disk_conf)->max_bio_bvecs; + dc = rcu_dereference(device->ldev->disk_conf); + max_segments = dc->max_bio_bvecs; + discard_zeroes_if_aligned = dc->discard_zeroes_if_aligned; rcu_read_unlock(); blk_set_stacking_limits(&q->limits); - blk_queue_max_write_same_sectors(q, 0); } - blk_queue_logical_block_size(q, 512); blk_queue_max_hw_sectors(q, max_hw_sectors); /* This is the workaround for "bio would need to, but cannot, be split" */ blk_queue_max_segments(q, max_segments ? max_segments : BLK_MAX_SEGMENTS); blk_queue_segment_boundary(q, PAGE_SIZE-1); + decide_on_discard_support(device, q, b, discard_zeroes_if_aligned); + decide_on_write_same_support(device, q, b, o); if (b) { - struct drbd_connection *connection = first_peer_device(device)->connection; - - blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS); - - if (blk_queue_discard(b) && - (connection->cstate < C_CONNECTED || connection->agreed_features & FF_TRIM)) { - /* We don't care, stacking below should fix it for the local device. - * Whether or not it is a suitable granularity on the remote device - * is not our problem, really. If you care, you need to - * use devices with similar topology on all peers. */ - q->limits.discard_granularity = 512; - queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); - } else { - blk_queue_max_discard_sectors(q, 0); - queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); - q->limits.discard_granularity = 0; - } - blk_queue_stack_limits(q, b); if (q->backing_dev_info.ra_pages != b->backing_dev_info.ra_pages) { @@ -1208,15 +1335,10 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi q->backing_dev_info.ra_pages = b->backing_dev_info.ra_pages; } } - /* To avoid confusion, if this queue does not support discard, clear - * max_discard_sectors, which is what lsblk -D reports to the user. */ - if (!blk_queue_discard(q)) { - blk_queue_max_discard_sectors(q, 0); - q->limits.discard_granularity = 0; - } + fixup_discard_if_not_supported(q); } -void drbd_reconsider_max_bio_size(struct drbd_device *device, struct drbd_backing_dev *bdev) +void drbd_reconsider_queue_parameters(struct drbd_device *device, struct drbd_backing_dev *bdev, struct o_qlim *o) { unsigned int now, new, local, peer; @@ -1259,7 +1381,7 @@ void drbd_reconsider_max_bio_size(struct drbd_device *device, struct drbd_backin if (new != now) drbd_info(device, "max BIO size = %u\n", new); - drbd_setup_queue_param(device, bdev, new); + drbd_setup_queue_param(device, bdev, new, o); } /* Starts the worker thread */ @@ -1348,6 +1470,43 @@ static bool write_ordering_changed(struct disk_conf *a, struct disk_conf *b) a->disk_drain != b->disk_drain; } +static void sanitize_disk_conf(struct drbd_device *device, struct disk_conf *disk_conf, + struct drbd_backing_dev *nbc) +{ + struct request_queue * const q = nbc->backing_bdev->bd_disk->queue; + + if (disk_conf->al_extents < DRBD_AL_EXTENTS_MIN) + disk_conf->al_extents = DRBD_AL_EXTENTS_MIN; + if (disk_conf->al_extents > drbd_al_extents_max(nbc)) + disk_conf->al_extents = drbd_al_extents_max(nbc); + + if (!blk_queue_discard(q) + || (!q->limits.discard_zeroes_data && !disk_conf->discard_zeroes_if_aligned)) { + if (disk_conf->rs_discard_granularity) { + disk_conf->rs_discard_granularity = 0; /* disable feature */ + drbd_info(device, "rs_discard_granularity feature disabled\n"); + } + } + + if (disk_conf->rs_discard_granularity) { + int orig_value = disk_conf->rs_discard_granularity; + int remainder; + + if (q->limits.discard_granularity > disk_conf->rs_discard_granularity) + disk_conf->rs_discard_granularity = q->limits.discard_granularity; + + remainder = disk_conf->rs_discard_granularity % q->limits.discard_granularity; + disk_conf->rs_discard_granularity += remainder; + + if (disk_conf->rs_discard_granularity > q->limits.max_discard_sectors << 9) + disk_conf->rs_discard_granularity = q->limits.max_discard_sectors << 9; + + if (disk_conf->rs_discard_granularity != orig_value) + drbd_info(device, "rs_discard_granularity changed to %d\n", + disk_conf->rs_discard_granularity); + } +} + int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) { struct drbd_config_context adm_ctx; @@ -1395,10 +1554,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) if (!expect(new_disk_conf->resync_rate >= 1)) new_disk_conf->resync_rate = 1; - if (new_disk_conf->al_extents < DRBD_AL_EXTENTS_MIN) - new_disk_conf->al_extents = DRBD_AL_EXTENTS_MIN; - if (new_disk_conf->al_extents > drbd_al_extents_max(device->ldev)) - new_disk_conf->al_extents = drbd_al_extents_max(device->ldev); + sanitize_disk_conf(device, new_disk_conf, device->ldev); if (new_disk_conf->c_plan_ahead > DRBD_C_PLAN_AHEAD_MAX) new_disk_conf->c_plan_ahead = DRBD_C_PLAN_AHEAD_MAX; @@ -1457,6 +1613,9 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) if (write_ordering_changed(old_disk_conf, new_disk_conf)) drbd_bump_write_ordering(device->resource, NULL, WO_BDEV_FLUSH); + if (old_disk_conf->discard_zeroes_if_aligned != new_disk_conf->discard_zeroes_if_aligned) + drbd_reconsider_queue_parameters(device, device->ldev, NULL); + drbd_md_sync(device); if (device->state.conn >= C_CONNECTED) { @@ -1693,10 +1852,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) if (retcode != NO_ERROR) goto fail; - if (new_disk_conf->al_extents < DRBD_AL_EXTENTS_MIN) - new_disk_conf->al_extents = DRBD_AL_EXTENTS_MIN; - if (new_disk_conf->al_extents > drbd_al_extents_max(nbc)) - new_disk_conf->al_extents = drbd_al_extents_max(nbc); + sanitize_disk_conf(device, new_disk_conf, nbc); if (drbd_get_max_capacity(nbc) < new_disk_conf->disk_size) { drbd_err(device, "max capacity %llu smaller than disk size %llu\n", @@ -1838,7 +1994,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) device->read_cnt = 0; device->writ_cnt = 0; - drbd_reconsider_max_bio_size(device, device->ldev); + drbd_reconsider_queue_parameters(device, device->ldev, NULL); /* If I am currently not R_PRIMARY, * but meta data primary indicator is set, diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c index 6537b25db9c1..be2b93fd2c11 100644 --- a/drivers/block/drbd/drbd_proc.c +++ b/drivers/block/drbd/drbd_proc.c @@ -25,7 +25,7 @@ #include -#include +#include #include #include #include @@ -122,18 +122,18 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se x = res/50; y = 20-x; - seq_printf(seq, "\t["); + seq_puts(seq, "\t["); for (i = 1; i < x; i++) - seq_printf(seq, "="); - seq_printf(seq, ">"); + seq_putc(seq, '='); + seq_putc(seq, '>'); for (i = 0; i < y; i++) seq_printf(seq, "."); - seq_printf(seq, "] "); + seq_puts(seq, "] "); if (state.conn == C_VERIFY_S || state.conn == C_VERIFY_T) - seq_printf(seq, "verified:"); + seq_puts(seq, "verified:"); else - seq_printf(seq, "sync'ed:"); + seq_puts(seq, "sync'ed:"); seq_printf(seq, "%3u.%u%% ", res / 10, res % 10); /* if more than a few GB, display in MB */ @@ -146,7 +146,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se (unsigned long) Bit2KB(rs_left), (unsigned long) Bit2KB(rs_total)); - seq_printf(seq, "\n\t"); + seq_puts(seq, "\n\t"); /* see drivers/md/md.c * We do not want to overflow, so the order of operands and @@ -175,9 +175,9 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se rt / 3600, (rt % 3600) / 60, rt % 60); dbdt = Bit2KB(db/dt); - seq_printf(seq, " speed: "); + seq_puts(seq, " speed: "); seq_printf_with_thousands_grouping(seq, dbdt); - seq_printf(seq, " ("); + seq_puts(seq, " ("); /* ------------------------- ~3s average ------------------------ */ if (proc_details >= 1) { /* this is what drbd_rs_should_slow_down() uses */ @@ -188,7 +188,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se db = device->rs_mark_left[i] - rs_left; dbdt = Bit2KB(db/dt); seq_printf_with_thousands_grouping(seq, dbdt); - seq_printf(seq, " -- "); + seq_puts(seq, " -- "); } /* --------------------- long term average ---------------------- */ @@ -200,11 +200,11 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se db = rs_total - rs_left; dbdt = Bit2KB(db/dt); seq_printf_with_thousands_grouping(seq, dbdt); - seq_printf(seq, ")"); + seq_putc(seq, ')'); if (state.conn == C_SYNC_TARGET || state.conn == C_VERIFY_S) { - seq_printf(seq, " want: "); + seq_puts(seq, " want: "); seq_printf_with_thousands_grouping(seq, device->c_sync_rate); } seq_printf(seq, " K/sec%s\n", stalled ? " (stalled)" : ""); @@ -231,7 +231,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se (unsigned long long)bm_bits * BM_SECT_PER_BIT); if (stop_sector != 0 && stop_sector != ULLONG_MAX) seq_printf(seq, " stop sector: %llu", stop_sector); - seq_printf(seq, "\n"); + seq_putc(seq, '\n'); } } @@ -276,7 +276,7 @@ static int drbd_seq_show(struct seq_file *seq, void *v) rcu_read_lock(); idr_for_each_entry(&drbd_devices, device, i) { if (prev_i != i - 1) - seq_printf(seq, "\n"); + seq_putc(seq, '\n'); prev_i = i; state = device->state; diff --git a/drivers/block/drbd/drbd_protocol.h b/drivers/block/drbd/drbd_protocol.h index ef9245363dcc..4d296800f706 100644 --- a/drivers/block/drbd/drbd_protocol.h +++ b/drivers/block/drbd/drbd_protocol.h @@ -60,6 +60,15 @@ enum drbd_packet { * which is why I chose TRIM here, to disambiguate. */ P_TRIM = 0x31, + /* Only use these two if both support FF_THIN_RESYNC */ + P_RS_THIN_REQ = 0x32, /* Request a block for resync or reply P_RS_DEALLOCATED */ + P_RS_DEALLOCATED = 0x33, /* Contains only zeros on sync source node */ + + /* REQ_WRITE_SAME. + * On a receiving side without REQ_WRITE_SAME, + * we may fall back to an opencoded loop instead. */ + P_WSAME = 0x34, + P_MAY_IGNORE = 0x100, /* Flag to test if (cmd > P_MAY_IGNORE) ... */ P_MAX_OPT_CMD = 0x101, @@ -106,16 +115,20 @@ struct p_header100 { u32 pad; } __packed; -/* these defines must not be changed without changing the protocol version */ -#define DP_HARDBARRIER 1 /* depricated */ +/* These defines must not be changed without changing the protocol version. + * New defines may only be introduced together with protocol version bump or + * new protocol feature flags. + */ +#define DP_HARDBARRIER 1 /* no longer used */ #define DP_RW_SYNC 2 /* equals REQ_SYNC */ #define DP_MAY_SET_IN_SYNC 4 #define DP_UNPLUG 8 /* not used anymore */ #define DP_FUA 16 /* equals REQ_FUA */ -#define DP_FLUSH 32 /* equals REQ_FLUSH */ +#define DP_FLUSH 32 /* equals REQ_PREFLUSH */ #define DP_DISCARD 64 /* equals REQ_DISCARD */ #define DP_SEND_RECEIVE_ACK 128 /* This is a proto B write request */ #define DP_SEND_WRITE_ACK 256 /* This is a proto C write request */ +#define DP_WSAME 512 /* equiv. REQ_WRITE_SAME */ struct p_data { u64 sector; /* 64 bits sector number */ @@ -129,6 +142,11 @@ struct p_trim { u32 size; /* == bio->bi_size */ } __packed; +struct p_wsame { + struct p_data p_data; + u32 size; /* == bio->bi_size */ +} __packed; + /* * commands which share a struct: * p_block_ack: @@ -160,7 +178,23 @@ struct p_block_req { * ReportParams */ -#define FF_TRIM 1 +/* supports TRIM/DISCARD on the "wire" protocol */ +#define DRBD_FF_TRIM 1 + +/* Detect all-zeros during resync, and rather TRIM/UNMAP/DISCARD those blocks + * instead of fully allocate a supposedly thin volume on initial resync */ +#define DRBD_FF_THIN_RESYNC 2 + +/* supports REQ_WRITE_SAME on the "wire" protocol. + * Note: this flag is overloaded, + * its presence also + * - indicates support for 128 MiB "batch bios", + * max discard size of 128 MiB + * instead of 4M before that. + * - indicates that we exchange additional settings in p_sizes + * drbd_send_sizes()/receive_sizes() + */ +#define DRBD_FF_WSAME 4 struct p_connection_features { u32 protocol_min; @@ -235,6 +269,40 @@ struct p_rs_uuid { u64 uuid; } __packed; +/* optional queue_limits if (agreed_features & DRBD_FF_WSAME) + * see also struct queue_limits, as of late 2015 */ +struct o_qlim { + /* we don't need it yet, but we may as well communicate it now */ + u32 physical_block_size; + + /* so the original in struct queue_limits is unsigned short, + * but I'd have to put in padding anyways. */ + u32 logical_block_size; + + /* One incoming bio becomes one DRBD request, + * which may be translated to several bio on the receiving side. + * We don't need to communicate chunk/boundary/segment ... limits. + */ + + /* various IO hints may be useful with "diskless client" setups */ + u32 alignment_offset; + u32 io_min; + u32 io_opt; + + /* We may need to communicate integrity stuff at some point, + * but let's not get ahead of ourselves. */ + + /* Backend discard capabilities. + * Receiving side uses "blkdev_issue_discard()", no need to communicate + * more specifics. If the backend cannot do discards, the DRBD peer + * may fall back to blkdev_issue_zeroout(). + */ + u8 discard_enabled; + u8 discard_zeroes_data; + u8 write_same_capable; + u8 _pad; +} __packed; + struct p_sizes { u64 d_size; /* size of disk */ u64 u_size; /* user requested size */ @@ -242,6 +310,9 @@ struct p_sizes { u32 max_bio_size; /* Maximal size of a BIO */ u16 queue_order_type; /* not yet implemented in DRBD*/ u16 dds_flags; /* use enum dds_flags here. */ + + /* optional queue_limits if (agreed_features & DRBD_FF_WSAME) */ + struct o_qlim qlim[0]; } __packed; struct p_state { diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 050aaa1c0350..df45713dfbe8 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -25,7 +25,7 @@ #include -#include +#include #include #include @@ -48,7 +48,7 @@ #include "drbd_req.h" #include "drbd_vli.h" -#define PRO_FEATURES (FF_TRIM) +#define PRO_FEATURES (DRBD_FF_TRIM|DRBD_FF_THIN_RESYNC|DRBD_FF_WSAME) struct packet_info { enum drbd_packet cmd; @@ -361,14 +361,17 @@ You must not have the req_lock: drbd_wait_ee_list_empty() */ +/* normal: payload_size == request size (bi_size) + * w_same: payload_size == logical_block_size + * trim: payload_size == 0 */ struct drbd_peer_request * drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t sector, - unsigned int data_size, bool has_payload, gfp_t gfp_mask) __must_hold(local) + unsigned int request_size, unsigned int payload_size, gfp_t gfp_mask) __must_hold(local) { struct drbd_device *device = peer_device->device; struct drbd_peer_request *peer_req; struct page *page = NULL; - unsigned nr_pages = (data_size + PAGE_SIZE -1) >> PAGE_SHIFT; + unsigned nr_pages = (payload_size + PAGE_SIZE -1) >> PAGE_SHIFT; if (drbd_insert_fault(device, DRBD_FAULT_AL_EE)) return NULL; @@ -380,7 +383,7 @@ drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t secto return NULL; } - if (has_payload && data_size) { + if (nr_pages) { page = drbd_alloc_pages(peer_device, nr_pages, gfpflags_allow_blocking(gfp_mask)); if (!page) @@ -390,7 +393,7 @@ drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t secto memset(peer_req, 0, sizeof(*peer_req)); INIT_LIST_HEAD(&peer_req->w.list); drbd_clear_interval(&peer_req->i); - peer_req->i.size = data_size; + peer_req->i.size = request_size; peer_req->i.sector = sector; peer_req->submit_jif = jiffies; peer_req->peer_device = peer_device; @@ -1204,13 +1207,84 @@ static int drbd_recv_header(struct drbd_connection *connection, struct packet_in return err; } -static void drbd_flush(struct drbd_connection *connection) +/* This is blkdev_issue_flush, but asynchronous. + * We want to submit to all component volumes in parallel, + * then wait for all completions. + */ +struct issue_flush_context { + atomic_t pending; + int error; + struct completion done; +}; +struct one_flush_context { + struct drbd_device *device; + struct issue_flush_context *ctx; +}; + +void one_flush_endio(struct bio *bio) { - int rv; - struct drbd_peer_device *peer_device; - int vnr; + struct one_flush_context *octx = bio->bi_private; + struct drbd_device *device = octx->device; + struct issue_flush_context *ctx = octx->ctx; + + if (bio->bi_error) { + ctx->error = bio->bi_error; + drbd_info(device, "local disk FLUSH FAILED with status %d\n", bio->bi_error); + } + kfree(octx); + bio_put(bio); + + clear_bit(FLUSH_PENDING, &device->flags); + put_ldev(device); + kref_put(&device->kref, drbd_destroy_device); + + if (atomic_dec_and_test(&ctx->pending)) + complete(&ctx->done); +} + +static void submit_one_flush(struct drbd_device *device, struct issue_flush_context *ctx) +{ + struct bio *bio = bio_alloc(GFP_NOIO, 0); + struct one_flush_context *octx = kmalloc(sizeof(*octx), GFP_NOIO); + if (!bio || !octx) { + drbd_warn(device, "Could not allocate a bio, CANNOT ISSUE FLUSH\n"); + /* FIXME: what else can I do now? disconnecting or detaching + * really does not help to improve the state of the world, either. + */ + kfree(octx); + if (bio) + bio_put(bio); + + ctx->error = -ENOMEM; + put_ldev(device); + kref_put(&device->kref, drbd_destroy_device); + return; + } + octx->device = device; + octx->ctx = ctx; + bio->bi_bdev = device->ldev->backing_bdev; + bio->bi_private = octx; + bio->bi_end_io = one_flush_endio; + bio_set_op_attrs(bio, REQ_OP_FLUSH, WRITE_FLUSH); + + device->flush_jif = jiffies; + set_bit(FLUSH_PENDING, &device->flags); + atomic_inc(&ctx->pending); + submit_bio(bio); +} + +static void drbd_flush(struct drbd_connection *connection) +{ if (connection->resource->write_ordering >= WO_BDEV_FLUSH) { + struct drbd_peer_device *peer_device; + struct issue_flush_context ctx; + int vnr; + + atomic_set(&ctx.pending, 1); + ctx.error = 0; + init_completion(&ctx.done); + rcu_read_lock(); idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { struct drbd_device *device = peer_device->device; @@ -1220,31 +1294,24 @@ static void drbd_flush(struct drbd_connection *connection) kref_get(&device->kref); rcu_read_unlock(); - /* Right now, we have only this one synchronous code path - * for flushes between request epochs. - * We may want to make those asynchronous, - * or at least parallelize the flushes to the volume devices. - */ - device->flush_jif = jiffies; - set_bit(FLUSH_PENDING, &device->flags); - rv = blkdev_issue_flush(device->ldev->backing_bdev, - GFP_NOIO, NULL); - clear_bit(FLUSH_PENDING, &device->flags); - if (rv) { - drbd_info(device, "local disk flush failed with status %d\n", rv); - /* would rather check on EOPNOTSUPP, but that is not reliable. - * don't try again for ANY return value != 0 - * if (rv == -EOPNOTSUPP) */ - drbd_bump_write_ordering(connection->resource, NULL, WO_DRAIN_IO); - } - put_ldev(device); - kref_put(&device->kref, drbd_destroy_device); + submit_one_flush(device, &ctx); rcu_read_lock(); - if (rv) - break; } rcu_read_unlock(); + + /* Do we want to add a timeout, + * if disk-timeout is set? */ + if (!atomic_dec_and_test(&ctx.pending)) + wait_for_completion(&ctx.done); + + if (ctx.error) { + /* would rather check on EOPNOTSUPP, but that is not reliable. + * don't try again for ANY return value != 0 + * if (rv == -EOPNOTSUPP) */ + /* Any error is already reported by bio_endio callback. */ + drbd_bump_write_ordering(connection->resource, NULL, WO_DRAIN_IO); + } } } @@ -1379,6 +1446,120 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin drbd_info(resource, "Method to ensure write ordering: %s\n", write_ordering_str[resource->write_ordering]); } +/* + * We *may* ignore the discard-zeroes-data setting, if so configured. + * + * Assumption is that it "discard_zeroes_data=0" is only because the backend + * may ignore partial unaligned discards. + * + * LVM/DM thin as of at least + * LVM version: 2.02.115(2)-RHEL7 (2015-01-28) + * Library version: 1.02.93-RHEL7 (2015-01-28) + * Driver version: 4.29.0 + * still behaves this way. + * + * For unaligned (wrt. alignment and granularity) or too small discards, + * we zero-out the initial (and/or) trailing unaligned partial chunks, + * but discard all the aligned full chunks. + * + * At least for LVM/DM thin, the result is effectively "discard_zeroes_data=1". + */ +int drbd_issue_discard_or_zero_out(struct drbd_device *device, sector_t start, unsigned int nr_sectors, bool discard) +{ + struct block_device *bdev = device->ldev->backing_bdev; + struct request_queue *q = bdev_get_queue(bdev); + sector_t tmp, nr; + unsigned int max_discard_sectors, granularity; + int alignment; + int err = 0; + + if (!discard) + goto zero_out; + + /* Zero-sector (unknown) and one-sector granularities are the same. */ + granularity = max(q->limits.discard_granularity >> 9, 1U); + alignment = (bdev_discard_alignment(bdev) >> 9) % granularity; + + max_discard_sectors = min(q->limits.max_discard_sectors, (1U << 22)); + max_discard_sectors -= max_discard_sectors % granularity; + if (unlikely(!max_discard_sectors)) + goto zero_out; + + if (nr_sectors < granularity) + goto zero_out; + + tmp = start; + if (sector_div(tmp, granularity) != alignment) { + if (nr_sectors < 2*granularity) + goto zero_out; + /* start + gran - (start + gran - align) % gran */ + tmp = start + granularity - alignment; + tmp = start + granularity - sector_div(tmp, granularity); + + nr = tmp - start; + err |= blkdev_issue_zeroout(bdev, start, nr, GFP_NOIO, 0); + nr_sectors -= nr; + start = tmp; + } + while (nr_sectors >= granularity) { + nr = min_t(sector_t, nr_sectors, max_discard_sectors); + err |= blkdev_issue_discard(bdev, start, nr, GFP_NOIO, 0); + nr_sectors -= nr; + start += nr; + } + zero_out: + if (nr_sectors) { + err |= blkdev_issue_zeroout(bdev, start, nr_sectors, GFP_NOIO, 0); + } + return err != 0; +} + +static bool can_do_reliable_discards(struct drbd_device *device) +{ + struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev); + struct disk_conf *dc; + bool can_do; + + if (!blk_queue_discard(q)) + return false; + + if (q->limits.discard_zeroes_data) + return true; + + rcu_read_lock(); + dc = rcu_dereference(device->ldev->disk_conf); + can_do = dc->discard_zeroes_if_aligned; + rcu_read_unlock(); + return can_do; +} + +static void drbd_issue_peer_discard(struct drbd_device *device, struct drbd_peer_request *peer_req) +{ + /* If the backend cannot discard, or does not guarantee + * read-back zeroes in discarded ranges, we fall back to + * zero-out. Unless configuration specifically requested + * otherwise. */ + if (!can_do_reliable_discards(device)) + peer_req->flags |= EE_IS_TRIM_USE_ZEROOUT; + + if (drbd_issue_discard_or_zero_out(device, peer_req->i.sector, + peer_req->i.size >> 9, !(peer_req->flags & EE_IS_TRIM_USE_ZEROOUT))) + peer_req->flags |= EE_WAS_ERROR; + drbd_endio_write_sec_final(peer_req); +} + +static void drbd_issue_peer_wsame(struct drbd_device *device, + struct drbd_peer_request *peer_req) +{ + struct block_device *bdev = device->ldev->backing_bdev; + sector_t s = peer_req->i.sector; + sector_t nr = peer_req->i.size >> 9; + if (blkdev_issue_write_same(bdev, s, nr, GFP_NOIO, peer_req->pages)) + peer_req->flags |= EE_WAS_ERROR; + drbd_endio_write_sec_final(peer_req); +} + + /** * drbd_submit_peer_request() * @device: DRBD device. @@ -1398,7 +1579,8 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin /* TODO allocate from our own bio_set. */ int drbd_submit_peer_request(struct drbd_device *device, struct drbd_peer_request *peer_req, - const unsigned rw, const int fault_type) + const unsigned op, const unsigned op_flags, + const int fault_type) { struct bio *bios = NULL; struct bio *bio; @@ -1409,7 +1591,13 @@ int drbd_submit_peer_request(struct drbd_device *device, unsigned nr_pages = (data_size + PAGE_SIZE -1) >> PAGE_SHIFT; int err = -ENOMEM; - if (peer_req->flags & EE_IS_TRIM_USE_ZEROOUT) { + /* TRIM/DISCARD: for now, always use the helper function + * blkdev_issue_zeroout(..., discard=true). + * It's synchronous, but it does the right thing wrt. bio splitting. + * Correctness first, performance later. Next step is to code an + * asynchronous variant of the same. + */ + if (peer_req->flags & (EE_IS_TRIM|EE_WRITE_SAME)) { /* wait for all pending IO completions, before we start * zeroing things out. */ conn_wait_active_ee_empty(peer_req->peer_device->connection); @@ -1417,22 +1605,22 @@ int drbd_submit_peer_request(struct drbd_device *device, * so we can find it to present it in debugfs */ peer_req->submit_jif = jiffies; peer_req->flags |= EE_SUBMITTED; - spin_lock_irq(&device->resource->req_lock); - list_add_tail(&peer_req->w.list, &device->active_ee); - spin_unlock_irq(&device->resource->req_lock); - if (blkdev_issue_zeroout(device->ldev->backing_bdev, - sector, data_size >> 9, GFP_NOIO, false)) - peer_req->flags |= EE_WAS_ERROR; - drbd_endio_write_sec_final(peer_req); + + /* If this was a resync request from receive_rs_deallocated(), + * it is already on the sync_ee list */ + if (list_empty(&peer_req->w.list)) { + spin_lock_irq(&device->resource->req_lock); + list_add_tail(&peer_req->w.list, &device->active_ee); + spin_unlock_irq(&device->resource->req_lock); + } + + if (peer_req->flags & EE_IS_TRIM) + drbd_issue_peer_discard(device, peer_req); + else /* EE_WRITE_SAME */ + drbd_issue_peer_wsame(device, peer_req); return 0; } - /* Discards don't have any payload. - * But the scsi layer still expects a bio_vec it can use internally, - * see sd_setup_discard_cmnd() and blk_add_request_payload(). */ - if (peer_req->flags & EE_IS_TRIM) - nr_pages = 1; - /* In most cases, we will only need one bio. But in case the lower * level restrictions happen to be different at this offset on this * side than those of the sending peer, we may need to submit the @@ -1450,7 +1638,7 @@ next_bio: /* > peer_req->i.sector, unless this is the first bio */ bio->bi_iter.bi_sector = sector; bio->bi_bdev = device->ldev->backing_bdev; - bio->bi_rw = rw; + bio_set_op_attrs(bio, op, op_flags); bio->bi_private = peer_req; bio->bi_end_io = drbd_peer_request_endio; @@ -1458,11 +1646,6 @@ next_bio: bios = bio; ++n_bios; - if (rw & REQ_DISCARD) { - bio->bi_iter.bi_size = data_size; - goto submit; - } - page_chain_for_each(page) { unsigned len = min_t(unsigned, data_size, PAGE_SIZE); if (!bio_add_page(bio, page, len, 0)) { @@ -1484,7 +1667,6 @@ next_bio: --nr_pages; } D_ASSERT(device, data_size == 0); -submit: D_ASSERT(device, page == NULL); atomic_set(&peer_req->pending_bios, n_bios); @@ -1608,8 +1790,26 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf return 0; } +/* quick wrapper in case payload size != request_size (write same) */ +static void drbd_csum_ee_size(struct crypto_ahash *h, + struct drbd_peer_request *r, void *d, + unsigned int payload_size) +{ + unsigned int tmp = r->i.size; + r->i.size = payload_size; + drbd_csum_ee(h, r, d); + r->i.size = tmp; +} + /* used from receive_RSDataReply (recv_resync_read) - * and from receive_Data */ + * and from receive_Data. + * data_size: actual payload ("data in") + * for normal writes that is bi_size. + * for discards, that is zero. + * for write same, it is logical_block_size. + * both trim and write same have the bi_size ("data len to be affected") + * as extra argument in the packet header. + */ static struct drbd_peer_request * read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, struct packet_info *pi) __must_hold(local) @@ -1624,6 +1824,7 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, void *dig_vv = peer_device->connection->int_dig_vv; unsigned long *data; struct p_trim *trim = (pi->cmd == P_TRIM) ? pi->data : NULL; + struct p_trim *wsame = (pi->cmd == P_WSAME) ? pi->data : NULL; digest_size = 0; if (!trim && peer_device->connection->peer_integrity_tfm) { @@ -1638,38 +1839,60 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, data_size -= digest_size; } + /* assume request_size == data_size, but special case trim and wsame. */ + ds = data_size; if (trim) { - D_ASSERT(peer_device, data_size == 0); - data_size = be32_to_cpu(trim->size); + if (!expect(data_size == 0)) + return NULL; + ds = be32_to_cpu(trim->size); + } else if (wsame) { + if (data_size != queue_logical_block_size(device->rq_queue)) { + drbd_err(peer_device, "data size (%u) != drbd logical block size (%u)\n", + data_size, queue_logical_block_size(device->rq_queue)); + return NULL; + } + if (data_size != bdev_logical_block_size(device->ldev->backing_bdev)) { + drbd_err(peer_device, "data size (%u) != backend logical block size (%u)\n", + data_size, bdev_logical_block_size(device->ldev->backing_bdev)); + return NULL; + } + ds = be32_to_cpu(wsame->size); } - if (!expect(IS_ALIGNED(data_size, 512))) + if (!expect(IS_ALIGNED(ds, 512))) return NULL; - /* prepare for larger trim requests. */ - if (!trim && !expect(data_size <= DRBD_MAX_BIO_SIZE)) + if (trim || wsame) { + if (!expect(ds <= (DRBD_MAX_BBIO_SECTORS << 9))) + return NULL; + } else if (!expect(ds <= DRBD_MAX_BIO_SIZE)) return NULL; /* even though we trust out peer, * we sometimes have to double check. */ - if (sector + (data_size>>9) > capacity) { + if (sector + (ds>>9) > capacity) { drbd_err(device, "request from peer beyond end of local disk: " "capacity: %llus < sector: %llus + size: %u\n", (unsigned long long)capacity, - (unsigned long long)sector, data_size); + (unsigned long long)sector, ds); return NULL; } /* GFP_NOIO, because we must not cause arbitrary write-out: in a DRBD * "criss-cross" setup, that might cause write-out on some other DRBD, * which in turn might block on the other node at this very place. */ - peer_req = drbd_alloc_peer_req(peer_device, id, sector, data_size, trim == NULL, GFP_NOIO); + peer_req = drbd_alloc_peer_req(peer_device, id, sector, ds, data_size, GFP_NOIO); if (!peer_req) return NULL; peer_req->flags |= EE_WRITE; - if (trim) + if (trim) { + peer_req->flags |= EE_IS_TRIM; return peer_req; + } + if (wsame) + peer_req->flags |= EE_WRITE_SAME; + /* receive payload size bytes into page chain */ ds = data_size; page = peer_req->pages; page_chain_for_each(page) { @@ -1689,7 +1912,7 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, } if (digest_size) { - drbd_csum_ee(peer_device->connection->peer_integrity_tfm, peer_req, dig_vv); + drbd_csum_ee_size(peer_device->connection->peer_integrity_tfm, peer_req, dig_vv, data_size); if (memcmp(dig_in, dig_vv, digest_size)) { drbd_err(device, "Digest integrity check FAILED: %llus +%u\n", (unsigned long long)sector, data_size); @@ -1830,7 +2053,8 @@ static int recv_resync_read(struct drbd_peer_device *peer_device, sector_t secto spin_unlock_irq(&device->resource->req_lock); atomic_add(pi->size >> 9, &device->rs_sect_ev); - if (drbd_submit_peer_request(device, peer_req, WRITE, DRBD_FAULT_RS_WR) == 0) + if (drbd_submit_peer_request(device, peer_req, REQ_OP_WRITE, 0, + DRBD_FAULT_RS_WR) == 0) return 0; /* don't care for the reason here */ @@ -2065,13 +2289,13 @@ static inline int overlaps(sector_t s1, int l1, sector_t s2, int l2) static bool overlapping_resync_write(struct drbd_device *device, struct drbd_peer_request *peer_req) { struct drbd_peer_request *rs_req; - bool rv = 0; + bool rv = false; spin_lock_irq(&device->resource->req_lock); list_for_each_entry(rs_req, &device->sync_ee, w.list) { if (overlaps(peer_req->i.sector, peer_req->i.size, rs_req->i.sector, rs_req->i.size)) { - rv = 1; + rv = true; break; } } @@ -2152,12 +2376,19 @@ static int wait_for_and_update_peer_seq(struct drbd_peer_device *peer_device, co /* see also bio_flags_to_wire() * DRBD_REQ_*, because we need to semantically map the flags to data packet * flags and back. We may replicate to other kernel versions. */ -static unsigned long wire_flags_to_bio(u32 dpf) +static unsigned long wire_flags_to_bio_flags(u32 dpf) { return (dpf & DP_RW_SYNC ? REQ_SYNC : 0) | (dpf & DP_FUA ? REQ_FUA : 0) | - (dpf & DP_FLUSH ? REQ_FLUSH : 0) | - (dpf & DP_DISCARD ? REQ_DISCARD : 0); + (dpf & DP_FLUSH ? REQ_PREFLUSH : 0); +} + +static unsigned long wire_flags_to_bio_op(u32 dpf) +{ + if (dpf & DP_DISCARD) + return REQ_OP_DISCARD; + else + return REQ_OP_WRITE; } static void fail_postponed_requests(struct drbd_device *device, sector_t sector, @@ -2303,7 +2534,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * struct drbd_peer_request *peer_req; struct p_data *p = pi->data; u32 peer_seq = be32_to_cpu(p->seq_num); - int rw = WRITE; + int op, op_flags; u32 dp_flags; int err, tp; @@ -2342,14 +2573,11 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * peer_req->flags |= EE_APPLICATION; dp_flags = be32_to_cpu(p->dp_flags); - rw |= wire_flags_to_bio(dp_flags); + op = wire_flags_to_bio_op(dp_flags); + op_flags = wire_flags_to_bio_flags(dp_flags); if (pi->cmd == P_TRIM) { - struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev); - peer_req->flags |= EE_IS_TRIM; - if (!blk_queue_discard(q)) - peer_req->flags |= EE_IS_TRIM_USE_ZEROOUT; D_ASSERT(peer_device, peer_req->i.size > 0); - D_ASSERT(peer_device, rw & REQ_DISCARD); + D_ASSERT(peer_device, op == REQ_OP_DISCARD); D_ASSERT(peer_device, peer_req->pages == NULL); } else if (peer_req->pages == NULL) { D_ASSERT(device, peer_req->i.size == 0); @@ -2414,11 +2642,11 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * update_peer_seq(peer_device, peer_seq); spin_lock_irq(&device->resource->req_lock); } - /* if we use the zeroout fallback code, we process synchronously - * and we wait for all pending requests, respectively wait for + /* TRIM and WRITE_SAME are processed synchronously, + * we wait for all pending requests, respectively wait for * active_ee to become empty in drbd_submit_peer_request(); * better not add ourselves here. */ - if ((peer_req->flags & EE_IS_TRIM_USE_ZEROOUT) == 0) + if ((peer_req->flags & (EE_IS_TRIM|EE_WRITE_SAME)) == 0) list_add_tail(&peer_req->w.list, &device->active_ee); spin_unlock_irq(&device->resource->req_lock); @@ -2433,7 +2661,8 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * peer_req->flags |= EE_CALL_AL_COMPLETE_IO; } - err = drbd_submit_peer_request(device, peer_req, rw, DRBD_FAULT_DT_WR); + err = drbd_submit_peer_request(device, peer_req, op, op_flags, + DRBD_FAULT_DT_WR); if (!err) return 0; @@ -2449,7 +2678,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * } out_interrupted: - drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT + EV_CLEANUP); + drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT | EV_CLEANUP); put_ldev(device); drbd_free_peer_req(device, peer_req); return err; @@ -2574,6 +2803,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet case P_DATA_REQUEST: drbd_send_ack_rp(peer_device, P_NEG_DREPLY, p); break; + case P_RS_THIN_REQ: case P_RS_DATA_REQUEST: case P_CSUM_RS_REQUEST: case P_OV_REQUEST: @@ -2599,7 +2829,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet * "criss-cross" setup, that might cause write-out on some other DRBD, * which in turn might block on the other node at this very place. */ peer_req = drbd_alloc_peer_req(peer_device, p->block_id, sector, size, - true /* has real payload */, GFP_NOIO); + size, GFP_NOIO); if (!peer_req) { put_ldev(device); return -ENOMEM; @@ -2613,6 +2843,12 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet peer_req->flags |= EE_APPLICATION; goto submit; + case P_RS_THIN_REQ: + /* If at some point in the future we have a smart way to + find out if this data block is completely deallocated, + then we would do something smarter here than reading + the block... */ + peer_req->flags |= EE_RS_THIN_REQ; case P_RS_DATA_REQUEST: peer_req->w.cb = w_e_end_rsdata_req; fault_type = DRBD_FAULT_RS_RD; @@ -2723,7 +2959,8 @@ submit_for_resync: submit: update_receiver_timing_details(connection, drbd_submit_peer_request); inc_unacked(device); - if (drbd_submit_peer_request(device, peer_req, READ, fault_type) == 0) + if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0, + fault_type) == 0) return 0; /* don't care for the reason here */ @@ -2957,7 +3194,8 @@ static void drbd_uuid_dump(struct drbd_device *device, char *text, u64 *uuid, -1091 requires proto 91 -1096 requires proto 96 */ -static int drbd_uuid_compare(struct drbd_device *const device, int *rule_nr) __must_hold(local) + +static int drbd_uuid_compare(struct drbd_device *const device, enum drbd_role const peer_role, int *rule_nr) __must_hold(local) { struct drbd_peer_device *const peer_device = first_peer_device(device); struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL; @@ -3037,8 +3275,39 @@ static int drbd_uuid_compare(struct drbd_device *const device, int *rule_nr) __m * next bit (weight 2) is set when peer was primary */ *rule_nr = 40; + /* Neither has the "crashed primary" flag set, + * only a replication link hickup. */ + if (rct == 0) + return 0; + + /* Current UUID equal and no bitmap uuid; does not necessarily + * mean this was a "simultaneous hard crash", maybe IO was + * frozen, so no UUID-bump happened. + * This is a protocol change, overload DRBD_FF_WSAME as flag + * for "new-enough" peer DRBD version. */ + if (device->state.role == R_PRIMARY || peer_role == R_PRIMARY) { + *rule_nr = 41; + if (!(connection->agreed_features & DRBD_FF_WSAME)) { + drbd_warn(peer_device, "Equivalent unrotated UUIDs, but current primary present.\n"); + return -(0x10000 | PRO_VERSION_MAX | (DRBD_FF_WSAME << 8)); + } + if (device->state.role == R_PRIMARY && peer_role == R_PRIMARY) { + /* At least one has the "crashed primary" bit set, + * both are primary now, but neither has rotated its UUIDs? + * "Can not happen." */ + drbd_err(peer_device, "Equivalent unrotated UUIDs, but both are primary. Can not resolve this.\n"); + return -100; + } + if (device->state.role == R_PRIMARY) + return 1; + return -1; + } + + /* Both are secondary. + * Really looks like recovery from simultaneous hard crash. + * Check which had been primary before, and arbitrate. */ switch (rct) { - case 0: /* !self_pri && !peer_pri */ return 0; + case 0: /* !self_pri && !peer_pri */ return 0; /* already handled */ case 1: /* self_pri && !peer_pri */ return 1; case 2: /* !self_pri && peer_pri */ return -1; case 3: /* self_pri && peer_pri */ @@ -3165,7 +3434,7 @@ static enum drbd_conns drbd_sync_handshake(struct drbd_peer_device *peer_device, drbd_uuid_dump(device, "peer", device->p_uuid, device->p_uuid[UI_SIZE], device->p_uuid[UI_FLAGS]); - hg = drbd_uuid_compare(device, &rule_nr); + hg = drbd_uuid_compare(device, peer_role, &rule_nr); spin_unlock_irq(&device->ldev->md.uuid_lock); drbd_info(device, "uuid_compare()=%d by rule %d\n", hg, rule_nr); @@ -3174,6 +3443,15 @@ static enum drbd_conns drbd_sync_handshake(struct drbd_peer_device *peer_device, drbd_alert(device, "Unrelated data, aborting!\n"); return C_MASK; } + if (hg < -0x10000) { + int proto, fflags; + hg = -hg; + proto = hg & 0xff; + fflags = (hg >> 8) & 0xff; + drbd_alert(device, "To resolve this both sides have to support at least protocol %d and feature flags 0x%x\n", + proto, fflags); + return C_MASK; + } if (hg < -1000) { drbd_alert(device, "To resolve this both sides have to support at least protocol %d\n", -hg - 1000); return C_MASK; @@ -3403,7 +3681,8 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in */ peer_integrity_tfm = crypto_alloc_ahash(integrity_alg, 0, CRYPTO_ALG_ASYNC); - if (!peer_integrity_tfm) { + if (IS_ERR(peer_integrity_tfm)) { + peer_integrity_tfm = NULL; drbd_err(connection, "peer data-integrity-alg %s not supported\n", integrity_alg); goto disconnect; @@ -3754,6 +4033,7 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info struct drbd_peer_device *peer_device; struct drbd_device *device; struct p_sizes *p = pi->data; + struct o_qlim *o = (connection->agreed_features & DRBD_FF_WSAME) ? p->qlim : NULL; enum determine_dev_size dd = DS_UNCHANGED; sector_t p_size, p_usize, p_csize, my_usize; int ldsc = 0; /* local disk size changed */ @@ -3773,6 +4053,7 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info device->p_size = p_size; if (get_ldev(device)) { + sector_t new_size, cur_size; rcu_read_lock(); my_usize = rcu_dereference(device->ldev->disk_conf)->disk_size; rcu_read_unlock(); @@ -3789,11 +4070,13 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info /* Never shrink a device with usable data during connect. But allow online shrinking if we are connected. */ - if (drbd_new_dev_size(device, device->ldev, p_usize, 0) < - drbd_get_capacity(device->this_bdev) && + new_size = drbd_new_dev_size(device, device->ldev, p_usize, 0); + cur_size = drbd_get_capacity(device->this_bdev); + if (new_size < cur_size && device->state.disk >= D_OUTDATED && device->state.conn < C_CONNECTED) { - drbd_err(device, "The peer's disk size is too small!\n"); + drbd_err(device, "The peer's disk size is too small! (%llu < %llu sectors)\n", + (unsigned long long)new_size, (unsigned long long)cur_size); conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD); put_ldev(device); return -EIO; @@ -3827,14 +4110,14 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info } device->peer_max_bio_size = be32_to_cpu(p->max_bio_size); - /* Leave drbd_reconsider_max_bio_size() before drbd_determine_dev_size(). + /* Leave drbd_reconsider_queue_parameters() before drbd_determine_dev_size(). In case we cleared the QUEUE_FLAG_DISCARD from our queue in - drbd_reconsider_max_bio_size(), we can be sure that after + drbd_reconsider_queue_parameters(), we can be sure that after drbd_determine_dev_size() no REQ_DISCARDs are in the queue. */ ddsf = be16_to_cpu(p->dds_flags); if (get_ldev(device)) { - drbd_reconsider_max_bio_size(device, device->ldev); + drbd_reconsider_queue_parameters(device, device->ldev, o); dd = drbd_determine_dev_size(device, ddsf, NULL); put_ldev(device); if (dd == DS_ERROR) @@ -3854,7 +4137,7 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info * However, if he sends a zero current size, * take his (user-capped or) backing disk size anyways. */ - drbd_reconsider_max_bio_size(device, NULL); + drbd_reconsider_queue_parameters(device, NULL, o); drbd_set_my_capacity(device, p_csize ?: p_usize ?: p_size); } @@ -4587,9 +4870,75 @@ static int receive_out_of_sync(struct drbd_connection *connection, struct packet return 0; } +static int receive_rs_deallocated(struct drbd_connection *connection, struct packet_info *pi) +{ + struct drbd_peer_device *peer_device; + struct p_block_desc *p = pi->data; + struct drbd_device *device; + sector_t sector; + int size, err = 0; + + peer_device = conn_peer_device(connection, pi->vnr); + if (!peer_device) + return -EIO; + device = peer_device->device; + + sector = be64_to_cpu(p->sector); + size = be32_to_cpu(p->blksize); + + dec_rs_pending(device); + + if (get_ldev(device)) { + struct drbd_peer_request *peer_req; + const int op = REQ_OP_DISCARD; + + peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER, sector, + size, 0, GFP_NOIO); + if (!peer_req) { + put_ldev(device); + return -ENOMEM; + } + + peer_req->w.cb = e_end_resync_block; + peer_req->submit_jif = jiffies; + peer_req->flags |= EE_IS_TRIM; + + spin_lock_irq(&device->resource->req_lock); + list_add_tail(&peer_req->w.list, &device->sync_ee); + spin_unlock_irq(&device->resource->req_lock); + + atomic_add(pi->size >> 9, &device->rs_sect_ev); + err = drbd_submit_peer_request(device, peer_req, op, 0, DRBD_FAULT_RS_WR); + + if (err) { + spin_lock_irq(&device->resource->req_lock); + list_del(&peer_req->w.list); + spin_unlock_irq(&device->resource->req_lock); + + drbd_free_peer_req(device, peer_req); + put_ldev(device); + err = 0; + goto fail; + } + + inc_unacked(device); + + /* No put_ldev() here. Gets called in drbd_endio_write_sec_final(), + as well as drbd_rs_complete_io() */ + } else { + fail: + drbd_rs_complete_io(device, sector); + drbd_send_ack_ex(peer_device, P_NEG_ACK, sector, size, ID_SYNCER); + } + + atomic_add(size >> 9, &device->rs_sect_in); + + return err; +} + struct data_cmd { int expect_payload; - size_t pkt_size; + unsigned int pkt_size; int (*fn)(struct drbd_connection *, struct packet_info *); }; @@ -4614,11 +4963,14 @@ static struct data_cmd drbd_cmd_handler[] = { [P_OV_REQUEST] = { 0, sizeof(struct p_block_req), receive_DataRequest }, [P_OV_REPLY] = { 1, sizeof(struct p_block_req), receive_DataRequest }, [P_CSUM_RS_REQUEST] = { 1, sizeof(struct p_block_req), receive_DataRequest }, + [P_RS_THIN_REQ] = { 0, sizeof(struct p_block_req), receive_DataRequest }, [P_DELAY_PROBE] = { 0, sizeof(struct p_delay_probe93), receive_skip }, [P_OUT_OF_SYNC] = { 0, sizeof(struct p_block_desc), receive_out_of_sync }, [P_CONN_ST_CHG_REQ] = { 0, sizeof(struct p_req_state), receive_req_conn_state }, [P_PROTOCOL_UPDATE] = { 1, sizeof(struct p_protocol), receive_protocol }, [P_TRIM] = { 0, sizeof(struct p_trim), receive_Data }, + [P_RS_DEALLOCATED] = { 0, sizeof(struct p_block_desc), receive_rs_deallocated }, + [P_WSAME] = { 1, sizeof(struct p_wsame), receive_Data }, }; static void drbdd(struct drbd_connection *connection) @@ -4628,7 +4980,7 @@ static void drbdd(struct drbd_connection *connection) int err; while (get_t_state(&connection->receiver) == RUNNING) { - struct data_cmd *cmd; + struct data_cmd const *cmd; drbd_thread_current_set_cpu(&connection->receiver); update_receiver_timing_details(connection, drbd_recv_header); @@ -4643,11 +4995,18 @@ static void drbdd(struct drbd_connection *connection) } shs = cmd->pkt_size; + if (pi.cmd == P_SIZES && connection->agreed_features & DRBD_FF_WSAME) + shs += sizeof(struct o_qlim); if (pi.size > shs && !cmd->expect_payload) { drbd_err(connection, "No payload expected %s l:%d\n", cmdname(pi.cmd), pi.size); goto err_out; } + if (pi.size < shs) { + drbd_err(connection, "%s: unexpected packet size, expected:%d received:%d\n", + cmdname(pi.cmd), (int)shs, pi.size); + goto err_out; + } if (shs) { update_receiver_timing_details(connection, drbd_recv_all_warn); @@ -4783,9 +5142,11 @@ static int drbd_disconnected(struct drbd_peer_device *peer_device) drbd_md_sync(device); - /* serialize with bitmap writeout triggered by the state change, - * if any. */ - wait_event(device->misc_wait, !test_bit(BITMAP_IO, &device->flags)); + if (get_ldev(device)) { + drbd_bitmap_io(device, &drbd_bm_write_copy_pages, + "write from disconnected", BM_LOCKED_CHANGE_ALLOWED); + put_ldev(device); + } /* tcp_close and release of sendpage pages can be deferred. I don't * want to use SO_LINGER, because apparently it can be deferred for @@ -4892,8 +5253,12 @@ static int drbd_do_features(struct drbd_connection *connection) drbd_info(connection, "Handshake successful: " "Agreed network protocol version %d\n", connection->agreed_pro_version); - drbd_info(connection, "Agreed to%ssupport TRIM on protocol level\n", - connection->agreed_features & FF_TRIM ? " " : " not "); + drbd_info(connection, "Feature flags enabled on protocol level: 0x%x%s%s%s.\n", + connection->agreed_features, + connection->agreed_features & DRBD_FF_TRIM ? " TRIM" : "", + connection->agreed_features & DRBD_FF_THIN_RESYNC ? " THIN_RESYNC" : "", + connection->agreed_features & DRBD_FF_WSAME ? " WRITE_SAME" : + connection->agreed_features ? "" : " none"); return 1; diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 2255dcfebd2b..66b8e4bb74d8 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -47,8 +47,7 @@ static void _drbd_end_io_acct(struct drbd_device *device, struct drbd_request *r &device->vdisk->part0, req->start_jif); } -static struct drbd_request *drbd_req_new(struct drbd_device *device, - struct bio *bio_src) +static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio *bio_src) { struct drbd_request *req; @@ -58,10 +57,12 @@ static struct drbd_request *drbd_req_new(struct drbd_device *device, memset(req, 0, sizeof(*req)); drbd_req_make_private_bio(req, bio_src); - req->rq_state = bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0; - req->device = device; - req->master_bio = bio_src; - req->epoch = 0; + req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0) + | (bio_op(bio_src) == REQ_OP_WRITE_SAME ? RQ_WSAME : 0) + | (bio_op(bio_src) == REQ_OP_DISCARD ? RQ_UNMAP : 0); + req->device = device; + req->master_bio = bio_src; + req->epoch = 0; drbd_clear_interval(&req->i); req->i.sector = bio_src->bi_iter.bi_sector; @@ -218,7 +219,6 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m) { const unsigned s = req->rq_state; struct drbd_device *device = req->device; - int rw; int error, ok; /* we must not complete the master bio, while it is @@ -242,8 +242,6 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m) return; } - rw = bio_rw(req->master_bio); - /* * figure out whether to report success or failure. * @@ -267,7 +265,7 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m) * epoch number. If they match, increase the current_tle_nr, * and reset the transfer log epoch write_cnt. */ - if (rw == WRITE && + if (op_is_write(bio_op(req->master_bio)) && req->epoch == atomic_read(&first_peer_device(device)->connection->current_tle_nr)) start_new_tl_epoch(first_peer_device(device)->connection); @@ -284,11 +282,14 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m) * because no path was available, in which case * it was not even added to the transfer_log. * - * READA may fail, and will not be retried. + * read-ahead may fail, and will not be retried. * * WRITE should have used all available paths already. */ - if (!ok && rw == READ && !list_empty(&req->tl_requests)) + if (!ok && + bio_op(req->master_bio) == REQ_OP_READ && + !(req->master_bio->bi_rw & REQ_RAHEAD) && + !list_empty(&req->tl_requests)) req->rq_state |= RQ_POSTPONED; if (!(req->rq_state & RQ_POSTPONED)) { @@ -644,7 +645,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, __drbd_chk_io_error(device, DRBD_READ_ERROR); /* fall through. */ case READ_AHEAD_COMPLETED_WITH_ERROR: - /* it is legal to fail READA, no __drbd_chk_io_error in that case. */ + /* it is legal to fail read-ahead, no __drbd_chk_io_error in that case. */ mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED); break; @@ -656,7 +657,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, break; case QUEUE_FOR_NET_READ: - /* READ or READA, and + /* READ, and * no local disk, * or target area marked as invalid, * or just got an io-error. */ @@ -977,16 +978,20 @@ static void complete_conflicting_writes(struct drbd_request *req) sector_t sector = req->i.sector; int size = req->i.size; - i = drbd_find_overlap(&device->write_requests, sector, size); - if (!i) - return; - for (;;) { - prepare_to_wait(&device->misc_wait, &wait, TASK_UNINTERRUPTIBLE); - i = drbd_find_overlap(&device->write_requests, sector, size); - if (!i) + drbd_for_each_overlap(i, &device->write_requests, sector, size) { + /* Ignore, if already completed to upper layers. */ + if (i->completed) + continue; + /* Handle the first found overlap. After the schedule + * we have to restart the tree walk. */ + break; + } + if (!i) /* if any */ break; + /* Indicate to wake up device->misc_wait on progress. */ + prepare_to_wait(&device->misc_wait, &wait, TASK_UNINTERRUPTIBLE); i->waiting = true; spin_unlock_irq(&device->resource->req_lock); schedule(); @@ -995,7 +1000,7 @@ static void complete_conflicting_writes(struct drbd_request *req) finish_wait(&device->misc_wait, &wait); } -/* called within req_lock and rcu_read_lock() */ +/* called within req_lock */ static void maybe_pull_ahead(struct drbd_device *device) { struct drbd_connection *connection = first_peer_device(device)->connection; @@ -1132,7 +1137,7 @@ static int drbd_process_write_request(struct drbd_request *req) * replicating, in which case there is no point. */ if (unlikely(req->i.size == 0)) { /* The only size==0 bios we expect are empty flushes. */ - D_ASSERT(device, req->master_bio->bi_rw & REQ_FLUSH); + D_ASSERT(device, req->master_bio->bi_rw & REQ_PREFLUSH); if (remote) _req_mod(req, QUEUE_AS_DRBD_BARRIER); return remote; @@ -1152,12 +1157,29 @@ static int drbd_process_write_request(struct drbd_request *req) return remote; } +static void drbd_process_discard_req(struct drbd_request *req) +{ + int err = drbd_issue_discard_or_zero_out(req->device, + req->i.sector, req->i.size >> 9, true); + + if (err) + req->private_bio->bi_error = -EIO; + bio_endio(req->private_bio); +} + static void drbd_submit_req_private_bio(struct drbd_request *req) { struct drbd_device *device = req->device; struct bio *bio = req->private_bio; - const int rw = bio_rw(bio); + unsigned int type; + + if (bio_op(bio) != REQ_OP_READ) + type = DRBD_FAULT_DT_WR; + else if (bio->bi_rw & REQ_RAHEAD) + type = DRBD_FAULT_DT_RA; + else + type = DRBD_FAULT_DT_RD; bio->bi_bdev = device->ldev->backing_bdev; @@ -1167,11 +1189,10 @@ drbd_submit_req_private_bio(struct drbd_request *req) * stable storage, and this is a WRITE, we may not even submit * this bio. */ if (get_ldev(device)) { - if (drbd_insert_fault(device, - rw == WRITE ? DRBD_FAULT_DT_WR - : rw == READ ? DRBD_FAULT_DT_RD - : DRBD_FAULT_DT_RA)) + if (drbd_insert_fault(device, type)) bio_io_error(bio); + else if (bio_op(bio) == REQ_OP_DISCARD) + drbd_process_discard_req(req); else generic_make_request(bio); put_ldev(device); @@ -1223,24 +1244,45 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio, unsigned long /* Update disk stats */ _drbd_start_io_acct(device, req); + /* process discards always from our submitter thread */ + if (bio_op(bio) & REQ_OP_DISCARD) + goto queue_for_submitter_thread; + if (rw == WRITE && req->private_bio && req->i.size && !test_bit(AL_SUSPENDED, &device->flags)) { - if (!drbd_al_begin_io_fastpath(device, &req->i)) { - atomic_inc(&device->ap_actlog_cnt); - drbd_queue_write(device, req); - return NULL; - } + if (!drbd_al_begin_io_fastpath(device, &req->i)) + goto queue_for_submitter_thread; req->rq_state |= RQ_IN_ACT_LOG; req->in_actlog_jif = jiffies; } - return req; + + queue_for_submitter_thread: + atomic_inc(&device->ap_actlog_cnt); + drbd_queue_write(device, req); + return NULL; +} + +/* Require at least one path to current data. + * We don't want to allow writes on C_STANDALONE D_INCONSISTENT: + * We would not allow to read what was written, + * we would not have bumped the data generation uuids, + * we would cause data divergence for all the wrong reasons. + * + * If we don't see at least one D_UP_TO_DATE, we will fail this request, + * which either returns EIO, or, if OND_SUSPEND_IO is set, suspends IO, + * and queues for retry later. + */ +static bool may_do_writes(struct drbd_device *device) +{ + const union drbd_dev_state s = device->state; + return s.disk == D_UP_TO_DATE || s.pdsk == D_UP_TO_DATE; } static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request *req) { struct drbd_resource *resource = device->resource; - const int rw = bio_rw(req->master_bio); + const int rw = bio_data_dir(req->master_bio); struct bio_and_error m = { NULL, }; bool no_remote = false; bool submit_private_bio = false; @@ -1270,7 +1312,7 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request goto out; } - /* We fail READ/READA early, if we can not serve it. + /* We fail READ early, if we can not serve it. * We must do this before req is registered on any lists. * Otherwise, drbd_req_complete() will queue failed READ for retry. */ if (rw != WRITE) { @@ -1291,6 +1333,12 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request } if (rw == WRITE) { + if (req->private_bio && !may_do_writes(device)) { + bio_put(req->private_bio); + req->private_bio = NULL; + put_ldev(device); + goto nodata; + } if (!drbd_process_write_request(req)) no_remote = true; } else { diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h index bb2ef78165e5..eb49e7f2da91 100644 --- a/drivers/block/drbd/drbd_req.h +++ b/drivers/block/drbd/drbd_req.h @@ -206,6 +206,8 @@ enum drbd_req_state_bits { /* Set when this is a write, clear for a read */ __RQ_WRITE, + __RQ_WSAME, + __RQ_UNMAP, /* Should call drbd_al_complete_io() for this request... */ __RQ_IN_ACT_LOG, @@ -241,10 +243,11 @@ enum drbd_req_state_bits { #define RQ_NET_OK (1UL << __RQ_NET_OK) #define RQ_NET_SIS (1UL << __RQ_NET_SIS) -/* 0x1f8 */ #define RQ_NET_MASK (((1UL << __RQ_NET_MAX)-1) & ~RQ_LOCAL_MASK) #define RQ_WRITE (1UL << __RQ_WRITE) +#define RQ_WSAME (1UL << __RQ_WSAME) +#define RQ_UNMAP (1UL << __RQ_UNMAP) #define RQ_IN_ACT_LOG (1UL << __RQ_IN_ACT_LOG) #define RQ_POSTPONED (1UL << __RQ_POSTPONED) #define RQ_COMPLETION_SUSP (1UL << __RQ_COMPLETION_SUSP) diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c index 5a7ef7873b67..eea0c4aec978 100644 --- a/drivers/block/drbd/drbd_state.c +++ b/drivers/block/drbd/drbd_state.c @@ -814,7 +814,7 @@ is_valid_state(struct drbd_device *device, union drbd_state ns) } if (rv <= 0) - /* already found a reason to abort */; + goto out; /* already found a reason to abort */ else if (ns.role == R_SECONDARY && device->open_cnt) rv = SS_DEVICE_IN_USE; @@ -862,6 +862,7 @@ is_valid_state(struct drbd_device *device, union drbd_state ns) else if (ns.conn >= C_CONNECTED && ns.pdsk == D_UNKNOWN) rv = SS_CONNECTED_OUTDATES; +out: rcu_read_unlock(); return rv; @@ -906,6 +907,15 @@ is_valid_soft_transition(union drbd_state os, union drbd_state ns, struct drbd_c (ns.conn >= C_CONNECTED && os.conn == C_WF_REPORT_PARAMS))) rv = SS_IN_TRANSIENT_STATE; + /* Do not promote during resync handshake triggered by "force primary". + * This is a hack. It should really be rejected by the peer during the + * cluster wide state change request. */ + if (os.role != R_PRIMARY && ns.role == R_PRIMARY + && ns.pdsk == D_UP_TO_DATE + && ns.disk != D_UP_TO_DATE && ns.disk != D_DISKLESS + && (ns.conn <= C_WF_SYNC_UUID || ns.conn != os.conn)) + rv = SS_IN_TRANSIENT_STATE; + if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && os.conn < C_CONNECTED) rv = SS_NEED_CONNECTION; @@ -1628,6 +1638,26 @@ static void broadcast_state_change(struct drbd_state_change *state_change) #undef REMEMBER_STATE_CHANGE } +/* takes old and new peer disk state */ +static bool lost_contact_to_peer_data(enum drbd_disk_state os, enum drbd_disk_state ns) +{ + if ((os >= D_INCONSISTENT && os != D_UNKNOWN && os != D_OUTDATED) + && (ns < D_INCONSISTENT || ns == D_UNKNOWN || ns == D_OUTDATED)) + return true; + + /* Scenario, starting with normal operation + * Connected Primary/Secondary UpToDate/UpToDate + * NetworkFailure Primary/Unknown UpToDate/DUnknown (frozen) + * ... + * Connected Primary/Secondary UpToDate/Diskless (resumed; needs to bump uuid!) + */ + if (os == D_UNKNOWN + && (ns == D_DISKLESS || ns == D_FAILED || ns == D_OUTDATED)) + return true; + + return false; +} + /** * after_state_ch() - Perform after state change actions that may sleep * @device: DRBD device. @@ -1675,7 +1705,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os, what = RESEND; if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) && - conn_lowest_disk(connection) > D_NEGOTIATING) + conn_lowest_disk(connection) == D_UP_TO_DATE) what = RESTART_FROZEN_DISK_IO; if (resource->susp_nod && what != NOTHING) { @@ -1699,6 +1729,13 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os, idr_for_each_entry(&connection->peer_devices, peer_device, vnr) clear_bit(NEW_CUR_UUID, &peer_device->device->flags); rcu_read_unlock(); + + /* We should actively create a new uuid, _before_ + * we resume/resent, if the peer is diskless + * (recovery from a multiple error scenario). + * Currently, this happens with a slight delay + * below when checking lost_contact_to_peer_data() ... + */ _tl_restart(connection, RESEND); _conn_request_state(connection, (union drbd_state) { { .susp_fen = 1 } }, @@ -1742,12 +1779,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os, BM_LOCKED_TEST_ALLOWED); /* Lost contact to peer's copy of the data */ - if ((os.pdsk >= D_INCONSISTENT && - os.pdsk != D_UNKNOWN && - os.pdsk != D_OUTDATED) - && (ns.pdsk < D_INCONSISTENT || - ns.pdsk == D_UNKNOWN || - ns.pdsk == D_OUTDATED)) { + if (lost_contact_to_peer_data(os.pdsk, ns.pdsk)) { if (get_ldev(device)) { if ((ns.role == R_PRIMARY || ns.peer == R_PRIMARY) && device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) { @@ -1934,12 +1966,17 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os, /* This triggers bitmap writeout of potentially still unwritten pages * if the resync finished cleanly, or aborted because of peer disk - * failure, or because of connection loss. + * failure, or on transition from resync back to AHEAD/BEHIND. + * + * Connection loss is handled in drbd_disconnected() by the receiver. + * * For resync aborted because of local disk failure, we cannot do * any bitmap writeout anymore. + * * No harm done if some bits change during this phase. */ - if (os.conn > C_CONNECTED && ns.conn <= C_CONNECTED && get_ldev(device)) { + if ((os.conn > C_CONNECTED && os.conn < C_AHEAD) && + (ns.conn == C_CONNECTED || ns.conn >= C_AHEAD) && get_ldev(device)) { drbd_queue_bitmap_io(device, &drbd_bm_write_copy_pages, NULL, "write from resync_finished", BM_LOCKED_CHANGE_ALLOWED); put_ldev(device); @@ -2160,9 +2197,7 @@ conn_set_state(struct drbd_connection *connection, union drbd_state mask, union ns.disk = os.disk; rv = _drbd_set_state(device, ns, flags, NULL); - if (rv < SS_SUCCESS) - BUG(); - + BUG_ON(rv < SS_SUCCESS); ns.i = device->state.i; ns_max.role = max_role(ns.role, ns_max.role); ns_max.peer = max_role(ns.peer, ns_max.peer); diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h index bd989536f888..6c9d5d4a8a75 100644 --- a/drivers/block/drbd/drbd_state.h +++ b/drivers/block/drbd/drbd_state.h @@ -140,7 +140,7 @@ extern void drbd_resume_al(struct drbd_device *device); extern bool conn_all_vols_unconf(struct drbd_connection *connection); /** - * drbd_request_state() - Reqest a state change + * drbd_request_state() - Request a state change * @device: DRBD device. * @mask: mask of state bits to change. * @val: value of new state bits. diff --git a/drivers/block/drbd/drbd_strings.c b/drivers/block/drbd/drbd_strings.c index 80b0f63c7075..0eeab14776e9 100644 --- a/drivers/block/drbd/drbd_strings.c +++ b/drivers/block/drbd/drbd_strings.c @@ -26,7 +26,7 @@ #include #include "drbd_strings.h" -static const char *drbd_conn_s_names[] = { +static const char * const drbd_conn_s_names[] = { [C_STANDALONE] = "StandAlone", [C_DISCONNECTING] = "Disconnecting", [C_UNCONNECTED] = "Unconnected", @@ -53,13 +53,13 @@ static const char *drbd_conn_s_names[] = { [C_BEHIND] = "Behind", }; -static const char *drbd_role_s_names[] = { +static const char * const drbd_role_s_names[] = { [R_PRIMARY] = "Primary", [R_SECONDARY] = "Secondary", [R_UNKNOWN] = "Unknown" }; -static const char *drbd_disk_s_names[] = { +static const char * const drbd_disk_s_names[] = { [D_DISKLESS] = "Diskless", [D_ATTACHING] = "Attaching", [D_FAILED] = "Failed", @@ -71,7 +71,7 @@ static const char *drbd_disk_s_names[] = { [D_UP_TO_DATE] = "UpToDate", }; -static const char *drbd_state_sw_errors[] = { +static const char * const drbd_state_sw_errors[] = { [-SS_TWO_PRIMARIES] = "Multiple primaries not allowed by config", [-SS_NO_UP_TO_DATE_DISK] = "Need access to UpToDate data", [-SS_NO_LOCAL_DISK] = "Can not resync without local disk", diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index 4d87499f0d54..35dbb3dca47e 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c @@ -173,8 +173,8 @@ void drbd_peer_request_endio(struct bio *bio) { struct drbd_peer_request *peer_req = bio->bi_private; struct drbd_device *device = peer_req->peer_device->device; - int is_write = bio_data_dir(bio) == WRITE; - int is_discard = !!(bio->bi_rw & REQ_DISCARD); + bool is_write = bio_data_dir(bio) == WRITE; + bool is_discard = !!(bio_op(bio) == REQ_OP_DISCARD); if (bio->bi_error && __ratelimit(&drbd_ratelimit_state)) drbd_warn(device, "%s: error=%d s=%llus\n", @@ -248,18 +248,26 @@ void drbd_request_endio(struct bio *bio) /* to avoid recursion in __req_mod */ if (unlikely(bio->bi_error)) { - if (bio->bi_rw & REQ_DISCARD) - what = (bio->bi_error == -EOPNOTSUPP) - ? DISCARD_COMPLETED_NOTSUPP - : DISCARD_COMPLETED_WITH_ERROR; - else - what = (bio_data_dir(bio) == WRITE) - ? WRITE_COMPLETED_WITH_ERROR - : (bio_rw(bio) == READ) - ? READ_COMPLETED_WITH_ERROR - : READ_AHEAD_COMPLETED_WITH_ERROR; - } else + switch (bio_op(bio)) { + case REQ_OP_DISCARD: + if (bio->bi_error == -EOPNOTSUPP) + what = DISCARD_COMPLETED_NOTSUPP; + else + what = DISCARD_COMPLETED_WITH_ERROR; + break; + case REQ_OP_READ: + if (bio->bi_rw & REQ_RAHEAD) + what = READ_AHEAD_COMPLETED_WITH_ERROR; + else + what = READ_COMPLETED_WITH_ERROR; + break; + default: + what = WRITE_COMPLETED_WITH_ERROR; + break; + } + } else { what = COMPLETED_OK; + } bio_put(req->private_bio); req->private_bio = ERR_PTR(bio->bi_error); @@ -320,6 +328,10 @@ void drbd_csum_bio(struct crypto_ahash *tfm, struct bio *bio, void *digest) sg_set_page(&sg, bvec.bv_page, bvec.bv_len, bvec.bv_offset); ahash_request_set_crypt(req, &sg, NULL, sg.length); crypto_ahash_update(req); + /* REQ_OP_WRITE_SAME has only one segment, + * checksum the payload only once. */ + if (bio_op(bio) == REQ_OP_WRITE_SAME) + break; } ahash_request_set_crypt(req, NULL, digest, 0); crypto_ahash_final(req); @@ -387,7 +399,7 @@ static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, /* GFP_TRY, because if there is no memory available right now, this may * be rescheduled for later. It is "only" background resync, after all. */ peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER /* unused */, sector, - size, true /* has real payload */, GFP_TRY); + size, size, GFP_TRY); if (!peer_req) goto defer; @@ -397,7 +409,8 @@ static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, spin_unlock_irq(&device->resource->req_lock); atomic_add(size >> 9, &device->rs_sect_ev); - if (drbd_submit_peer_request(device, peer_req, READ, DRBD_FAULT_RS_RD) == 0) + if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0, + DRBD_FAULT_RS_RD) == 0) return 0; /* If it failed because of ENOMEM, retry should help. If it failed @@ -582,6 +595,7 @@ static int make_resync_request(struct drbd_device *const device, int cancel) int number, rollback_i, size; int align, requeue = 0; int i = 0; + int discard_granularity = 0; if (unlikely(cancel)) return 0; @@ -601,6 +615,12 @@ static int make_resync_request(struct drbd_device *const device, int cancel) return 0; } + if (connection->agreed_features & DRBD_FF_THIN_RESYNC) { + rcu_read_lock(); + discard_granularity = rcu_dereference(device->ldev->disk_conf)->rs_discard_granularity; + rcu_read_unlock(); + } + max_bio_size = queue_max_hw_sectors(device->rq_queue) << 9; number = drbd_rs_number_requests(device); if (number <= 0) @@ -665,6 +685,9 @@ next_sector: if (sector & ((1<<(align+3))-1)) break; + if (discard_granularity && size == discard_granularity) + break; + /* do not cross extent boundaries */ if (((bit+1) & BM_BLOCKS_PER_BM_EXT_MASK) == 0) break; @@ -711,7 +734,8 @@ next_sector: int err; inc_rs_pending(device); - err = drbd_send_drequest(peer_device, P_RS_DATA_REQUEST, + err = drbd_send_drequest(peer_device, + size == discard_granularity ? P_RS_THIN_REQ : P_RS_DATA_REQUEST, sector, size, ID_SYNCER); if (err) { drbd_err(device, "drbd_send_drequest() failed, aborting...\n"); @@ -828,6 +852,7 @@ static void ping_peer(struct drbd_device *device) int drbd_resync_finished(struct drbd_device *device) { + struct drbd_connection *connection = first_peer_device(device)->connection; unsigned long db, dt, dbdt; unsigned long n_oos; union drbd_state os, ns; @@ -849,8 +874,7 @@ int drbd_resync_finished(struct drbd_device *device) if (dw) { dw->w.cb = w_resync_finished; dw->device = device; - drbd_queue_work(&first_peer_device(device)->connection->sender_work, - &dw->w); + drbd_queue_work(&connection->sender_work, &dw->w); return 1; } drbd_err(device, "Warn failed to drbd_rs_del_all() and to kmalloc(dw).\n"); @@ -963,6 +987,30 @@ int drbd_resync_finished(struct drbd_device *device) _drbd_set_state(device, ns, CS_VERBOSE, NULL); out_unlock: spin_unlock_irq(&device->resource->req_lock); + + /* If we have been sync source, and have an effective fencing-policy, + * once *all* volumes are back in sync, call "unfence". */ + if (os.conn == C_SYNC_SOURCE) { + enum drbd_disk_state disk_state = D_MASK; + enum drbd_disk_state pdsk_state = D_MASK; + enum drbd_fencing_p fp = FP_DONT_CARE; + + rcu_read_lock(); + fp = rcu_dereference(device->ldev->disk_conf)->fencing; + if (fp != FP_DONT_CARE) { + struct drbd_peer_device *peer_device; + int vnr; + idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { + struct drbd_device *device = peer_device->device; + disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk); + pdsk_state = min_t(enum drbd_disk_state, pdsk_state, device->state.pdsk); + } + } + rcu_read_unlock(); + if (disk_state == D_UP_TO_DATE && pdsk_state == D_UP_TO_DATE) + conn_khelper(connection, "unfence-peer"); + } + put_ldev(device); out: device->rs_total = 0; @@ -999,7 +1047,6 @@ static void move_to_net_ee_or_free(struct drbd_device *device, struct drbd_peer_ /** * w_e_end_data_req() - Worker callback, to send a P_DATA_REPLY packet in response to a P_DATA_REQUEST - * @device: DRBD device. * @w: work object. * @cancel: The connection will be closed anyways */ @@ -1035,6 +1082,30 @@ int w_e_end_data_req(struct drbd_work *w, int cancel) return err; } +static bool all_zero(struct drbd_peer_request *peer_req) +{ + struct page *page = peer_req->pages; + unsigned int len = peer_req->i.size; + + page_chain_for_each(page) { + unsigned int l = min_t(unsigned int, len, PAGE_SIZE); + unsigned int i, words = l / sizeof(long); + unsigned long *d; + + d = kmap_atomic(page); + for (i = 0; i < words; i++) { + if (d[i]) { + kunmap_atomic(d); + return false; + } + } + kunmap_atomic(d); + len -= l; + } + + return true; +} + /** * w_e_end_rsdata_req() - Worker callback to send a P_RS_DATA_REPLY packet in response to a P_RS_DATA_REQUEST * @w: work object. @@ -1063,7 +1134,10 @@ int w_e_end_rsdata_req(struct drbd_work *w, int cancel) } else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { if (likely(device->state.pdsk >= D_INCONSISTENT)) { inc_rs_pending(device); - err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req); + if (peer_req->flags & EE_RS_THIN_REQ && all_zero(peer_req)) + err = drbd_send_rs_deallocated(peer_device, peer_req); + else + err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req); } else { if (__ratelimit(&drbd_ratelimit_state)) drbd_err(device, "Not sending RSDataReply, " @@ -1633,7 +1707,7 @@ static bool use_checksum_based_resync(struct drbd_connection *connection, struct rcu_read_unlock(); return connection->agreed_pro_version >= 89 && /* supported? */ connection->csums_tfm && /* configured? */ - (csums_after_crash_only == 0 /* use for each resync? */ + (csums_after_crash_only == false /* use for each resync? */ || test_bit(CRASHED_PRIMARY, &device->flags)); /* or only after Primary crash? */ } @@ -1768,7 +1842,7 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side) device->bm_resync_fo = 0; device->use_csums = use_checksum_based_resync(connection, device); } else { - device->use_csums = 0; + device->use_csums = false; } /* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 84708a5f8c52..c557057fe8ae 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -3822,8 +3822,9 @@ static int __floppy_read_block_0(struct block_device *bdev, int drive) bio.bi_flags |= (1 << BIO_QUIET); bio.bi_private = &cbdata; bio.bi_end_io = floppy_rb0_cb; + bio_set_op_attrs(&bio, REQ_OP_READ, 0); - submit_bio(READ, &bio); + submit_bio(&bio); process_fd_request(); init_completion(&cbdata.complete); @@ -4349,8 +4350,7 @@ static int __init do_floppy_init(void) /* to be cleaned up... */ disks[drive]->private_data = (void *)(long)drive; disks[drive]->flags |= GENHD_FL_REMOVABLE; - disks[drive]->driverfs_dev = &floppy_device[drive].dev; - add_disk(disks[drive]); + device_add_disk(&floppy_device[drive].dev, disks[drive]); } return 0; diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 1fa8cc235977..075377eee0c0 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -447,7 +447,7 @@ static int lo_req_flush(struct loop_device *lo, struct request *rq) static inline void handle_partial_read(struct loop_cmd *cmd, long bytes) { - if (bytes < 0 || (cmd->rq->cmd_flags & REQ_WRITE)) + if (bytes < 0 || op_is_write(req_op(cmd->rq))) return; if (unlikely(bytes < blk_rq_bytes(cmd->rq))) { @@ -541,10 +541,10 @@ static int do_req_filebacked(struct loop_device *lo, struct request *rq) pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset; - if (rq->cmd_flags & REQ_WRITE) { - if (rq->cmd_flags & REQ_FLUSH) + if (op_is_write(req_op(rq))) { + if (req_op(rq) == REQ_OP_FLUSH) ret = lo_req_flush(lo, rq); - else if (rq->cmd_flags & REQ_DISCARD) + else if (req_op(rq) == REQ_OP_DISCARD) ret = lo_discard(lo, rq, pos); else if (lo->transfer) ret = lo_write_transfer(lo, rq, pos); @@ -1659,8 +1659,8 @@ static int loop_queue_rq(struct blk_mq_hw_ctx *hctx, if (lo->lo_state != Lo_bound) return -EIO; - if (lo->use_dio && !(cmd->rq->cmd_flags & (REQ_FLUSH | - REQ_DISCARD))) + if (lo->use_dio && (req_op(cmd->rq) != REQ_OP_FLUSH || + req_op(cmd->rq) == REQ_OP_DISCARD)) cmd->use_aio = true; else cmd->use_aio = false; @@ -1672,7 +1672,7 @@ static int loop_queue_rq(struct blk_mq_hw_ctx *hctx, static void loop_handle_cmd(struct loop_cmd *cmd) { - const bool write = cmd->rq->cmd_flags & REQ_WRITE; + const bool write = op_is_write(req_op(cmd->rq)); struct loop_device *lo = cmd->rq->q->queuedata; int ret = 0; @@ -1765,6 +1765,7 @@ static int loop_add(struct loop_device **l, int i) */ queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, lo->lo_queue); + err = -ENOMEM; disk = lo->lo_disk = alloc_disk(1 << part_shift); if (!disk) goto out_free_queue; diff --git a/drivers/block/mg_disk.c b/drivers/block/mg_disk.c index 145ce2aa2e78..e937fcf71769 100644 --- a/drivers/block/mg_disk.c +++ b/drivers/block/mg_disk.c @@ -687,15 +687,13 @@ static unsigned int mg_issue_req(struct request *req, unsigned int sect_num, unsigned int sect_cnt) { - switch (rq_data_dir(req)) { - case READ: + if (rq_data_dir(req) == READ) { if (mg_out(host, sect_num, sect_cnt, MG_CMD_RD, &mg_read_intr) != MG_ERR_NONE) { mg_bad_rw_intr(host); return host->error; } - break; - case WRITE: + } else { /* TODO : handler */ outb(ATA_NIEN, (unsigned long)host->dev_base + MG_REG_DRV_CTRL); if (mg_out(host, sect_num, sect_cnt, MG_CMD_WR, &mg_write_intr) @@ -714,7 +712,6 @@ static unsigned int mg_issue_req(struct request *req, mod_timer(&host->timer, jiffies + 3 * HZ); outb(MG_CMD_WR_CONF, (unsigned long)host->dev_base + MG_REG_COMMAND); - break; } return MG_ERR_NONE; } @@ -1018,7 +1015,7 @@ probe_err_7: probe_err_6: blk_cleanup_queue(host->breq); probe_err_5: - unregister_blkdev(MG_DISK_MAJ, MG_DISK_NAME); + unregister_blkdev(host->major, MG_DISK_NAME); probe_err_4: if (!prv_data->use_polling) free_irq(host->irq, host); diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 6053e4659fa2..2aca98e8e427 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3765,7 +3765,7 @@ static int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq) return -ENODATA; } - if (rq->cmd_flags & REQ_DISCARD) { + if (req_op(rq) == REQ_OP_DISCARD) { int err; err = mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq)); @@ -3956,7 +3956,6 @@ static int mtip_block_initialize(struct driver_data *dd) if (rv) goto disk_index_error; - dd->disk->driverfs_dev = &dd->pdev->dev; dd->disk->major = dd->major; dd->disk->first_minor = index * MTIP_MAX_MINORS; dd->disk->minors = MTIP_MAX_MINORS; @@ -4008,7 +4007,7 @@ skip_create_disk: /* * if rebuild pending, start the service thread, and delay the block - * queue creation and add_disk() + * queue creation and device_add_disk() */ if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) goto start_service_thread; @@ -4042,7 +4041,7 @@ skip_create_disk: set_capacity(dd->disk, capacity); /* Enable the block device and add it to /dev */ - add_disk(dd->disk); + device_add_disk(&dd->pdev->dev, dd->disk); dd->bdev = bdget_disk(dd->disk, 0); /* diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 6a48ed41963f..6f55b262b5ce 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -282,9 +282,9 @@ static int nbd_send_req(struct nbd_device *nbd, struct request *req) if (req->cmd_type == REQ_TYPE_DRV_PRIV) type = NBD_CMD_DISC; - else if (req->cmd_flags & REQ_DISCARD) + else if (req_op(req) == REQ_OP_DISCARD) type = NBD_CMD_TRIM; - else if (req->cmd_flags & REQ_FLUSH) + else if (req_op(req) == REQ_OP_FLUSH) type = NBD_CMD_FLUSH; else if (rq_data_dir(req) == WRITE) type = NBD_CMD_WRITE; diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index cab97593ba54..75a7f88d6717 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -448,7 +448,7 @@ static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) struct request *rq; struct bio *bio = rqd->bio; - rq = blk_mq_alloc_request(q, bio_rw(bio), 0); + rq = blk_mq_alloc_request(q, bio_data_dir(bio), 0); if (IS_ERR(rq)) return -ENOMEM; diff --git a/drivers/block/osdblk.c b/drivers/block/osdblk.c index c2854a2bfdb0..92900f5f0b47 100644 --- a/drivers/block/osdblk.c +++ b/drivers/block/osdblk.c @@ -321,7 +321,7 @@ static void osdblk_rq_fn(struct request_queue *q) * driver-specific, etc. */ - do_flush = rq->cmd_flags & REQ_FLUSH; + do_flush = (req_op(rq) == REQ_OP_FLUSH); do_write = (rq_data_dir(rq) == WRITE); if (!do_flush) { /* osd_flush does not use a bio */ diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index d06c62eccdf0..9393bc730acf 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -1074,7 +1074,7 @@ static void pkt_gather_data(struct pktcdvd_device *pd, struct packet_data *pkt) BUG(); atomic_inc(&pkt->io_wait); - bio->bi_rw = READ; + bio_set_op_attrs(bio, REQ_OP_READ, 0); pkt_queue_bio(pd, bio); frames_read++; } @@ -1336,7 +1336,7 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt) /* Start the write request */ atomic_set(&pkt->io_wait, 1); - pkt->w_bio->bi_rw = WRITE; + bio_set_op_attrs(pkt->w_bio, REQ_OP_WRITE, 0); pkt_queue_bio(pd, pkt->w_bio); } diff --git a/drivers/block/ps3disk.c b/drivers/block/ps3disk.c index 4b7e405830d7..76f33c84ce3d 100644 --- a/drivers/block/ps3disk.c +++ b/drivers/block/ps3disk.c @@ -196,7 +196,7 @@ static void ps3disk_do_request(struct ps3_storage_device *dev, dev_dbg(&dev->sbd.core, "%s:%u\n", __func__, __LINE__); while ((req = blk_fetch_request(q))) { - if (req->cmd_flags & REQ_FLUSH) { + if (req_op(req) == REQ_OP_FLUSH) { if (ps3disk_submit_flush_request(dev, req)) break; } else if (req->cmd_type == REQ_TYPE_FS) { @@ -256,7 +256,7 @@ static irqreturn_t ps3disk_interrupt(int irq, void *data) return IRQ_HANDLED; } - if (req->cmd_flags & REQ_FLUSH) { + if (req_op(req) == REQ_OP_FLUSH) { read = 0; op = "flush"; } else { @@ -487,7 +487,6 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev) gendisk->fops = &ps3disk_fops; gendisk->queue = queue; gendisk->private_data = dev; - gendisk->driverfs_dev = &dev->sbd.core; snprintf(gendisk->disk_name, sizeof(gendisk->disk_name), PS3DISK_NAME, devidx+'a'); priv->blocking_factor = dev->blk_size >> 9; @@ -499,7 +498,7 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev) gendisk->disk_name, priv->model, priv->raw_capacity >> 11, get_capacity(gendisk) >> 11); - add_disk(gendisk); + device_add_disk(&dev->sbd.core, gendisk); return 0; fail_cleanup_queue: diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index 56847fcda086..456b4fe21559 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -773,14 +773,13 @@ static int ps3vram_probe(struct ps3_system_bus_device *dev) gendisk->fops = &ps3vram_fops; gendisk->queue = queue; gendisk->private_data = dev; - gendisk->driverfs_dev = &dev->core; strlcpy(gendisk->disk_name, DEVICE_NAME, sizeof(gendisk->disk_name)); set_capacity(gendisk, priv->size >> 9); dev_info(&dev->core, "%s: Using %lu MiB of GPU memory\n", gendisk->disk_name, get_capacity(gendisk) >> 11); - add_disk(gendisk); + device_add_disk(&dev->core, gendisk); return 0; fail_cleanup_queue: diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 81666a56415e..450662055d97 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -3286,9 +3286,9 @@ static void rbd_queue_workfn(struct work_struct *work) goto err; } - if (rq->cmd_flags & REQ_DISCARD) + if (req_op(rq) == REQ_OP_DISCARD) op_type = OBJ_OP_DISCARD; - else if (rq->cmd_flags & REQ_WRITE) + else if (req_op(rq) == REQ_OP_WRITE) op_type = OBJ_OP_WRITE; else op_type = OBJ_OP_READ; diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index e1b8b7061d2f..f81d70b39d10 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c @@ -230,8 +230,7 @@ int rsxx_attach_dev(struct rsxx_cardinfo *card) set_capacity(card->gendisk, card->size8 >> 9); else set_capacity(card->gendisk, 0); - add_disk(card->gendisk); - + device_add_disk(CARD_TO_DEV(card), card->gendisk); card->bdev_attached = 1; } @@ -308,7 +307,6 @@ int rsxx_setup_dev(struct rsxx_cardinfo *card) snprintf(card->gendisk->disk_name, sizeof(card->gendisk->disk_name), "rsxx%d", card->disk_id); - card->gendisk->driverfs_dev = &card->dev->dev; card->gendisk->major = card->major; card->gendisk->first_minor = 0; card->gendisk->fops = &rsxx_fops; diff --git a/drivers/block/rsxx/dma.c b/drivers/block/rsxx/dma.c index cf8cd293abb5..5a20385f87d0 100644 --- a/drivers/block/rsxx/dma.c +++ b/drivers/block/rsxx/dma.c @@ -705,7 +705,7 @@ int rsxx_dma_queue_bio(struct rsxx_cardinfo *card, dma_cnt[i] = 0; } - if (bio->bi_rw & REQ_DISCARD) { + if (bio_op(bio) == REQ_OP_DISCARD) { bv_len = bio->bi_iter.bi_size; while (bv_len > 0) { diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c index 910e065918af..3822eae102db 100644 --- a/drivers/block/skd_main.c +++ b/drivers/block/skd_main.c @@ -597,7 +597,7 @@ static void skd_request_fn(struct request_queue *q) data_dir = rq_data_dir(req); io_flags = req->cmd_flags; - if (io_flags & REQ_FLUSH) + if (req_op(req) == REQ_OP_FLUSH) flush++; if (io_flags & REQ_FUA) @@ -4690,10 +4690,10 @@ static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo) return -EIO; } -static int skd_bdev_attach(struct skd_device *skdev) +static int skd_bdev_attach(struct device *parent, struct skd_device *skdev) { pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__); - add_disk(skdev->disk); + device_add_disk(parent, skdev->disk); return 0; } @@ -4812,8 +4812,6 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_drvdata(pdev, skdev); - skdev->disk->driverfs_dev = &pdev->dev; - for (i = 0; i < SKD_MAX_BARS; i++) { skdev->mem_phys[i] = pci_resource_start(pdev, i); skdev->mem_size[i] = (u32)pci_resource_len(pdev, i); @@ -4851,7 +4849,7 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) (SKD_START_WAIT_SECONDS * HZ)); if (skdev->gendisk_on > 0) { /* device came on-line after reset */ - skd_bdev_attach(skdev); + skd_bdev_attach(&pdev->dev, skdev); rc = 0; } else { /* we timed out, something is wrong with the device, diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index 4b911ed96ea3..cab157331c4e 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -804,7 +804,6 @@ static int probe_disk(struct vdc_port *port) g->fops = &vdc_fops; g->queue = q; g->private_data = port; - g->driverfs_dev = &port->vio.vdev->dev; set_capacity(g, port->vdisk_size); @@ -835,7 +834,7 @@ static int probe_disk(struct vdc_port *port) port->vdisk_size, (port->vdisk_size >> (20 - 9)), port->vio.ver.major, port->vio.ver.minor); - add_disk(g); + device_add_disk(&port->vio.vdev->dev, g); return 0; } diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 7939b9f87441..d0a3e6d4515f 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c @@ -344,7 +344,6 @@ static int add_bio(struct cardinfo *card) int offset; struct bio *bio; struct bio_vec vec; - int rw; bio = card->currentbio; if (!bio && card->bio) { @@ -359,7 +358,6 @@ static int add_bio(struct cardinfo *card) if (!bio) return 0; - rw = bio_rw(bio); if (card->mm_pages[card->Ready].cnt >= DESC_PER_PAGE) return 0; @@ -369,7 +367,7 @@ static int add_bio(struct cardinfo *card) vec.bv_page, vec.bv_offset, vec.bv_len, - (rw == READ) ? + bio_op(bio) == REQ_OP_READ ? PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE); p = &card->mm_pages[card->Ready]; @@ -398,7 +396,7 @@ static int add_bio(struct cardinfo *card) DMASCR_CHAIN_EN | DMASCR_SEM_EN | pci_cmds); - if (rw == WRITE) + if (bio_op(bio) == REQ_OP_WRITE) desc->control_bits |= cpu_to_le32(DMASCR_TRANSFER_READ); desc->sem_control_bits = desc->control_bits; @@ -462,7 +460,7 @@ static void process_page(unsigned long data) le32_to_cpu(desc->local_addr)>>9, le32_to_cpu(desc->transfer_size)); dump_dmastat(card, control); - } else if ((bio->bi_rw & REQ_WRITE) && + } else if (op_is_write(bio_op(bio)) && le32_to_cpu(desc->local_addr) >> 9 == card->init_size) { card->init_size += le32_to_cpu(desc->transfer_size) >> 9; diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 42758b52768c..1523e05c46fc 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -172,7 +172,7 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); vbr->req = req; - if (req->cmd_flags & REQ_FLUSH) { + if (req_op(req) == REQ_OP_FLUSH) { vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_FLUSH); vbr->out_hdr.sector = 0; vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req)); @@ -236,25 +236,22 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, static int virtblk_get_id(struct gendisk *disk, char *id_str) { struct virtio_blk *vblk = disk->private_data; + struct request_queue *q = vblk->disk->queue; struct request *req; - struct bio *bio; int err; - bio = bio_map_kern(vblk->disk->queue, id_str, VIRTIO_BLK_ID_BYTES, - GFP_KERNEL); - if (IS_ERR(bio)) - return PTR_ERR(bio); - - req = blk_make_request(vblk->disk->queue, bio, GFP_KERNEL); - if (IS_ERR(req)) { - bio_put(bio); + req = blk_get_request(q, READ, GFP_KERNEL); + if (IS_ERR(req)) return PTR_ERR(req); - } - req->cmd_type = REQ_TYPE_DRV_PRIV; + + err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL); + if (err) + goto out; + err = blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); +out: blk_put_request(req); - return err; } @@ -656,7 +653,6 @@ static int virtblk_probe(struct virtio_device *vdev) vblk->disk->first_minor = index_to_minor(index); vblk->disk->private_data = vblk; vblk->disk->fops = &virtblk_fops; - vblk->disk->driverfs_dev = &vdev->dev; vblk->disk->flags |= GENHD_FL_EXT_DEVT; vblk->index = index; @@ -733,7 +729,7 @@ static int virtblk_probe(struct virtio_device *vdev) virtio_device_ready(vdev); - add_disk(vblk->disk); + device_add_disk(&vdev->dev, vblk->disk); err = device_create_file(disk_to_dev(vblk->disk), &dev_attr_serial); if (err) goto out_del_disk; diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index 4809c1501d7e..4a80ee752597 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -501,7 +501,7 @@ static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif, struct xen_vbd *vbd = &blkif->vbd; int rc = -EACCES; - if ((operation != READ) && vbd->readonly) + if ((operation != REQ_OP_READ) && vbd->readonly) goto out; if (likely(req->nr_sects)) { @@ -1014,7 +1014,7 @@ static int dispatch_discard_io(struct xen_blkif_ring *ring, preq.sector_number = req->u.discard.sector_number; preq.nr_sects = req->u.discard.nr_sectors; - err = xen_vbd_translate(&preq, blkif, WRITE); + err = xen_vbd_translate(&preq, blkif, REQ_OP_WRITE); if (err) { pr_warn("access denied: DISCARD [%llu->%llu] on dev=%04x\n", preq.sector_number, @@ -1229,6 +1229,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, struct bio **biolist = pending_req->biolist; int i, nbio = 0; int operation; + int operation_flags = 0; struct blk_plug plug; bool drain = false; struct grant_page **pages = pending_req->segments; @@ -1247,17 +1248,19 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, switch (req_operation) { case BLKIF_OP_READ: ring->st_rd_req++; - operation = READ; + operation = REQ_OP_READ; break; case BLKIF_OP_WRITE: ring->st_wr_req++; - operation = WRITE_ODIRECT; + operation = REQ_OP_WRITE; + operation_flags = WRITE_ODIRECT; break; case BLKIF_OP_WRITE_BARRIER: drain = true; case BLKIF_OP_FLUSH_DISKCACHE: ring->st_f_req++; - operation = WRITE_FLUSH; + operation = REQ_OP_WRITE; + operation_flags = WRITE_FLUSH; break; default: operation = 0; /* make gcc happy */ @@ -1269,7 +1272,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, nseg = req->operation == BLKIF_OP_INDIRECT ? req->u.indirect.nr_segments : req->u.rw.nr_segments; - if (unlikely(nseg == 0 && operation != WRITE_FLUSH) || + if (unlikely(nseg == 0 && operation_flags != WRITE_FLUSH) || unlikely((req->operation != BLKIF_OP_INDIRECT) && (nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) || unlikely((req->operation == BLKIF_OP_INDIRECT) && @@ -1310,7 +1313,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, if (xen_vbd_translate(&preq, ring->blkif, operation) != 0) { pr_debug("access denied: %s of [%llu,%llu] on dev=%04x\n", - operation == READ ? "read" : "write", + operation == REQ_OP_READ ? "read" : "write", preq.sector_number, preq.sector_number + preq.nr_sects, ring->blkif->vbd.pdevice); @@ -1369,6 +1372,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, bio->bi_private = pending_req; bio->bi_end_io = end_block_io_op; bio->bi_iter.bi_sector = preq.sector_number; + bio_set_op_attrs(bio, operation, operation_flags); } preq.sector_number += seg[i].nsec; @@ -1376,7 +1380,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, /* This will be hit if the operation was a flush or discard. */ if (!bio) { - BUG_ON(operation != WRITE_FLUSH); + BUG_ON(operation_flags != WRITE_FLUSH); bio = bio_alloc(GFP_KERNEL, 0); if (unlikely(bio == NULL)) @@ -1386,20 +1390,21 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, bio->bi_bdev = preq.bdev; bio->bi_private = pending_req; bio->bi_end_io = end_block_io_op; + bio_set_op_attrs(bio, operation, operation_flags); } atomic_set(&pending_req->pendcnt, nbio); blk_start_plug(&plug); for (i = 0; i < nbio; i++) - submit_bio(operation, biolist[i]); + submit_bio(biolist[i]); /* Let the I/Os go.. */ blk_finish_plug(&plug); - if (operation == READ) + if (operation == REQ_OP_READ) ring->st_rd_sect += preq.nr_sects; - else if (operation & WRITE) + else if (operation == REQ_OP_WRITE) ring->st_wr_sect += preq.nr_sects; return 0; diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index 3355f1cdd4e5..3cc6d1d86f1e 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -379,7 +379,7 @@ static struct attribute *xen_vbdstat_attrs[] = { NULL }; -static struct attribute_group xen_vbdstat_group = { +static const struct attribute_group xen_vbdstat_group = { .name = "statistics", .attrs = xen_vbdstat_attrs, }; @@ -480,7 +480,7 @@ static int xen_vbd_create(struct xen_blkif *blkif, blkif_vdev_t handle, if (q && test_bit(QUEUE_FLAG_WC, &q->queue_flags)) vbd->flush_support = true; - if (q && blk_queue_secdiscard(q)) + if (q && blk_queue_secure_erase(q)) vbd->discard_secure = true; pr_debug("Successful creation of handle=%04x (dom=%u)\n", @@ -715,8 +715,11 @@ static void backend_changed(struct xenbus_watch *watch, /* Front end dir is a number, which is used as the handle. */ err = kstrtoul(strrchr(dev->otherend, '/') + 1, 0, &handle); - if (err) + if (err) { + kfree(be->mode); + be->mode = NULL; return; + } be->major = major; be->minor = minor; @@ -1022,9 +1025,9 @@ static int connect_ring(struct backend_info *be) pr_debug("%s %s\n", __func__, dev->otherend); be->blkif->blk_protocol = BLKIF_PROTOCOL_DEFAULT; - err = xenbus_gather(XBT_NIL, dev->otherend, "protocol", - "%63s", protocol, NULL); - if (err) + err = xenbus_scanf(XBT_NIL, dev->otherend, "protocol", + "%63s", protocol); + if (err <= 0) strcpy(protocol, "unspecified, assuming default"); else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE)) be->blkif->blk_protocol = BLKIF_PROTOCOL_NATIVE; @@ -1036,10 +1039,9 @@ static int connect_ring(struct backend_info *be) xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol); return -ENOSYS; } - err = xenbus_gather(XBT_NIL, dev->otherend, - "feature-persistent", "%u", - &pers_grants, NULL); - if (err) + err = xenbus_scanf(XBT_NIL, dev->otherend, + "feature-persistent", "%u", &pers_grants); + if (err <= 0) pers_grants = 0; be->blkif->vbd.feature_gnt_persistent = pers_grants; diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 2e6d1e9c3345..be4fea6a5dd3 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -196,6 +196,7 @@ struct blkfront_info unsigned int nr_ring_pages; struct request_queue *rq; unsigned int feature_flush; + unsigned int feature_fua; unsigned int feature_discard:1; unsigned int feature_secdiscard:1; unsigned int discard_granularity; @@ -207,6 +208,9 @@ struct blkfront_info struct blk_mq_tag_set tag_set; struct blkfront_ring_info *rinfo; unsigned int nr_rings; + /* Save uncomplete reqs and bios for migration. */ + struct list_head requests; + struct bio_list bio_list; }; static unsigned int nr_minors; @@ -544,7 +548,7 @@ static int blkif_queue_discard_req(struct request *req, struct blkfront_ring_inf ring_req->u.discard.nr_sectors = blk_rq_sectors(req); ring_req->u.discard.id = id; ring_req->u.discard.sector_number = (blkif_sector_t)blk_rq_pos(req); - if ((req->cmd_flags & REQ_SECURE) && info->feature_secdiscard) + if (req_op(req) == REQ_OP_SECURE_ERASE && info->feature_secdiscard) ring_req->u.discard.flag = BLKIF_DISCARD_SECURE; else ring_req->u.discard.flag = 0; @@ -743,7 +747,7 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri * The indirect operation can only be a BLKIF_OP_READ or * BLKIF_OP_WRITE */ - BUG_ON(req->cmd_flags & (REQ_FLUSH | REQ_FUA)); + BUG_ON(req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA); ring_req->operation = BLKIF_OP_INDIRECT; ring_req->u.indirect.indirect_op = rq_data_dir(req) ? BLKIF_OP_WRITE : BLKIF_OP_READ; @@ -755,7 +759,7 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri ring_req->u.rw.handle = info->handle; ring_req->operation = rq_data_dir(req) ? BLKIF_OP_WRITE : BLKIF_OP_READ; - if (req->cmd_flags & (REQ_FLUSH | REQ_FUA)) { + if (req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA) { /* * Ideally we can do an unordered flush-to-disk. * In case the backend onlysupports barriers, use that. @@ -763,19 +767,14 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri * implement it the same way. (It's also a FLUSH+FUA, * since it is guaranteed ordered WRT previous writes.) */ - switch (info->feature_flush & - ((REQ_FLUSH|REQ_FUA))) { - case REQ_FLUSH|REQ_FUA: + if (info->feature_flush && info->feature_fua) ring_req->operation = BLKIF_OP_WRITE_BARRIER; - break; - case REQ_FLUSH: + else if (info->feature_flush) ring_req->operation = BLKIF_OP_FLUSH_DISKCACHE; - break; - default: + else ring_req->operation = 0; - } } ring_req->u.rw.nr_segments = num_grant; if (unlikely(require_extra_req)) { @@ -844,7 +843,8 @@ static int blkif_queue_request(struct request *req, struct blkfront_ring_info *r if (unlikely(rinfo->dev_info->connected != BLKIF_STATE_CONNECTED)) return 1; - if (unlikely(req->cmd_flags & (REQ_DISCARD | REQ_SECURE))) + if (unlikely(req_op(req) == REQ_OP_DISCARD || + req_op(req) == REQ_OP_SECURE_ERASE)) return blkif_queue_discard_req(req, rinfo); else return blkif_queue_rw_req(req, rinfo); @@ -864,10 +864,10 @@ static inline bool blkif_request_flush_invalid(struct request *req, struct blkfront_info *info) { return ((req->cmd_type != REQ_TYPE_FS) || - ((req->cmd_flags & REQ_FLUSH) && - !(info->feature_flush & REQ_FLUSH)) || + ((req_op(req) == REQ_OP_FLUSH) && + !info->feature_flush) || ((req->cmd_flags & REQ_FUA) && - !(info->feature_flush & REQ_FUA))); + !info->feature_fua)); } static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx, @@ -952,7 +952,7 @@ static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size, rq->limits.discard_granularity = info->discard_granularity; rq->limits.discard_alignment = info->discard_alignment; if (info->feature_secdiscard) - queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, rq); + queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, rq); } /* Hard sector size and max sectors impersonate the equiv. hardware. */ @@ -978,24 +978,22 @@ static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size, return 0; } -static const char *flush_info(unsigned int feature_flush) +static const char *flush_info(struct blkfront_info *info) { - switch (feature_flush & ((REQ_FLUSH | REQ_FUA))) { - case REQ_FLUSH|REQ_FUA: + if (info->feature_flush && info->feature_fua) return "barrier: enabled;"; - case REQ_FLUSH: + else if (info->feature_flush) return "flush diskcache: enabled;"; - default: + else return "barrier or flush: disabled;"; - } } static void xlvbd_flush(struct blkfront_info *info) { - blk_queue_write_cache(info->rq, info->feature_flush & REQ_FLUSH, - info->feature_flush & REQ_FUA); + blk_queue_write_cache(info->rq, info->feature_flush ? true : false, + info->feature_fua ? true : false); pr_info("blkfront: %s: %s %s %s %s %s\n", - info->gd->disk_name, flush_info(info->feature_flush), + info->gd->disk_name, flush_info(info), "persistent grants:", info->feature_persistent ? "enabled;" : "disabled;", "indirect descriptors:", info->max_indirect_segments ? "enabled;" : "disabled;"); @@ -1136,7 +1134,6 @@ static int xlvbd_alloc_gendisk(blkif_sector_t capacity, gd->first_minor = minor; gd->fops = &xlvbd_block_fops; gd->private_data = info; - gd->driverfs_dev = &(info->xbdev->dev); set_capacity(gd, capacity); if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size, @@ -1594,7 +1591,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) info->feature_discard = 0; info->feature_secdiscard = 0; queue_flag_clear(QUEUE_FLAG_DISCARD, rq); - queue_flag_clear(QUEUE_FLAG_SECDISCARD, rq); + queue_flag_clear(QUEUE_FLAG_SECERASE, rq); } blk_mq_complete_request(req, error); break; @@ -1614,6 +1611,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) if (unlikely(error)) { if (error == -EOPNOTSUPP) error = 0; + info->feature_fua = 0; info->feature_flush = 0; xlvbd_flush(info); } @@ -2002,69 +2000,22 @@ static int blkif_recover(struct blkfront_info *info) { unsigned int i, r_index; struct request *req, *n; - struct blk_shadow *copy; int rc; struct bio *bio, *cloned_bio; - struct bio_list bio_list, merge_bio; unsigned int segs, offset; int pending, size; struct split_bio *split_bio; - struct list_head requests; blkfront_gather_backend_features(info); segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST; blk_queue_max_segments(info->rq, segs); - bio_list_init(&bio_list); - INIT_LIST_HEAD(&requests); for (r_index = 0; r_index < info->nr_rings; r_index++) { - struct blkfront_ring_info *rinfo; - - rinfo = &info->rinfo[r_index]; - /* Stage 1: Make a safe copy of the shadow state. */ - copy = kmemdup(rinfo->shadow, sizeof(rinfo->shadow), - GFP_NOIO | __GFP_REPEAT | __GFP_HIGH); - if (!copy) - return -ENOMEM; - - /* Stage 2: Set up free list. */ - memset(&rinfo->shadow, 0, sizeof(rinfo->shadow)); - for (i = 0; i < BLK_RING_SIZE(info); i++) - rinfo->shadow[i].req.u.rw.id = i+1; - rinfo->shadow_free = rinfo->ring.req_prod_pvt; - rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff; + struct blkfront_ring_info *rinfo = &info->rinfo[r_index]; rc = blkfront_setup_indirect(rinfo); - if (rc) { - kfree(copy); + if (rc) return rc; - } - - for (i = 0; i < BLK_RING_SIZE(info); i++) { - /* Not in use? */ - if (!copy[i].request) - continue; - - /* - * Get the bios in the request so we can re-queue them. - */ - if (copy[i].request->cmd_flags & - (REQ_FLUSH | REQ_FUA | REQ_DISCARD | REQ_SECURE)) { - /* - * Flush operations don't contain bios, so - * we need to requeue the whole request - */ - list_add(©[i].request->queuelist, &requests); - continue; - } - merge_bio.head = copy[i].request->bio; - merge_bio.tail = copy[i].request->biotail; - bio_list_merge(&bio_list, &merge_bio); - copy[i].request->bio = NULL; - blk_end_request_all(copy[i].request, 0); - } - - kfree(copy); } xenbus_switch_state(info->xbdev, XenbusStateConnected); @@ -2079,7 +2030,7 @@ static int blkif_recover(struct blkfront_info *info) kick_pending_request_queues(rinfo); } - list_for_each_entry_safe(req, n, &requests, queuelist) { + list_for_each_entry_safe(req, n, &info->requests, queuelist) { /* Requeue pending requests (flush or discard) */ list_del_init(&req->queuelist); BUG_ON(req->nr_phys_segments > segs); @@ -2087,7 +2038,7 @@ static int blkif_recover(struct blkfront_info *info) } blk_mq_kick_requeue_list(info->rq); - while ((bio = bio_list_pop(&bio_list)) != NULL) { + while ((bio = bio_list_pop(&info->bio_list)) != NULL) { /* Traverse the list of pending bios and re-queue them */ if (bio_segments(bio) > segs) { /* @@ -2108,7 +2059,7 @@ static int blkif_recover(struct blkfront_info *info) bio_trim(cloned_bio, offset, size); cloned_bio->bi_private = split_bio; cloned_bio->bi_end_io = split_bio_end; - submit_bio(cloned_bio->bi_rw, cloned_bio); + submit_bio(cloned_bio); } /* * Now we have to wait for all those smaller bios to @@ -2117,7 +2068,7 @@ static int blkif_recover(struct blkfront_info *info) continue; } /* We don't need to split this bio */ - submit_bio(bio->bi_rw, bio); + submit_bio(bio); } return 0; @@ -2133,9 +2084,47 @@ static int blkfront_resume(struct xenbus_device *dev) { struct blkfront_info *info = dev_get_drvdata(&dev->dev); int err = 0; + unsigned int i, j; dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename); + bio_list_init(&info->bio_list); + INIT_LIST_HEAD(&info->requests); + for (i = 0; i < info->nr_rings; i++) { + struct blkfront_ring_info *rinfo = &info->rinfo[i]; + struct bio_list merge_bio; + struct blk_shadow *shadow = rinfo->shadow; + + for (j = 0; j < BLK_RING_SIZE(info); j++) { + /* Not in use? */ + if (!shadow[j].request) + continue; + + /* + * Get the bios in the request so we can re-queue them. + */ + if (req_op(shadow[i].request) == REQ_OP_FLUSH || + req_op(shadow[i].request) == REQ_OP_DISCARD || + req_op(shadow[i].request) == REQ_OP_SECURE_ERASE || + shadow[j].request->cmd_flags & REQ_FUA) { + /* + * Flush operations don't contain bios, so + * we need to requeue the whole request + * + * XXX: but this doesn't make any sense for a + * write with the FUA flag set.. + */ + list_add(&shadow[j].request->queuelist, &info->requests); + continue; + } + merge_bio.head = shadow[j].request->bio; + merge_bio.tail = shadow[j].request->biotail; + bio_list_merge(&info->bio_list, &merge_bio); + shadow[j].request->bio = NULL; + blk_mq_end_request(shadow[j].request, 0); + } + } + blkif_free(info, info->connected == BLKIF_STATE_CONNECTED); err = negotiate_mq(info); @@ -2208,10 +2197,9 @@ static void blkfront_setup_discard(struct blkfront_info *info) info->discard_granularity = discard_granularity; info->discard_alignment = discard_alignment; } - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "discard-secure", "%d", &discard_secure, - NULL); - if (!err) + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "discard-secure", "%u", &discard_secure); + if (err > 0) info->feature_secdiscard = !!discard_secure; } @@ -2309,10 +2297,10 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) unsigned int indirect_segments; info->feature_flush = 0; + info->feature_fua = 0; - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-barrier", "%d", &barrier, - NULL); + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-barrier", "%d", &barrier); /* * If there's no "feature-barrier" defined, then it means @@ -2321,38 +2309,40 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) * * If there are barriers, then we use flush. */ - if (!err && barrier) - info->feature_flush = REQ_FLUSH | REQ_FUA; + if (err > 0 && barrier) { + info->feature_flush = 1; + info->feature_fua = 1; + } + /* * And if there is "feature-flush-cache" use that above * barriers. */ - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-flush-cache", "%d", &flush, - NULL); + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-flush-cache", "%d", &flush); - if (!err && flush) - info->feature_flush = REQ_FLUSH; + if (err > 0 && flush) { + info->feature_flush = 1; + info->feature_fua = 0; + } - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-discard", "%d", &discard, - NULL); + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-discard", "%d", &discard); - if (!err && discard) + if (err > 0 && discard) blkfront_setup_discard(info); - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-persistent", "%u", &persistent, - NULL); - if (err) + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-persistent", "%d", &persistent); + if (err <= 0) info->feature_persistent = 0; else info->feature_persistent = persistent; - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-max-indirect-segments", "%u", &indirect_segments, - NULL); - if (err) + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-max-indirect-segments", "%u", + &indirect_segments); + if (err <= 0) info->max_indirect_segments = 0; else info->max_indirect_segments = min(indirect_segments, @@ -2452,7 +2442,7 @@ static void blkfront_connect(struct blkfront_info *info) for (i = 0; i < info->nr_rings; i++) kick_pending_request_queues(&info->rinfo[i]); - add_disk(info->gd); + device_add_disk(&info->xbdev->dev, info->gd); info->is_ready = 1; } diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig index 386ba3d1a6ee..b8ecba6dcd3b 100644 --- a/drivers/block/zram/Kconfig +++ b/drivers/block/zram/Kconfig @@ -1,8 +1,7 @@ config ZRAM tristate "Compressed RAM block device support" - depends on BLOCK && SYSFS && ZSMALLOC - select LZO_COMPRESS - select LZO_DECOMPRESS + depends on BLOCK && SYSFS && ZSMALLOC && CRYPTO + select CRYPTO_LZO default n help Creates virtual block devices called /dev/zramX (X = 0, 1, ...). @@ -14,13 +13,3 @@ config ZRAM disks and maybe many more. See zram.txt for more information. - -config ZRAM_LZ4_COMPRESS - bool "Enable LZ4 algorithm support" - depends on ZRAM - select LZ4_COMPRESS - select LZ4_DECOMPRESS - default n - help - This option enables LZ4 compression algorithm support. Compression - algorithm can be changed using `comp_algorithm' device attribute. \ No newline at end of file diff --git a/drivers/block/zram/Makefile b/drivers/block/zram/Makefile index be0763ff57a2..9e2b79e9a990 100644 --- a/drivers/block/zram/Makefile +++ b/drivers/block/zram/Makefile @@ -1,5 +1,3 @@ -zram-y := zcomp_lzo.o zcomp.o zram_drv.o - -zram-$(CONFIG_ZRAM_LZ4_COMPRESS) += zcomp_lz4.o +zram-y := zcomp.o zram_drv.o obj-$(CONFIG_ZRAM) += zram.o diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c index b51a816d766b..4b5cd3a7b2b6 100644 --- a/drivers/block/zram/zcomp.c +++ b/drivers/block/zram/zcomp.c @@ -14,108 +14,150 @@ #include #include #include +#include #include "zcomp.h" -#include "zcomp_lzo.h" -#ifdef CONFIG_ZRAM_LZ4_COMPRESS -#include "zcomp_lz4.h" -#endif -static struct zcomp_backend *backends[] = { - &zcomp_lzo, -#ifdef CONFIG_ZRAM_LZ4_COMPRESS - &zcomp_lz4, +static const char * const backends[] = { + "lzo", +#if IS_ENABLED(CONFIG_CRYPTO_LZ4) + "lz4", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_DEFLATE) + "deflate", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_LZ4HC) + "lz4hc", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_842) + "842", #endif NULL }; -static struct zcomp_backend *find_backend(const char *compress) -{ - int i = 0; - while (backends[i]) { - if (sysfs_streq(compress, backends[i]->name)) - break; - i++; - } - return backends[i]; -} - -static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm) +static void zcomp_strm_free(struct zcomp_strm *zstrm) { - if (zstrm->private) - comp->backend->destroy(zstrm->private); + if (!IS_ERR_OR_NULL(zstrm->tfm)) + crypto_free_comp(zstrm->tfm); free_pages((unsigned long)zstrm->buffer, 1); kfree(zstrm); } /* - * allocate new zcomp_strm structure with ->private initialized by + * allocate new zcomp_strm structure with ->tfm initialized by * backend, return NULL on error */ -static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp, gfp_t flags) +static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp) { - struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), flags); + struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL); if (!zstrm) return NULL; - zstrm->private = comp->backend->create(flags); + zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0); /* * allocate 2 pages. 1 for compressed data, plus 1 extra for the * case when compressed size is larger than the original one */ - zstrm->buffer = (void *)__get_free_pages(flags | __GFP_ZERO, 1); - if (!zstrm->private || !zstrm->buffer) { - zcomp_strm_free(comp, zstrm); + zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); + if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) { + zcomp_strm_free(zstrm); zstrm = NULL; } return zstrm; } +bool zcomp_available_algorithm(const char *comp) +{ + int i = 0; + + while (backends[i]) { + if (sysfs_streq(comp, backends[i])) + return true; + i++; + } + + /* + * Crypto does not ignore a trailing new line symbol, + * so make sure you don't supply a string containing + * one. + * This also means that we permit zcomp initialisation + * with any compressing algorithm known to crypto api. + */ + return crypto_has_comp(comp, 0, 0) == 1; +} + /* show available compressors */ ssize_t zcomp_available_show(const char *comp, char *buf) { + bool known_algorithm = false; ssize_t sz = 0; int i = 0; - while (backends[i]) { - if (!strcmp(comp, backends[i]->name)) + for (; backends[i]; i++) { + if (!strcmp(comp, backends[i])) { + known_algorithm = true; sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, - "[%s] ", backends[i]->name); - else + "[%s] ", backends[i]); + } else { sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, - "%s ", backends[i]->name); - i++; + "%s ", backends[i]); + } } + + /* + * Out-of-tree module known to crypto api or a missing + * entry in `backends'. + */ + if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1) + sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, + "[%s] ", comp); + sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); return sz; } -bool zcomp_available_algorithm(const char *comp) -{ - return find_backend(comp) != NULL; -} - -struct zcomp_strm *zcomp_strm_find(struct zcomp *comp) +struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) { return *get_cpu_ptr(comp->stream); } -void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm) +void zcomp_stream_put(struct zcomp *comp) { put_cpu_ptr(comp->stream); } -int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, - const unsigned char *src, size_t *dst_len) +int zcomp_compress(struct zcomp_strm *zstrm, + const void *src, unsigned int *dst_len) { - return comp->backend->compress(src, zstrm->buffer, dst_len, - zstrm->private); + /* + * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized + * because sometimes we can endup having a bigger compressed data + * due to various reasons: for example compression algorithms tend + * to add some padding to the compressed buffer. Speaking of padding, + * comp algorithm `842' pads the compressed length to multiple of 8 + * and returns -ENOSP when the dst memory is not big enough, which + * is not something that ZRAM wants to see. We can handle the + * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we + * receive -ERRNO from the compressing backend we can't help it + * anymore. To make `842' happy we need to tell the exact size of + * the dst buffer, zram_drv will take care of the fact that + * compressed buffer is too big. + */ + *dst_len = PAGE_SIZE * 2; + + return crypto_comp_compress(zstrm->tfm, + src, PAGE_SIZE, + zstrm->buffer, dst_len); } -int zcomp_decompress(struct zcomp *comp, const unsigned char *src, - size_t src_len, unsigned char *dst) +int zcomp_decompress(struct zcomp_strm *zstrm, + const void *src, unsigned int src_len, void *dst) { - return comp->backend->decompress(src, src_len, dst); + unsigned int dst_len = PAGE_SIZE; + + return crypto_comp_decompress(zstrm->tfm, + src, src_len, + dst, &dst_len); } static int __zcomp_cpu_notifier(struct zcomp *comp, @@ -127,7 +169,7 @@ static int __zcomp_cpu_notifier(struct zcomp *comp, case CPU_UP_PREPARE: if (WARN_ON(*per_cpu_ptr(comp->stream, cpu))) break; - zstrm = zcomp_strm_alloc(comp, GFP_KERNEL); + zstrm = zcomp_strm_alloc(comp); if (IS_ERR_OR_NULL(zstrm)) { pr_err("Can't allocate a compression stream\n"); return NOTIFY_BAD; @@ -138,7 +180,7 @@ static int __zcomp_cpu_notifier(struct zcomp *comp, case CPU_UP_CANCELED: zstrm = *per_cpu_ptr(comp->stream, cpu); if (!IS_ERR_OR_NULL(zstrm)) - zcomp_strm_free(comp, zstrm); + zcomp_strm_free(zstrm); *per_cpu_ptr(comp->stream, cpu) = NULL; break; default: @@ -209,18 +251,16 @@ void zcomp_destroy(struct zcomp *comp) struct zcomp *zcomp_create(const char *compress) { struct zcomp *comp; - struct zcomp_backend *backend; int error; - backend = find_backend(compress); - if (!backend) + if (!zcomp_available_algorithm(compress)) return ERR_PTR(-EINVAL); comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); if (!comp) return ERR_PTR(-ENOMEM); - comp->backend = backend; + comp->name = compress; error = zcomp_init(comp); if (error) { kfree(comp); diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h index ffd88cb747fe..478cac2ed465 100644 --- a/drivers/block/zram/zcomp.h +++ b/drivers/block/zram/zcomp.h @@ -13,33 +13,15 @@ struct zcomp_strm { /* compression/decompression buffer */ void *buffer; - /* - * The private data of the compression stream, only compression - * stream backend can touch this (e.g. compression algorithm - * working memory) - */ - void *private; -}; - -/* static compression backend */ -struct zcomp_backend { - int (*compress)(const unsigned char *src, unsigned char *dst, - size_t *dst_len, void *private); - - int (*decompress)(const unsigned char *src, size_t src_len, - unsigned char *dst); - - void *(*create)(gfp_t flags); - void (*destroy)(void *private); - - const char *name; + struct crypto_comp *tfm; }; /* dynamic per-device compression frontend */ struct zcomp { struct zcomp_strm * __percpu *stream; - struct zcomp_backend *backend; struct notifier_block notifier; + + const char *name; }; ssize_t zcomp_available_show(const char *comp, char *buf); @@ -48,14 +30,14 @@ bool zcomp_available_algorithm(const char *comp); struct zcomp *zcomp_create(const char *comp); void zcomp_destroy(struct zcomp *comp); -struct zcomp_strm *zcomp_strm_find(struct zcomp *comp); -void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm); +struct zcomp_strm *zcomp_stream_get(struct zcomp *comp); +void zcomp_stream_put(struct zcomp *comp); -int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, - const unsigned char *src, size_t *dst_len); +int zcomp_compress(struct zcomp_strm *zstrm, + const void *src, unsigned int *dst_len); -int zcomp_decompress(struct zcomp *comp, const unsigned char *src, - size_t src_len, unsigned char *dst); +int zcomp_decompress(struct zcomp_strm *zstrm, + const void *src, unsigned int src_len, void *dst); bool zcomp_set_max_streams(struct zcomp *comp, int num_strm); #endif /* _ZCOMP_H_ */ diff --git a/drivers/block/zram/zcomp_lz4.c b/drivers/block/zram/zcomp_lz4.c deleted file mode 100644 index 0110086accba..000000000000 --- a/drivers/block/zram/zcomp_lz4.c +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2014 Sergey Senozhatsky. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include -#include -#include -#include -#include - -#include "zcomp_lz4.h" - -static void *zcomp_lz4_create(gfp_t flags) -{ - void *ret; - - ret = kmalloc(LZ4_MEM_COMPRESS, flags); - if (!ret) - ret = __vmalloc(LZ4_MEM_COMPRESS, - flags | __GFP_HIGHMEM, - PAGE_KERNEL); - return ret; -} - -static void zcomp_lz4_destroy(void *private) -{ - kvfree(private); -} - -static int zcomp_lz4_compress(const unsigned char *src, unsigned char *dst, - size_t *dst_len, void *private) -{ - /* return : Success if return 0 */ - return lz4_compress(src, PAGE_SIZE, dst, dst_len, private); -} - -static int zcomp_lz4_decompress(const unsigned char *src, size_t src_len, - unsigned char *dst) -{ - size_t dst_len = PAGE_SIZE; - /* return : Success if return 0 */ - return lz4_decompress_unknownoutputsize(src, src_len, dst, &dst_len); -} - -struct zcomp_backend zcomp_lz4 = { - .compress = zcomp_lz4_compress, - .decompress = zcomp_lz4_decompress, - .create = zcomp_lz4_create, - .destroy = zcomp_lz4_destroy, - .name = "lz4", -}; diff --git a/drivers/block/zram/zcomp_lz4.h b/drivers/block/zram/zcomp_lz4.h deleted file mode 100644 index 60613fb29dd8..000000000000 --- a/drivers/block/zram/zcomp_lz4.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (C) 2014 Sergey Senozhatsky. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#ifndef _ZCOMP_LZ4_H_ -#define _ZCOMP_LZ4_H_ - -#include "zcomp.h" - -extern struct zcomp_backend zcomp_lz4; - -#endif /* _ZCOMP_LZ4_H_ */ diff --git a/drivers/block/zram/zcomp_lzo.c b/drivers/block/zram/zcomp_lzo.c deleted file mode 100644 index ed7a1f0549ec..000000000000 --- a/drivers/block/zram/zcomp_lzo.c +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2014 Sergey Senozhatsky. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include -#include -#include -#include -#include - -#include "zcomp_lzo.h" - -static void *lzo_create(gfp_t flags) -{ - void *ret; - - ret = kmalloc(LZO1X_MEM_COMPRESS, flags); - if (!ret) - ret = __vmalloc(LZO1X_MEM_COMPRESS, - flags | __GFP_HIGHMEM, - PAGE_KERNEL); - return ret; -} - -static void lzo_destroy(void *private) -{ - kvfree(private); -} - -static int lzo_compress(const unsigned char *src, unsigned char *dst, - size_t *dst_len, void *private) -{ - int ret = lzo1x_1_compress(src, PAGE_SIZE, dst, dst_len, private); - return ret == LZO_E_OK ? 0 : ret; -} - -static int lzo_decompress(const unsigned char *src, size_t src_len, - unsigned char *dst) -{ - size_t dst_len = PAGE_SIZE; - int ret = lzo1x_decompress_safe(src, src_len, dst, &dst_len); - return ret == LZO_E_OK ? 0 : ret; -} - -struct zcomp_backend zcomp_lzo = { - .compress = lzo_compress, - .decompress = lzo_decompress, - .create = lzo_create, - .destroy = lzo_destroy, - .name = "lzo", -}; diff --git a/drivers/block/zram/zcomp_lzo.h b/drivers/block/zram/zcomp_lzo.h deleted file mode 100644 index 128c5807fa14..000000000000 --- a/drivers/block/zram/zcomp_lzo.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (C) 2014 Sergey Senozhatsky. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#ifndef _ZCOMP_LZO_H_ -#define _ZCOMP_LZO_H_ - -#include "zcomp.h" - -extern struct zcomp_backend zcomp_lzo; - -#endif /* _ZCOMP_LZO_H_ */ diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 8fcad8b761f1..7454cf188c8e 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -342,9 +342,16 @@ static ssize_t comp_algorithm_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); + char compressor[CRYPTO_MAX_ALG_NAME]; size_t sz; - if (!zcomp_available_algorithm(buf)) + strlcpy(compressor, buf, sizeof(compressor)); + /* ignore trailing newline */ + sz = strlen(compressor); + if (sz > 0 && compressor[sz - 1] == '\n') + compressor[sz - 1] = 0x00; + + if (!zcomp_available_algorithm(compressor)) return -EINVAL; down_write(&zram->init_lock); @@ -353,13 +360,8 @@ static ssize_t comp_algorithm_store(struct device *dev, pr_info("Can't change algorithm for initialized device\n"); return -EBUSY; } - strlcpy(zram->compressor, buf, sizeof(zram->compressor)); - - /* ignore trailing newline */ - sz = strlen(zram->compressor); - if (sz > 0 && zram->compressor[sz - 1] == '\n') - zram->compressor[sz - 1] = 0x00; + strlcpy(zram->compressor, compressor, sizeof(compressor)); up_write(&zram->init_lock); return len; } @@ -563,7 +565,7 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index) unsigned char *cmem; struct zram_meta *meta = zram->meta; unsigned long handle; - size_t size; + unsigned int size; bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value); handle = meta->table[index].handle; @@ -576,10 +578,14 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index) } cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO); - if (size == PAGE_SIZE) + if (size == PAGE_SIZE) { copy_page(mem, cmem); - else - ret = zcomp_decompress(zram->comp, cmem, size, mem); + } else { + struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp); + + ret = zcomp_decompress(zstrm, cmem, size, mem); + zcomp_stream_put(zram->comp); + } zs_unmap_object(meta->mem_pool, handle); bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); @@ -646,7 +652,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, int offset) { int ret = 0; - size_t clen; + unsigned int clen; unsigned long handle = 0; struct page *page; unsigned char *user_mem, *cmem, *src, *uncmem = NULL; @@ -695,8 +701,8 @@ compress_again: goto out; } - zstrm = zcomp_strm_find(zram->comp); - ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen); + zstrm = zcomp_stream_get(zram->comp); + ret = zcomp_compress(zstrm, uncmem, &clen); if (!is_partial_io(bvec)) { kunmap_atomic(user_mem); user_mem = NULL; @@ -732,19 +738,21 @@ compress_again: handle = zs_malloc(meta->mem_pool, clen, __GFP_KSWAPD_RECLAIM | __GFP_NOWARN | - __GFP_HIGHMEM); + __GFP_HIGHMEM | + __GFP_MOVABLE); if (!handle) { - zcomp_strm_release(zram->comp, zstrm); + zcomp_stream_put(zram->comp); zstrm = NULL; atomic64_inc(&zram->stats.writestall); handle = zs_malloc(meta->mem_pool, clen, - GFP_NOIO | __GFP_HIGHMEM); + GFP_NOIO | __GFP_HIGHMEM | + __GFP_MOVABLE); if (handle) goto compress_again; - pr_err("Error allocating memory for compressed page: %u, size=%zu\n", + pr_err("Error allocating memory for compressed page: %u, size=%u\n", index, clen); ret = -ENOMEM; goto out; @@ -769,7 +777,7 @@ compress_again: memcpy(cmem, src, clen); } - zcomp_strm_release(zram->comp, zstrm); + zcomp_stream_put(zram->comp); zstrm = NULL; zs_unmap_object(meta->mem_pool, handle); @@ -789,7 +797,7 @@ compress_again: atomic64_inc(&zram->stats.pages_stored); out: if (zstrm) - zcomp_strm_release(zram->comp, zstrm); + zcomp_stream_put(zram->comp); if (is_partial_io(bvec)) kfree(uncmem); return ret; @@ -874,7 +882,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) offset = (bio->bi_iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; - if (unlikely(bio->bi_rw & REQ_DISCARD)) { + if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { zram_bio_discard(zram, index, offset, bio); bio_endio(bio); return; diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 3f5bf66a27e4..74fcf10da374 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -15,8 +15,9 @@ #ifndef _ZRAM_DRV_H_ #define _ZRAM_DRV_H_ -#include +#include #include +#include #include "zcomp.h" @@ -113,7 +114,7 @@ struct zram { * we can store in a disk. */ u64 disksize; /* bytes */ - char compressor[10]; + char compressor[CRYPTO_MAX_ALG_NAME]; /* * zram is claimed so open request will be failed */ diff --git a/drivers/bluetooth/ath3k.c b/drivers/bluetooth/ath3k.c index 25894687c168..fadba88745dc 100644 --- a/drivers/bluetooth/ath3k.c +++ b/drivers/bluetooth/ath3k.c @@ -123,6 +123,7 @@ static const struct usb_device_id ath3k_table[] = { { USB_DEVICE(0x13d3, 0x3472) }, { USB_DEVICE(0x13d3, 0x3474) }, { USB_DEVICE(0x13d3, 0x3487) }, + { USB_DEVICE(0x13d3, 0x3490) }, /* Atheros AR5BBU12 with sflash firmware */ { USB_DEVICE(0x0489, 0xE02C) }, @@ -190,6 +191,7 @@ static const struct usb_device_id ath3k_blist_tbl[] = { { USB_DEVICE(0x13d3, 0x3472), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3474), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3487), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x13d3, 0x3490), .driver_info = BTUSB_ATH3012 }, /* Atheros AR5BBU22 with sflash firmware */ { USB_DEVICE(0x0489, 0xE036), .driver_info = BTUSB_ATH3012 }, diff --git a/drivers/bluetooth/bpa10x.c b/drivers/bluetooth/bpa10x.c index fd6b53e9bbf2..a9932fe57d92 100644 --- a/drivers/bluetooth/bpa10x.c +++ b/drivers/bluetooth/bpa10x.c @@ -274,6 +274,8 @@ static int bpa10x_setup(struct hci_dev *hdev) BT_INFO("%s: %s", hdev->name, (char *)(skb->data + 1)); + hci_set_fw_info(hdev, "%s", skb->data + 1); + kfree_skb(skb); return 0; } diff --git a/drivers/bluetooth/btmrvl_main.c b/drivers/bluetooth/btmrvl_main.c index 7ad8d61c0c61..e6a85f0e6309 100644 --- a/drivers/bluetooth/btmrvl_main.c +++ b/drivers/bluetooth/btmrvl_main.c @@ -138,7 +138,7 @@ int btmrvl_process_event(struct btmrvl_private *priv, struct sk_buff *skb) if (event->length > 3 && event->data[3]) priv->btmrvl_dev.dev_type = HCI_AMP; else - priv->btmrvl_dev.dev_type = HCI_BREDR; + priv->btmrvl_dev.dev_type = HCI_PRIMARY; BT_DBG("dev_type: %d", priv->btmrvl_dev.dev_type); } else if (priv->btmrvl_dev.sendcmdflag && diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c index f425ddf91a24..d02f2c14df32 100644 --- a/drivers/bluetooth/btmrvl_sdio.c +++ b/drivers/bluetooth/btmrvl_sdio.c @@ -1071,7 +1071,6 @@ static int btmrvl_sdio_host_to_card(struct btmrvl_private *priv, { struct btmrvl_sdio_card *card = priv->btmrvl_dev.card; int ret = 0; - int buf_block_len; int blksz; int i = 0; u8 *buf = NULL; @@ -1083,9 +1082,13 @@ static int btmrvl_sdio_host_to_card(struct btmrvl_private *priv, return -EINVAL; } + blksz = DIV_ROUND_UP(nb, SDIO_BLOCK_SIZE) * SDIO_BLOCK_SIZE; + buf = payload; - if ((unsigned long) payload & (BTSDIO_DMA_ALIGN - 1)) { - tmpbufsz = ALIGN_SZ(nb, BTSDIO_DMA_ALIGN); + if ((unsigned long) payload & (BTSDIO_DMA_ALIGN - 1) || + nb < blksz) { + tmpbufsz = ALIGN_SZ(blksz, BTSDIO_DMA_ALIGN) + + BTSDIO_DMA_ALIGN; tmpbuf = kzalloc(tmpbufsz, GFP_KERNEL); if (!tmpbuf) return -ENOMEM; @@ -1093,15 +1096,12 @@ static int btmrvl_sdio_host_to_card(struct btmrvl_private *priv, memcpy(buf, payload, nb); } - blksz = SDIO_BLOCK_SIZE; - buf_block_len = DIV_ROUND_UP(nb, blksz); - sdio_claim_host(card->func); do { /* Transfer data to card */ ret = sdio_writesb(card->func, card->ioport, buf, - buf_block_len * blksz); + blksz); if (ret < 0) { i++; BT_ERR("i=%d writesb failed: %d", i, ret); @@ -1625,6 +1625,7 @@ static int btmrvl_sdio_suspend(struct device *dev) if (priv->adapter->hs_state != HS_ACTIVATED) { if (btmrvl_enable_hs(priv)) { BT_ERR("HS not actived, suspend failed!"); + priv->adapter->is_suspending = false; return -EBUSY; } } diff --git a/drivers/bluetooth/btsdio.c b/drivers/bluetooth/btsdio.c index 2b05661e3818..1cb958e199eb 100644 --- a/drivers/bluetooth/btsdio.c +++ b/drivers/bluetooth/btsdio.c @@ -311,7 +311,7 @@ static int btsdio_probe(struct sdio_func *func, if (id->class == SDIO_CLASS_BT_AMP) hdev->dev_type = HCI_AMP; else - hdev->dev_type = HCI_BREDR; + hdev->dev_type = HCI_PRIMARY; data->hdev = hdev; diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index a3be65e6231a..811f9b97e360 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -237,6 +237,7 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x13d3, 0x3472), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3474), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3487), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x13d3, 0x3490), .driver_info = BTUSB_ATH3012 }, /* Atheros AR5BBU12 with sflash firmware */ { USB_DEVICE(0x0489, 0xe02c), .driver_info = BTUSB_IGNORE }, @@ -249,6 +250,7 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x0cf3, 0xe007), .driver_info = BTUSB_QCA_ROME }, { USB_DEVICE(0x0cf3, 0xe300), .driver_info = BTUSB_QCA_ROME }, { USB_DEVICE(0x0cf3, 0xe360), .driver_info = BTUSB_QCA_ROME }, + { USB_DEVICE(0x0489, 0xe092), .driver_info = BTUSB_QCA_ROME }, /* Broadcom BCM2035 */ { USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 }, @@ -314,6 +316,7 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x8087, 0x07dc), .driver_info = BTUSB_INTEL }, { USB_DEVICE(0x8087, 0x0a2a), .driver_info = BTUSB_INTEL }, { USB_DEVICE(0x8087, 0x0a2b), .driver_info = BTUSB_INTEL_NEW }, + { USB_DEVICE(0x8087, 0x0aa7), .driver_info = BTUSB_INTEL }, /* Other Intel Bluetooth devices */ { USB_VENDOR_AND_INTERFACE_INFO(0x8087, 0xe0, 0x01, 0x01), @@ -2103,10 +2106,14 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) /* With this Intel bootloader only the hardware variant and device * revision information are used to select the right firmware. * - * Currently this bootloader support is limited to hardware variant - * iBT 3.0 (LnP/SfP) which is identified by the value 11 (0x0b). + * The firmware filename is ibt--.sfi. + * + * Currently the supported hardware variants are: + * 11 (0x0b) for iBT3.0 (LnP/SfP) + * 12 (0x0c) for iBT3.5 (WsP) */ - snprintf(fwname, sizeof(fwname), "intel/ibt-11-%u.sfi", + snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.sfi", + le16_to_cpu(ver.hw_variant), le16_to_cpu(params->dev_revid)); err = request_firmware(&fw, fwname, &hdev->dev); @@ -2122,7 +2129,8 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) /* Save the DDC file name for later use to apply once the firmware * downloading is done. */ - snprintf(fwname, sizeof(fwname), "intel/ibt-11-%u.ddc", + snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.ddc", + le16_to_cpu(ver.hw_variant), le16_to_cpu(params->dev_revid)); kfree_skb(skb); @@ -2825,7 +2833,7 @@ static int btusb_probe(struct usb_interface *intf, if (id->driver_info & BTUSB_AMP) hdev->dev_type = HCI_AMP; else - hdev->dev_type = HCI_BREDR; + hdev->dev_type = HCI_PRIMARY; data->hdev = hdev; diff --git a/drivers/bluetooth/btwilink.c b/drivers/bluetooth/btwilink.c index 24a652f9252b..485281b3f167 100644 --- a/drivers/bluetooth/btwilink.c +++ b/drivers/bluetooth/btwilink.c @@ -51,7 +51,7 @@ */ struct ti_st { struct hci_dev *hdev; - char reg_status; + int reg_status; long (*st_write) (struct sk_buff *); struct completion wait_reg_completion; }; @@ -83,7 +83,7 @@ static inline void ti_st_tx_complete(struct ti_st *hst, int pkt_type) * status.ti_st_open() function will wait for signal from this * API when st_register() function returns ST_PENDING. */ -static void st_reg_completion_cb(void *priv_data, char data) +static void st_reg_completion_cb(void *priv_data, int data) { struct ti_st *lhst = priv_data; diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c index f6f2b01a1fea..ed0a4201b551 100644 --- a/drivers/bluetooth/hci_intel.c +++ b/drivers/bluetooth/hci_intel.c @@ -537,9 +537,7 @@ static int intel_setup(struct hci_uart *hu) { static const u8 reset_param[] = { 0x00, 0x01, 0x00, 0x01, 0x00, 0x08, 0x04, 0x00 }; - static const u8 lpm_param[] = { 0x03, 0x07, 0x01, 0x0b }; struct intel_data *intel = hu->priv; - struct intel_device *idev = NULL; struct hci_dev *hdev = hu->hdev; struct sk_buff *skb; struct intel_version ver; @@ -884,35 +882,23 @@ done: bt_dev_info(hdev, "Device booted in %llu usecs", duration); - /* Enable LPM if matching pdev with wakeup enabled */ + /* Enable LPM if matching pdev with wakeup enabled, set TX active + * until further LPM TX notification. + */ mutex_lock(&intel_device_list_lock); list_for_each(p, &intel_device_list) { struct intel_device *dev = list_entry(p, struct intel_device, list); if (hu->tty->dev->parent == dev->pdev->dev.parent) { - if (device_may_wakeup(&dev->pdev->dev)) - idev = dev; + if (device_may_wakeup(&dev->pdev->dev)) { + set_bit(STATE_LPM_ENABLED, &intel->flags); + set_bit(STATE_TX_ACTIVE, &intel->flags); + } break; } } mutex_unlock(&intel_device_list_lock); - if (!idev) - goto no_lpm; - - bt_dev_info(hdev, "Enabling LPM"); - - skb = __hci_cmd_sync(hdev, 0xfc8b, sizeof(lpm_param), lpm_param, - HCI_CMD_TIMEOUT); - if (IS_ERR(skb)) { - bt_dev_err(hdev, "Failed to enable LPM"); - goto no_lpm; - } - kfree_skb(skb); - - set_bit(STATE_LPM_ENABLED, &intel->flags); - -no_lpm: /* Ignore errors, device can work without DDC parameters */ btintel_load_ddc_config(hdev, fwname); diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index 49b3e1e2d236..dda97398c59a 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -609,7 +609,7 @@ static int hci_uart_register_dev(struct hci_uart *hu) if (test_bit(HCI_UART_CREATE_AMP, &hu->hdev_flags)) hdev->dev_type = HCI_AMP; else - hdev->dev_type = HCI_BREDR; + hdev->dev_type = HCI_PRIMARY; if (test_bit(HCI_UART_INIT_PENDING, &hu->hdev_flags)) return 0; diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c index aba31210c802..3ff229b2e7f3 100644 --- a/drivers/bluetooth/hci_vhci.c +++ b/drivers/bluetooth/hci_vhci.c @@ -97,10 +97,10 @@ static int __vhci_create_device(struct vhci_data *data, __u8 opcode) if (data->hdev) return -EBADFD; - /* bits 0-1 are dev_type (BR/EDR or AMP) */ + /* bits 0-1 are dev_type (Primary or AMP) */ dev_type = opcode & 0x03; - if (dev_type != HCI_BREDR && dev_type != HCI_AMP) + if (dev_type != HCI_PRIMARY && dev_type != HCI_AMP) return -EINVAL; /* bits 2-5 are reserved (must be zero) */ @@ -316,7 +316,7 @@ static void vhci_open_timeout(struct work_struct *work) struct vhci_data *data = container_of(work, struct vhci_data, open_timeout.work); - vhci_create_device(data, amp ? HCI_AMP : HCI_BREDR); + vhci_create_device(data, amp ? HCI_AMP : HCI_PRIMARY); } static int vhci_open(struct inode *inode, struct file *file) diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index 1b257ea9776a..5d475b3a0b2e 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -2032,7 +2032,7 @@ static int cdrom_read_subchannel(struct cdrom_device_info *cdi, init_cdrom_command(&cgc, buffer, 16, CGC_DATA_READ); cgc.cmd[0] = GPCMD_READ_SUBCHANNEL; - cgc.cmd[1] = 2; /* MSF addressing */ + cgc.cmd[1] = subchnl->cdsc_format;/* MSF or LBA addressing */ cgc.cmd[2] = 0x40; /* request subQ data */ cgc.cmd[3] = mcn ? 2 : 1; cgc.cmd[8] = 16; @@ -2041,17 +2041,27 @@ static int cdrom_read_subchannel(struct cdrom_device_info *cdi, return ret; subchnl->cdsc_audiostatus = cgc.buffer[1]; - subchnl->cdsc_format = CDROM_MSF; subchnl->cdsc_ctrl = cgc.buffer[5] & 0xf; subchnl->cdsc_trk = cgc.buffer[6]; subchnl->cdsc_ind = cgc.buffer[7]; - subchnl->cdsc_reladdr.msf.minute = cgc.buffer[13]; - subchnl->cdsc_reladdr.msf.second = cgc.buffer[14]; - subchnl->cdsc_reladdr.msf.frame = cgc.buffer[15]; - subchnl->cdsc_absaddr.msf.minute = cgc.buffer[9]; - subchnl->cdsc_absaddr.msf.second = cgc.buffer[10]; - subchnl->cdsc_absaddr.msf.frame = cgc.buffer[11]; + if (subchnl->cdsc_format == CDROM_LBA) { + subchnl->cdsc_absaddr.lba = ((cgc.buffer[8] << 24) | + (cgc.buffer[9] << 16) | + (cgc.buffer[10] << 8) | + (cgc.buffer[11])); + subchnl->cdsc_reladdr.lba = ((cgc.buffer[12] << 24) | + (cgc.buffer[13] << 16) | + (cgc.buffer[14] << 8) | + (cgc.buffer[15])); + } else { + subchnl->cdsc_reladdr.msf.minute = cgc.buffer[13]; + subchnl->cdsc_reladdr.msf.second = cgc.buffer[14]; + subchnl->cdsc_reladdr.msf.frame = cgc.buffer[15]; + subchnl->cdsc_absaddr.msf.minute = cgc.buffer[9]; + subchnl->cdsc_absaddr.msf.second = cgc.buffer[10]; + subchnl->cdsc_absaddr.msf.frame = cgc.buffer[11]; + } return 0; } @@ -3022,7 +3032,7 @@ static noinline int mmc_ioctl_cdrom_subchannel(struct cdrom_device_info *cdi, if (!((requested == CDROM_MSF) || (requested == CDROM_LBA))) return -EINVAL; - q.cdsc_format = CDROM_MSF; + ret = cdrom_read_subchannel(cdi, &q, 0); if (ret) return ret; diff --git a/drivers/char/dsp56k.c b/drivers/char/dsp56k.c index 8bf70e8c3f79..50aa9ba91f25 100644 --- a/drivers/char/dsp56k.c +++ b/drivers/char/dsp56k.c @@ -325,7 +325,7 @@ static long dsp56k_ioctl(struct file *file, unsigned int cmd, if(get_user(bin, &binary->bin) < 0) return -EFAULT; - if (len == 0) { + if (len <= 0) { return -EINVAL; /* nothing to upload?!? */ } if (len > DSP56K_MAX_BINARY_LENGTH) { diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index ac51149e9777..56ad5a5936a9 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -90,7 +90,7 @@ config HW_RANDOM_BCM63XX config HW_RANDOM_BCM2835 tristate "Broadcom BCM2835 Random Number Generator support" - depends on ARCH_BCM2835 + depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X default HW_RANDOM ---help--- This driver provides kernel-side support for the Random Number @@ -396,6 +396,20 @@ config HW_RANDOM_PIC32 If unsure, say Y. +config HW_RANDOM_MESON + tristate "Amlogic Meson Random Number Generator support" + depends on HW_RANDOM + depends on ARCH_MESON || COMPILE_TEST + default y + ---help--- + This driver provides kernel-side support for the Random Number + Generator hardware found on Amlogic Meson SoCs. + + To compile this driver as a module, choose M here. the + module will be called meson-rng. + + If unsure, say Y. + endif # HW_RANDOM config UML_RANDOM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 63022b49f160..04bb0b03356f 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -34,3 +34,4 @@ obj-$(CONFIG_HW_RANDOM_ST) += st-rng.o obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o obj-$(CONFIG_HW_RANDOM_STM32) += stm32-rng.o obj-$(CONFIG_HW_RANDOM_PIC32) += pic32-rng.o +obj-$(CONFIG_HW_RANDOM_MESON) += meson-rng.o diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 7192ec25f667..af2149273fe0 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -19,6 +19,7 @@ #define RNG_CTRL 0x0 #define RNG_STATUS 0x4 #define RNG_DATA 0x8 +#define RNG_INT_MASK 0x10 /* enable rng */ #define RNG_RBGEN 0x1 @@ -26,10 +27,24 @@ /* the initial numbers generated are "less random" so will be discarded */ #define RNG_WARMUP_COUNT 0x40000 +#define RNG_INT_OFF 0x1 + +static void __init nsp_rng_init(void __iomem *base) +{ + u32 val; + + /* mask the interrupt */ + val = readl(base + RNG_INT_MASK); + val |= RNG_INT_OFF; + writel(val, base + RNG_INT_MASK); +} + static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) { void __iomem *rng_base = (void __iomem *)rng->priv; + u32 max_words = max / sizeof(u32); + u32 num_words, count; while ((__raw_readl(rng_base + RNG_STATUS) >> 24) == 0) { if (!wait) @@ -37,8 +52,14 @@ static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, cpu_relax(); } - *(u32 *)buf = __raw_readl(rng_base + RNG_DATA); - return sizeof(u32); + num_words = readl(rng_base + RNG_STATUS) >> 24; + if (num_words > max_words) + num_words = max_words; + + for (count = 0; count < num_words; count++) + ((u32 *)buf)[count] = readl(rng_base + RNG_DATA); + + return num_words * sizeof(u32); } static struct hwrng bcm2835_rng_ops = { @@ -46,10 +67,19 @@ static struct hwrng bcm2835_rng_ops = { .read = bcm2835_rng_read, }; +static const struct of_device_id bcm2835_rng_of_match[] = { + { .compatible = "brcm,bcm2835-rng"}, + { .compatible = "brcm,bcm-nsp-rng", .data = nsp_rng_init}, + { .compatible = "brcm,bcm5301x-rng", .data = nsp_rng_init}, + {}, +}; + static int bcm2835_rng_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; + void (*rng_setup)(void __iomem *base); + const struct of_device_id *rng_id; void __iomem *rng_base; int err; @@ -61,6 +91,15 @@ static int bcm2835_rng_probe(struct platform_device *pdev) } bcm2835_rng_ops.priv = (unsigned long)rng_base; + rng_id = of_match_node(bcm2835_rng_of_match, np); + if (!rng_id) + return -EINVAL; + + /* Check for rng init function, execute it */ + rng_setup = rng_id->data; + if (rng_setup) + rng_setup(rng_base); + /* set warm-up count & enable */ __raw_writel(RNG_WARMUP_COUNT, rng_base + RNG_STATUS); __raw_writel(RNG_RBGEN, rng_base + RNG_CTRL); @@ -90,10 +129,6 @@ static int bcm2835_rng_remove(struct platform_device *pdev) return 0; } -static const struct of_device_id bcm2835_rng_of_match[] = { - { .compatible = "brcm,bcm2835-rng", }, - {}, -}; MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match); static struct platform_driver bcm2835_rng_driver = { diff --git a/drivers/char/hw_random/exynos-rng.c b/drivers/char/hw_random/exynos-rng.c index ed44561ea647..23d358553b21 100644 --- a/drivers/char/hw_random/exynos-rng.c +++ b/drivers/char/hw_random/exynos-rng.c @@ -45,12 +45,12 @@ struct exynos_rng { static u32 exynos_rng_readl(struct exynos_rng *rng, u32 offset) { - return __raw_readl(rng->mem + offset); + return readl_relaxed(rng->mem + offset); } static void exynos_rng_writel(struct exynos_rng *rng, u32 val, u32 offset) { - __raw_writel(val, rng->mem + offset); + writel_relaxed(val, rng->mem + offset); } static int exynos_rng_configure(struct exynos_rng *exynos_rng) diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c new file mode 100644 index 000000000000..0cfd81bcaeac --- /dev/null +++ b/drivers/char/hw_random/meson-rng.c @@ -0,0 +1,131 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (c) 2016 BayLibre, SAS. + * Author: Neil Armstrong + * Copyright (C) 2014 Amlogic, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * The full GNU General Public License is included in this distribution + * in the file called COPYING. + * + * BSD LICENSE + * + * Copyright (c) 2016 BayLibre, SAS. + * Author: Neil Armstrong + * Copyright (C) 2014 Amlogic, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define RNG_DATA 0x00 + +struct meson_rng_data { + void __iomem *base; + struct platform_device *pdev; + struct hwrng rng; +}; + +static int meson_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct meson_rng_data *data = + container_of(rng, struct meson_rng_data, rng); + + if (max < sizeof(u32)) + return 0; + + *(u32 *)buf = readl_relaxed(data->base + RNG_DATA); + + return sizeof(u32); +} + +static int meson_rng_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct meson_rng_data *data; + struct resource *res; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->pdev = pdev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(data->base)) + return PTR_ERR(data->base); + + data->rng.name = pdev->name; + data->rng.read = meson_rng_read; + + platform_set_drvdata(pdev, data); + + return devm_hwrng_register(dev, &data->rng); +} + +static const struct of_device_id meson_rng_of_match[] = { + { .compatible = "amlogic,meson-rng", }, + {}, +}; + +static struct platform_driver meson_rng_driver = { + .probe = meson_rng_probe, + .driver = { + .name = "meson-rng", + .of_match_table = meson_rng_of_match, + }, +}; + +module_platform_driver(meson_rng_driver); + +MODULE_ALIAS("platform:meson-rng"); +MODULE_DESCRIPTION("Meson H/W Random Number Generator driver"); +MODULE_AUTHOR("Lawrence Mok "); +MODULE_AUTHOR("Neil Armstrong "); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index 8a1432e8bb80..01d4be2c354b 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -384,7 +384,12 @@ static int omap_rng_probe(struct platform_device *pdev) } pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); + if (ret) { + dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret); + pm_runtime_put_noidle(&pdev->dev); + goto err_ioremap; + } ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) : get_omap_rng_device_details(priv); @@ -435,8 +440,15 @@ static int __maybe_unused omap_rng_suspend(struct device *dev) static int __maybe_unused omap_rng_resume(struct device *dev) { struct omap_rng_dev *priv = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_get_sync(dev); + if (ret) { + dev_err(dev, "Failed to runtime_get device: %d\n", ret); + pm_runtime_put_noidle(dev); + return ret; + } - pm_runtime_get_sync(dev); priv->pdata->init(priv); return 0; diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 92a810648bd0..63d84e6f1891 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -69,8 +69,12 @@ static int stm32_rng_read(struct hwrng *rng, void *data, size_t max, bool wait) } /* If error detected or data not ready... */ - if (sr != RNG_SR_DRDY) + if (sr != RNG_SR_DRDY) { + if (WARN_ONCE(sr & (RNG_SR_SEIS | RNG_SR_CEIS), + "bad RNG status - %x\n", sr)) + writel_relaxed(0, priv->base + RNG_SR); break; + } *(u32 *)data = readl_relaxed(priv->base + RNG_DR); @@ -79,10 +83,6 @@ static int stm32_rng_read(struct hwrng *rng, void *data, size_t max, bool wait) max -= sizeof(u32); } - if (WARN_ONCE(sr & (RNG_SR_SEIS | RNG_SR_CEIS), - "bad RNG status - %x\n", sr)) - writel_relaxed(0, priv->base + RNG_SR); - pm_runtime_mark_last_busy((struct device *) priv->rng.priv); pm_runtime_put_sync_autosuspend((struct device *) priv->rng.priv); diff --git a/drivers/char/ipmi/Kconfig b/drivers/char/ipmi/Kconfig index 6ed9e9fe5233..5a9350b1069a 100644 --- a/drivers/char/ipmi/Kconfig +++ b/drivers/char/ipmi/Kconfig @@ -50,18 +50,6 @@ config IPMI_SI Currently, only KCS and SMIC are supported. If you are using IPMI, you should probably say "y" here. -config IPMI_SI_PROBE_DEFAULTS - bool 'Probe for all possible IPMI system interfaces by default' - default n - depends on IPMI_SI - help - Modern systems will usually expose IPMI interfaces via a discoverable - firmware mechanism such as ACPI or DMI. Older systems do not, and so - the driver is forced to probe hardware manually. This may cause boot - delays. Say "n" here to disable this manual probing. IPMI will then - only be available on older systems if the "ipmi_si_intf.trydefaults=1" - boot argument is passed. - config IPMI_SSIF tristate 'IPMI SMBus handler (SSIF)' select I2C diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 44b1bd6baa38..d8619998cfb5 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -474,12 +474,12 @@ static DEFINE_MUTEX(smi_watchers_mutex); static const char * const addr_src_to_str[] = { "invalid", "hotmod", "hardcoded", "SPMI", "ACPI", "SMBIOS", "PCI", - "device-tree", "default" + "device-tree" }; const char *ipmi_addr_src_to_str(enum ipmi_addr_src src) { - if (src > SI_DEFAULT) + if (src >= SI_LAST) src = 0; /* Invalid */ return addr_src_to_str[src]; } diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 7b1c412b40a2..a112c0146012 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -1322,7 +1322,6 @@ static bool si_tryplatform = true; #ifdef CONFIG_PCI static bool si_trypci = true; #endif -static bool si_trydefaults = IS_ENABLED(CONFIG_IPMI_SI_PROBE_DEFAULTS); static char *si_type[SI_MAX_PARMS]; #define MAX_SI_TYPE_STR 30 static char si_type_str[MAX_SI_TYPE_STR]; @@ -1371,10 +1370,6 @@ module_param_named(trypci, si_trypci, bool, 0); MODULE_PARM_DESC(trypci, "Setting this to zero will disable the" " default scan of the interfaces identified via pci"); #endif -module_param_named(trydefaults, si_trydefaults, bool, 0); -MODULE_PARM_DESC(trydefaults, "Setting this to 'false' will disable the" - " default scan of the KCS and SMIC interface at the standard" - " address"); module_param_string(type, si_type_str, MAX_SI_TYPE_STR, 0); MODULE_PARM_DESC(type, "Defines the type of each interface, each" " interface separated by commas. The types are 'kcs'," @@ -3461,62 +3456,6 @@ static inline void wait_for_timer_and_thread(struct smi_info *smi_info) del_timer_sync(&smi_info->si_timer); } -static const struct ipmi_default_vals -{ - const int type; - const int port; -} ipmi_defaults[] = -{ - { .type = SI_KCS, .port = 0xca2 }, - { .type = SI_SMIC, .port = 0xca9 }, - { .type = SI_BT, .port = 0xe4 }, - { .port = 0 } -}; - -static void default_find_bmc(void) -{ - struct smi_info *info; - int i; - - for (i = 0; ; i++) { - if (!ipmi_defaults[i].port) - break; -#ifdef CONFIG_PPC - if (check_legacy_ioport(ipmi_defaults[i].port)) - continue; -#endif - info = smi_info_alloc(); - if (!info) - return; - - info->addr_source = SI_DEFAULT; - - info->si_type = ipmi_defaults[i].type; - info->io_setup = port_setup; - info->io.addr_data = ipmi_defaults[i].port; - info->io.addr_type = IPMI_IO_ADDR_SPACE; - - info->io.addr = NULL; - info->io.regspacing = DEFAULT_REGSPACING; - info->io.regsize = DEFAULT_REGSPACING; - info->io.regshift = 0; - - if (add_smi(info) == 0) { - if ((try_smi_init(info)) == 0) { - /* Found one... */ - printk(KERN_INFO PFX "Found default %s" - " state machine at %s address 0x%lx\n", - si_to_str[info->si_type], - addr_space_to_str[info->io.addr_type], - info->io.addr_data); - } else - cleanup_one_si(info); - } else { - kfree(info); - } - } -} - static int is_new_interface(struct smi_info *info) { struct smi_info *e; @@ -3844,8 +3783,6 @@ static int init_ipmi_si(void) #ifdef CONFIG_PARISC register_parisc_driver(&ipmi_parisc_driver); parisc_registered = true; - /* poking PC IO addresses will crash machine, don't do it */ - si_trydefaults = 0; #endif /* We prefer devices with interrupts, but in the case of a machine @@ -3885,16 +3822,6 @@ static int init_ipmi_si(void) if (type) return 0; - if (si_trydefaults) { - mutex_lock(&smi_infos_lock); - if (list_empty(&smi_infos)) { - /* No BMC was found, try defaults. */ - mutex_unlock(&smi_infos_lock); - default_find_bmc(); - } else - mutex_unlock(&smi_infos_lock); - } - mutex_lock(&smi_infos_lock); if (unload_when_empty && list_empty(&smi_infos)) { mutex_unlock(&smi_infos_lock); diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 71025c2f6bbb..a33163dbb913 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -66,12 +67,8 @@ static inline int range_is_allowed(unsigned long pfn, unsigned long size) u64 cursor = from; while (cursor < to) { - if (!devmem_is_allowed(pfn)) { - printk(KERN_INFO - "Program %s tried to access /dev/mem between %Lx->%Lx.\n", - current->comm, from, to); + if (!devmem_is_allowed(pfn)) return 0; - } cursor += PAGE_SIZE; pfn++; } @@ -661,6 +658,28 @@ static int mmap_zero(struct file *file, struct vm_area_struct *vma) return 0; } +static unsigned long get_unmapped_area_zero(struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ +#ifdef CONFIG_MMU + if (flags & MAP_SHARED) { + /* + * mmap_zero() will call shmem_zero_setup() to create a file, + * so use shmem's get_unmapped_area in case it can be huge; + * and pass NULL for file as in mmap.c's get_unmapped_area(), + * so as not to confuse shmem with our handle on "/dev/zero". + */ + return shmem_get_unmapped_area(NULL, addr, len, pgoff, flags); + } + + /* Otherwise flags & MAP_PRIVATE: with no shmem object beneath it */ + return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); +#else + return -ENOSYS; +#endif +} + static ssize_t write_full(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { @@ -768,6 +787,7 @@ static const struct file_operations zero_fops = { .read_iter = read_iter_zero, .write_iter = write_iter_zero, .mmap = mmap_zero, + .get_unmapped_area = get_unmapped_area_zero, #ifndef CONFIG_MMU .mmap_capabilities = zero_mmap_capabilities, #endif diff --git a/drivers/clk/at91/clk-programmable.c b/drivers/clk/at91/clk-programmable.c index 10f846cc8db1..25d5906640c3 100644 --- a/drivers/clk/at91/clk-programmable.c +++ b/drivers/clk/at91/clk-programmable.c @@ -99,7 +99,7 @@ static int clk_programmable_set_parent(struct clk_hw *hw, u8 index) struct clk_programmable *prog = to_clk_programmable(hw); const struct clk_programmable_layout *layout = prog->layout; unsigned int mask = layout->css_mask; - unsigned int pckr = 0; + unsigned int pckr = index; if (layout->have_slck_mck) mask |= AT91_PMC_CSSMCK_MCK; diff --git a/drivers/clk/clk-oxnas.c b/drivers/clk/clk-oxnas.c index efba7d4dbcfc..79bcb2e42060 100644 --- a/drivers/clk/clk-oxnas.c +++ b/drivers/clk/clk-oxnas.c @@ -144,9 +144,9 @@ static int oxnas_stdclk_probe(struct platform_device *pdev) return -ENOMEM; regmap = syscon_node_to_regmap(of_get_parent(np)); - if (!regmap) { + if (IS_ERR(regmap)) { dev_err(&pdev->dev, "failed to have parent regmap\n"); - return -EINVAL; + return PTR_ERR(regmap); } for (i = 0; i < ARRAY_SIZE(clk_oxnas_init); i++) { diff --git a/drivers/clk/rockchip/clk-cpu.c b/drivers/clk/rockchip/clk-cpu.c index 4bb130cd0062..05b3d73bfefa 100644 --- a/drivers/clk/rockchip/clk-cpu.c +++ b/drivers/clk/rockchip/clk-cpu.c @@ -321,9 +321,9 @@ struct clk *rockchip_clk_register_cpuclk(const char *name, } cclk = clk_register(NULL, &cpuclk->hw); - if (IS_ERR(clk)) { + if (IS_ERR(cclk)) { pr_err("%s: could not register cpuclk %s\n", __func__, name); - ret = PTR_ERR(clk); + ret = PTR_ERR(cclk); goto free_rate_table; } diff --git a/drivers/clk/rockchip/clk-mmc-phase.c b/drivers/clk/rockchip/clk-mmc-phase.c index bc856f21f6b2..077fcdc7908b 100644 --- a/drivers/clk/rockchip/clk-mmc-phase.c +++ b/drivers/clk/rockchip/clk-mmc-phase.c @@ -41,8 +41,6 @@ static unsigned long rockchip_mmc_recalc(struct clk_hw *hw, #define ROCKCHIP_MMC_DEGREE_MASK 0x3 #define ROCKCHIP_MMC_DELAYNUM_OFFSET 2 #define ROCKCHIP_MMC_DELAYNUM_MASK (0xff << ROCKCHIP_MMC_DELAYNUM_OFFSET) -#define ROCKCHIP_MMC_INIT_STATE_RESET 0x1 -#define ROCKCHIP_MMC_INIT_STATE_SHIFT 1 #define PSECS_PER_SEC 1000000000000LL @@ -154,6 +152,7 @@ struct clk *rockchip_clk_register_mmc(const char *name, return ERR_PTR(-ENOMEM); init.name = name; + init.flags = 0; init.num_parents = num_parents; init.parent_names = parent_names; init.ops = &rockchip_mmc_clk_ops; @@ -162,15 +161,6 @@ struct clk *rockchip_clk_register_mmc(const char *name, mmc_clock->reg = reg; mmc_clock->shift = shift; - /* - * Assert init_state to soft reset the CLKGEN - * for mmc tuning phase and degree - */ - if (mmc_clock->shift == ROCKCHIP_MMC_INIT_STATE_SHIFT) - writel(HIWORD_UPDATE(ROCKCHIP_MMC_INIT_STATE_RESET, - ROCKCHIP_MMC_INIT_STATE_RESET, - mmc_clock->shift), mmc_clock->reg); - clk = clk_register(NULL, &mmc_clock->hw); if (IS_ERR(clk)) kfree(mmc_clock); diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c index 291543f52caa..8059a8d3ea36 100644 --- a/drivers/clk/rockchip/clk-rk3399.c +++ b/drivers/clk/rockchip/clk-rk3399.c @@ -832,9 +832,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(13), 1, GFLAGS), /* perihp */ - GATE(0, "cpll_aclk_perihp_src", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_aclk_perihp_src", "cpll", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(5), 0, GFLAGS), - GATE(0, "gpll_aclk_perihp_src", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(5), 1, GFLAGS), COMPOSITE(ACLK_PERIHP, "aclk_perihp", mux_aclk_perihp_p, CLK_IGNORE_UNUSED, RK3399_CLKSEL_CON(14), 7, 1, MFLAGS, 0, 5, DFLAGS, @@ -1466,6 +1466,8 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { static const char *const rk3399_cru_critical_clocks[] __initconst = { "aclk_cci_pre", + "aclk_gic", + "aclk_gic_noc", "pclk_perilp0", "pclk_perilp0", "hclk_perilp0", @@ -1508,6 +1510,7 @@ static void __init rk3399_clk_init(struct device_node *np) ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); if (IS_ERR(ctx)) { pr_err("%s: rockchip clk init failed\n", __func__); + iounmap(reg_base); return; } @@ -1553,6 +1556,7 @@ static void __init rk3399_pmu_clk_init(struct device_node *np) ctx = rockchip_clk_init(np, reg_base, CLKPMU_NR_CLKS); if (IS_ERR(ctx)) { pr_err("%s: rockchip pmu clk init failed\n", __func__); + iounmap(reg_base); return; } diff --git a/drivers/clk/sunxi/clk-sun4i-display.c b/drivers/clk/sunxi/clk-sun4i-display.c index 445a7498d6df..9780fac6d029 100644 --- a/drivers/clk/sunxi/clk-sun4i-display.c +++ b/drivers/clk/sunxi/clk-sun4i-display.c @@ -33,6 +33,8 @@ struct sun4i_a10_display_clk_data { u8 width_div; u8 width_mux; + + u32 flags; }; struct reset_data { @@ -166,7 +168,7 @@ static void __init sun4i_a10_display_init(struct device_node *node, data->has_div ? &div->hw : NULL, data->has_div ? &clk_divider_ops : NULL, &gate->hw, &clk_gate_ops, - 0); + data->flags); if (IS_ERR(clk)) { pr_err("%s: Couldn't register the clock\n", clk_name); goto free_div; @@ -232,6 +234,7 @@ static const struct sun4i_a10_display_clk_data sun4i_a10_tcon_ch0_data __initcon .offset_rst = 29, .offset_mux = 24, .width_mux = 2, + .flags = CLK_SET_RATE_PARENT, }; static void __init sun4i_a10_tcon_ch0_setup(struct device_node *node) diff --git a/drivers/clk/sunxi/clk-sun4i-tcon-ch1.c b/drivers/clk/sunxi/clk-sun4i-tcon-ch1.c index 98a4582de56a..b6d29d1bedca 100644 --- a/drivers/clk/sunxi/clk-sun4i-tcon-ch1.c +++ b/drivers/clk/sunxi/clk-sun4i-tcon-ch1.c @@ -79,15 +79,11 @@ static int tcon_ch1_is_enabled(struct clk_hw *hw) static u8 tcon_ch1_get_parent(struct clk_hw *hw) { struct tcon_ch1_clk *tclk = hw_to_tclk(hw); - int num_parents = clk_hw_get_num_parents(hw); u32 reg; reg = readl(tclk->reg) >> TCON_CH1_SCLK2_MUX_SHIFT; reg &= reg >> TCON_CH1_SCLK2_MUX_MASK; - if (reg >= num_parents) - return -EINVAL; - return reg; } diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 47352d25c15e..567788664723 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -27,6 +27,20 @@ config CLKBLD_I8253 config CLKSRC_MMIO bool +config BCM2835_TIMER + bool "BCM2835 timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables the support for the BCM2835 timer driver. + +config BCM_KONA_TIMER + bool "BCM mobile timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables the support for the BCM Kona mobile timer driver. + config DIGICOLOR_TIMER bool "Digicolor timer driver" if COMPILE_TEST depends on GENERIC_CLOCKEVENTS @@ -141,6 +155,72 @@ config CLKSRC_DBX500_PRCMU help Use the always on PRCMU Timer as clocksource +config CLPS711X_TIMER + bool "Cirrus logic timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Cirrus Logic PS711 timer. + +config ATLAS7_TIMER + bool "Atlas7 timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Atlas7 timer. + +config MOXART_TIMER + bool "Moxart timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Moxart timer. + +config MXS_TIMER + bool "Mxs timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + select STMP_DEVICE + help + Enables support for the Mxs timer. + +config PRIMA2_TIMER + bool "Prima2 timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Prima2 timer. + +config U300_TIMER + bool "U300 timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + depends on ARM + select CLKSRC_MMIO + help + Enables support for the U300 timer. + +config NSPIRE_TIMER + bool "NSpire timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Nspire timer. + +config KEYSTONE_TIMER + bool "Keystone timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + depends on ARM || ARM64 + select CLKSRC_MMIO + help + Enables support for the Keystone timer. + +config INTEGRATOR_AP_TIMER + bool "Integrator-ap timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Integrator-ap timer. + config CLKSRC_DBX500_PRCMU_SCHED_CLOCK bool "Clocksource PRCMU Timer sched_clock" depends on (CLKSRC_DBX500_PRCMU && !CLKSRC_NOMADIK_MTU_SCHED_CLOCK) @@ -208,14 +288,16 @@ config ARM_ARCH_TIMER select CLKSRC_ACPI if ACPI config ARM_ARCH_TIMER_EVTSTREAM - bool "Support for ARM architected timer event stream generation" + bool "Enable ARM architected timer event stream generation by default" default y if ARM_ARCH_TIMER depends on ARM_ARCH_TIMER help - This option enables support for event stream generation based on - the ARM architected timer. It is used for waking up CPUs executing - the wfe instruction at a frequency represented as a power-of-2 - divisor of the clock rate. + This option enables support by default for event stream generation + based on the ARM architected timer. It is used for waking up CPUs + executing the wfe instruction at a frequency represented as a + power-of-2 divisor of the clock rate. The behaviour can also be + overridden on the command line using the + clocksource.arm_arch_timer.evtstream parameter. The main use of the event stream is wfe-based timeouts of userspace locking implementations. It might also be useful for imposing timeout on wfe to safeguard against any programming errors in case an expected @@ -224,8 +306,9 @@ config ARM_ARCH_TIMER_EVTSTREAM hardware anomalies of missing events. config ARM_GLOBAL_TIMER - bool + bool "Support for the ARM global timer" if COMPILE_TEST select CLKSRC_OF if OF + depends on ARM help This options enables support for the ARM global timer unit @@ -243,7 +326,7 @@ config CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK Use ARM global timer clock source as sched_clock config ARMV7M_SYSTICK - bool + bool "Support for the ARMv7M system time" if COMPILE_TEST select CLKSRC_OF if OF select CLKSRC_MMIO help @@ -254,9 +337,12 @@ config ATMEL_PIT def_bool SOC_AT91SAM9 || SOC_SAMA5 config ATMEL_ST - bool + bool "Atmel ST timer support" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS select CLKSRC_OF select MFD_SYSCON + help + Support for the Atmel ST timer. config CLKSRC_METAG_GENERIC def_bool y if METAG @@ -270,7 +356,7 @@ config CLKSRC_EXYNOS_MCT Support for Multi Core Timer controller on Exynos SoCs. config CLKSRC_SAMSUNG_PWM - bool "PWM timer drvier for Samsung S3C, S5P" if COMPILE_TEST + bool "PWM timer driver for Samsung S3C, S5P" if COMPILE_TEST depends on GENERIC_CLOCKEVENTS depends on HAS_IOMEM help @@ -293,6 +379,14 @@ config VF_PIT_TIMER help Support for Period Interrupt Timer on Freescale Vybrid Family SoCs. +config OXNAS_RPS_TIMER + bool "Oxford Semiconductor OXNAS RPS Timers driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_OF + select CLKSRC_MMIO + help + This enables support for the Oxford Semiconductor OXNAS RPS timers. + config SYS_SUPPORTS_SH_CMT bool @@ -361,8 +455,8 @@ config CLKSRC_QCOM Qualcomm SoCs. config CLKSRC_VERSATILE - bool "ARM Versatile (Express) reference platforms clock source" - depends on PLAT_VERSATILE && GENERIC_SCHED_CLOCK && !ARCH_USES_GETTIMEOFFSET + bool "ARM Versatile (Express) reference platforms clock source" if COMPILE_TEST + depends on GENERIC_SCHED_CLOCK && !ARCH_USES_GETTIMEOFFSET select CLKSRC_OF default y if MFD_VEXPRESS_SYSREG help diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 473974f9590a..fd9d6df0bbc0 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -19,21 +19,21 @@ obj-$(CONFIG_CLKSRC_NOMADIK_MTU) += nomadik-mtu.o obj-$(CONFIG_CLKSRC_DBX500_PRCMU) += clksrc-dbx500-prcmu.o obj-$(CONFIG_ARMADA_370_XP_TIMER) += time-armada-370-xp.o obj-$(CONFIG_ORION_TIMER) += time-orion.o -obj-$(CONFIG_ARCH_BCM2835) += bcm2835_timer.o -obj-$(CONFIG_ARCH_CLPS711X) += clps711x-timer.o -obj-$(CONFIG_ARCH_ATLAS7) += timer-atlas7.o -obj-$(CONFIG_ARCH_MOXART) += moxart_timer.o -obj-$(CONFIG_ARCH_MXS) += mxs_timer.o +obj-$(CONFIG_BCM2835_TIMER) += bcm2835_timer.o +obj-$(CONFIG_CLPS711X_TIMER) += clps711x-timer.o +obj-$(CONFIG_ATLAS7_TIMER) += timer-atlas7.o +obj-$(CONFIG_MOXART_TIMER) += moxart_timer.o +obj-$(CONFIG_MXS_TIMER) += mxs_timer.o obj-$(CONFIG_CLKSRC_PXA) += pxa_timer.o -obj-$(CONFIG_ARCH_PRIMA2) += timer-prima2.o -obj-$(CONFIG_ARCH_U300) += timer-u300.o +obj-$(CONFIG_PRIMA2_TIMER) += timer-prima2.o +obj-$(CONFIG_U300_TIMER) += timer-u300.o obj-$(CONFIG_SUN4I_TIMER) += sun4i_timer.o obj-$(CONFIG_SUN5I_HSTIMER) += timer-sun5i.o obj-$(CONFIG_MESON6_TIMER) += meson6_timer.o obj-$(CONFIG_TEGRA_TIMER) += tegra20_timer.o obj-$(CONFIG_VT8500_TIMER) += vt8500_timer.o -obj-$(CONFIG_ARCH_NSPIRE) += zevio-timer.o -obj-$(CONFIG_ARCH_BCM_MOBILE) += bcm_kona_timer.o +obj-$(CONFIG_NSPIRE_TIMER) += zevio-timer.o +obj-$(CONFIG_BCM_KONA_TIMER) += bcm_kona_timer.o obj-$(CONFIG_CADENCE_TTC_TIMER) += cadence_ttc_timer.o obj-$(CONFIG_CLKSRC_EFM32) += time-efm32.o obj-$(CONFIG_CLKSRC_STM32) += timer-stm32.o @@ -48,6 +48,7 @@ obj-$(CONFIG_MTK_TIMER) += mtk_timer.o obj-$(CONFIG_CLKSRC_PISTACHIO) += time-pistachio.o obj-$(CONFIG_CLKSRC_TI_32K) += timer-ti-32k.o obj-$(CONFIG_CLKSRC_NPS) += timer-nps.o +obj-$(CONFIG_OXNAS_RPS_TIMER) += timer-oxnas-rps.o obj-$(CONFIG_ARM_ARCH_TIMER) += arm_arch_timer.o obj-$(CONFIG_ARM_GLOBAL_TIMER) += arm_global_timer.o @@ -55,8 +56,8 @@ obj-$(CONFIG_ARMV7M_SYSTICK) += armv7m_systick.o obj-$(CONFIG_ARM_TIMER_SP804) += timer-sp804.o obj-$(CONFIG_CLKSRC_METAG_GENERIC) += metag_generic.o obj-$(CONFIG_ARCH_HAS_TICK_BROADCAST) += dummy_timer.o -obj-$(CONFIG_ARCH_KEYSTONE) += timer-keystone.o -obj-$(CONFIG_ARCH_INTEGRATOR_AP) += timer-integrator-ap.o +obj-$(CONFIG_KEYSTONE_TIMER) += timer-keystone.o +obj-$(CONFIG_INTEGRATOR_AP_TIMER) += timer-integrator-ap.o obj-$(CONFIG_CLKSRC_VERSATILE) += versatile.o obj-$(CONFIG_CLKSRC_MIPS_GIC) += mips-gic-timer.o obj-$(CONFIG_CLKSRC_TANGO_XTAL) += tango_xtal.o diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 4814446a0024..5effd3027319 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -79,6 +79,14 @@ static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI; static bool arch_timer_c3stop; static bool arch_timer_mem_use_virtual; +static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); + +static int __init early_evtstrm_cfg(char *buf) +{ + return strtobool(buf, &evtstrm_enable); +} +early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); + /* * Architected system timer support. */ @@ -372,7 +380,7 @@ static int arch_timer_setup(struct clock_event_device *clk) enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0); arch_counter_set_user_access(); - if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM)) + if (evtstrm_enable) arch_timer_configure_evtstream(); return 0; @@ -693,25 +701,26 @@ arch_timer_needs_probing(int type, const struct of_device_id *matches) return needs_probing; } -static void __init arch_timer_common_init(void) +static int __init arch_timer_common_init(void) { unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER; /* Wait until both nodes are probed if we have two timers */ if ((arch_timers_present & mask) != mask) { if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match)) - return; + return 0; if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match)) - return; + return 0; } arch_timer_banner(arch_timers_present); arch_counter_register(arch_timers_present); - arch_timer_arch_init(); + return arch_timer_arch_init(); } -static void __init arch_timer_init(void) +static int __init arch_timer_init(void) { + int ret; /* * If HYP mode is available, we know that the physical timer * has been configured to be accessible from PL1. Use it, so @@ -739,23 +748,30 @@ static void __init arch_timer_init(void) if (!has_ppi) { pr_warn("arch_timer: No interrupt available, giving up\n"); - return; + return -EINVAL; } } - arch_timer_register(); - arch_timer_common_init(); + ret = arch_timer_register(); + if (ret) + return ret; + + ret = arch_timer_common_init(); + if (ret) + return ret; arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI]; + + return 0; } -static void __init arch_timer_of_init(struct device_node *np) +static int __init arch_timer_of_init(struct device_node *np) { int i; if (arch_timers_present & ARCH_CP15_TIMER) { pr_warn("arch_timer: multiple nodes in dt, skipping\n"); - return; + return 0; } arch_timers_present |= ARCH_CP15_TIMER; @@ -774,23 +790,23 @@ static void __init arch_timer_of_init(struct device_node *np) of_property_read_bool(np, "arm,cpu-registers-not-fw-configured")) arch_timer_uses_ppi = PHYS_SECURE_PPI; - arch_timer_init(); + return arch_timer_init(); } CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init); CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init); -static void __init arch_timer_mem_init(struct device_node *np) +static int __init arch_timer_mem_init(struct device_node *np) { struct device_node *frame, *best_frame = NULL; void __iomem *cntctlbase, *base; - unsigned int irq; + unsigned int irq, ret = -EINVAL; u32 cnttidr; arch_timers_present |= ARCH_MEM_TIMER; cntctlbase = of_iomap(np, 0); if (!cntctlbase) { pr_err("arch_timer: Can't find CNTCTLBase\n"); - return; + return -ENXIO; } cnttidr = readl_relaxed(cntctlbase + CNTTIDR); @@ -830,6 +846,7 @@ static void __init arch_timer_mem_init(struct device_node *np) best_frame = of_node_get(frame); } + ret= -ENXIO; base = arch_counter_base = of_iomap(best_frame, 0); if (!base) { pr_err("arch_timer: Can't map frame's registers\n"); @@ -841,6 +858,7 @@ static void __init arch_timer_mem_init(struct device_node *np) else irq = irq_of_parse_and_map(best_frame, 0); + ret = -EINVAL; if (!irq) { pr_err("arch_timer: Frame missing %s irq", arch_timer_mem_use_virtual ? "virt" : "phys"); @@ -848,11 +866,15 @@ static void __init arch_timer_mem_init(struct device_node *np) } arch_timer_detect_rate(base, np); - arch_timer_mem_register(base, irq); - arch_timer_common_init(); + ret = arch_timer_mem_register(base, irq); + if (ret) + goto out; + + return arch_timer_common_init(); out: iounmap(cntctlbase); of_node_put(best_frame); + return ret; } CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem", arch_timer_mem_init); diff --git a/drivers/clocksource/arm_global_timer.c b/drivers/clocksource/arm_global_timer.c index 9df0d1699d22..2a9ceb6e93f9 100644 --- a/drivers/clocksource/arm_global_timer.c +++ b/drivers/clocksource/arm_global_timer.c @@ -238,7 +238,7 @@ static void __init gt_delay_timer_init(void) register_current_timer_delay(>_delay_timer); } -static void __init gt_clocksource_init(void) +static int __init gt_clocksource_init(void) { writel(0, gt_base + GT_CONTROL); writel(0, gt_base + GT_COUNTER0); @@ -249,7 +249,7 @@ static void __init gt_clocksource_init(void) #ifdef CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK sched_clock_register(gt_sched_clock_read, 64, gt_clk_rate); #endif - clocksource_register_hz(>_clocksource, gt_clk_rate); + return clocksource_register_hz(>_clocksource, gt_clk_rate); } static int gt_cpu_notify(struct notifier_block *self, unsigned long action, @@ -270,7 +270,7 @@ static struct notifier_block gt_cpu_nb = { .notifier_call = gt_cpu_notify, }; -static void __init global_timer_of_register(struct device_node *np) +static int __init global_timer_of_register(struct device_node *np) { struct clk *gt_clk; int err = 0; @@ -283,19 +283,19 @@ static void __init global_timer_of_register(struct device_node *np) if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9 && (read_cpuid_id() & 0xf0000f) < 0x200000) { pr_warn("global-timer: non support for this cpu version.\n"); - return; + return -ENOSYS; } gt_ppi = irq_of_parse_and_map(np, 0); if (!gt_ppi) { pr_warn("global-timer: unable to parse irq\n"); - return; + return -EINVAL; } gt_base = of_iomap(np, 0); if (!gt_base) { pr_warn("global-timer: invalid base address\n"); - return; + return -ENXIO; } gt_clk = of_clk_get(np, 0); @@ -332,11 +332,17 @@ static void __init global_timer_of_register(struct device_node *np) } /* Immediately configure the timer on the boot CPU */ - gt_clocksource_init(); - gt_clockevents_init(this_cpu_ptr(gt_evt)); + err = gt_clocksource_init(); + if (err) + goto out_irq; + + err = gt_clockevents_init(this_cpu_ptr(gt_evt)); + if (err) + goto out_irq; + gt_delay_timer_init(); - return; + return 0; out_irq: free_percpu_irq(gt_ppi, gt_evt); @@ -347,6 +353,8 @@ out_clk: out_unmap: iounmap(gt_base); WARN(err, "ARM Global timer register failed (%d)\n", err); + + return err; } /* Only tested on r2p2 and r3p0 */ diff --git a/drivers/clocksource/armv7m_systick.c b/drivers/clocksource/armv7m_systick.c index addfd2c64f54..a315491b7047 100644 --- a/drivers/clocksource/armv7m_systick.c +++ b/drivers/clocksource/armv7m_systick.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,7 @@ #define SYSTICK_LOAD_RELOAD_MASK 0x00FFFFFF -static void __init system_timer_of_register(struct device_node *np) +static int __init system_timer_of_register(struct device_node *np) { struct clk *clk = NULL; void __iomem *base; @@ -31,22 +32,26 @@ static void __init system_timer_of_register(struct device_node *np) base = of_iomap(np, 0); if (!base) { pr_warn("system-timer: invalid base address\n"); - return; + return -ENXIO; } ret = of_property_read_u32(np, "clock-frequency", &rate); if (ret) { clk = of_clk_get(np, 0); - if (IS_ERR(clk)) + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); goto out_unmap; + } ret = clk_prepare_enable(clk); if (ret) goto out_clk_put; rate = clk_get_rate(clk); - if (!rate) + if (!rate) { + ret = -EINVAL; goto out_clk_disable; + } } writel_relaxed(SYSTICK_LOAD_RELOAD_MASK, base + SYST_RVR); @@ -64,7 +69,7 @@ static void __init system_timer_of_register(struct device_node *np) pr_info("ARM System timer initialized as clocksource\n"); - return; + return 0; out_clk_disable: clk_disable_unprepare(clk); @@ -73,6 +78,8 @@ out_clk_put: out_unmap: iounmap(base); pr_warn("ARM System timer register failed (%d)\n", ret); + + return ret; } CLOCKSOURCE_OF_DECLARE(arm_systick, "arm,armv7m-systick", diff --git a/drivers/clocksource/asm9260_timer.c b/drivers/clocksource/asm9260_timer.c index 217438d39eb3..1ba871b7fe11 100644 --- a/drivers/clocksource/asm9260_timer.c +++ b/drivers/clocksource/asm9260_timer.c @@ -184,7 +184,7 @@ static irqreturn_t asm9260_timer_interrupt(int irq, void *dev_id) * Timer initialization * --------------------------------------------------------------------------- */ -static void __init asm9260_timer_init(struct device_node *np) +static int __init asm9260_timer_init(struct device_node *np) { int irq; struct clk *clk; @@ -192,20 +192,26 @@ static void __init asm9260_timer_init(struct device_node *np) unsigned long rate; priv.base = of_io_request_and_map(np, 0, np->name); - if (IS_ERR(priv.base)) - panic("%s: unable to map resource", np->name); + if (IS_ERR(priv.base)) { + pr_err("%s: unable to map resource", np->name); + return PTR_ERR(priv.base); + } clk = of_clk_get(np, 0); ret = clk_prepare_enable(clk); - if (ret) - panic("Failed to enable clk!\n"); + if (ret) { + pr_err("Failed to enable clk!\n"); + return ret; + } irq = irq_of_parse_and_map(np, 0); ret = request_irq(irq, asm9260_timer_interrupt, IRQF_TIMER, DRIVER_NAME, &event_dev); - if (ret) - panic("Failed to setup irq!\n"); + if (ret) { + pr_err("Failed to setup irq!\n"); + return ret; + } /* set all timers for count-up */ writel_relaxed(BM_DIR_DEFAULT, priv.base + HW_DIR); @@ -229,6 +235,8 @@ static void __init asm9260_timer_init(struct device_node *np) priv.ticks_per_jiffy = DIV_ROUND_CLOSEST(rate, HZ); event_dev.cpumask = cpumask_of(0); clockevents_config_and_register(&event_dev, rate, 0x2c00, 0xfffffffe); + + return 0; } CLOCKSOURCE_OF_DECLARE(asm9260_timer, "alphascale,asm9260-timer", asm9260_timer_init); diff --git a/drivers/clocksource/bcm2835_timer.c b/drivers/clocksource/bcm2835_timer.c index 6f2822928963..e71acf231c89 100644 --- a/drivers/clocksource/bcm2835_timer.c +++ b/drivers/clocksource/bcm2835_timer.c @@ -80,19 +80,24 @@ static irqreturn_t bcm2835_time_interrupt(int irq, void *dev_id) } } -static void __init bcm2835_timer_init(struct device_node *node) +static int __init bcm2835_timer_init(struct device_node *node) { void __iomem *base; u32 freq; - int irq; + int irq, ret; struct bcm2835_timer *timer; base = of_iomap(node, 0); - if (!base) - panic("Can't remap registers"); + if (!base) { + pr_err("Can't remap registers"); + return -ENXIO; + } - if (of_property_read_u32(node, "clock-frequency", &freq)) - panic("Can't read clock-frequency"); + ret = of_property_read_u32(node, "clock-frequency", &freq); + if (ret) { + pr_err("Can't read clock-frequency"); + return ret; + } system_clock = base + REG_COUNTER_LO; sched_clock_register(bcm2835_sched_read, 32, freq); @@ -101,12 +106,16 @@ static void __init bcm2835_timer_init(struct device_node *node) freq, 300, 32, clocksource_mmio_readl_up); irq = irq_of_parse_and_map(node, DEFAULT_TIMER); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_err("Can't parse IRQ"); + return -EINVAL; + } timer = kzalloc(sizeof(*timer), GFP_KERNEL); - if (!timer) - panic("Can't allocate timer struct\n"); + if (!timer) { + pr_err("Can't allocate timer struct\n"); + return -ENOMEM; + } timer->control = base + REG_CONTROL; timer->compare = base + REG_COMPARE(DEFAULT_TIMER); @@ -121,12 +130,17 @@ static void __init bcm2835_timer_init(struct device_node *node) timer->act.dev_id = timer; timer->act.handler = bcm2835_time_interrupt; - if (setup_irq(irq, &timer->act)) - panic("Can't set up timer IRQ\n"); + ret = setup_irq(irq, &timer->act); + if (ret) { + pr_err("Can't set up timer IRQ\n"); + return ret; + } clockevents_config_and_register(&timer->evt, freq, 0xf, 0xffffffff); pr_info("bcm2835: system timer (irq = %d)\n", irq); + + return 0; } CLOCKSOURCE_OF_DECLARE(bcm2835, "brcm,bcm2835-system-timer", bcm2835_timer_init); diff --git a/drivers/clocksource/bcm_kona_timer.c b/drivers/clocksource/bcm_kona_timer.c index e717e87df9bc..7e3fd375a627 100644 --- a/drivers/clocksource/bcm_kona_timer.c +++ b/drivers/clocksource/bcm_kona_timer.c @@ -20,7 +20,6 @@ #include #include -#include #include #include @@ -163,16 +162,11 @@ static struct irqaction kona_timer_irq = { .handler = kona_timer_interrupt, }; -static void __init kona_timer_init(struct device_node *node) +static int __init kona_timer_init(struct device_node *node) { u32 freq; struct clk *external_clk; - if (!of_device_is_available(node)) { - pr_info("Kona Timer v1 marked as disabled in device tree\n"); - return; - } - external_clk = of_clk_get_by_name(node, NULL); if (!IS_ERR(external_clk)) { @@ -182,7 +176,7 @@ static void __init kona_timer_init(struct device_node *node) arch_timer_rate = freq; } else { pr_err("Kona Timer v1 unable to determine clock-frequency"); - return; + return -EINVAL; } /* Setup IRQ numbers */ @@ -196,6 +190,8 @@ static void __init kona_timer_init(struct device_node *node) kona_timer_clockevents_init(); setup_irq(timers.tmr_irq, &kona_timer_irq); kona_timer_set_next_event((arch_timer_rate / HZ), NULL); + + return 0; } CLOCKSOURCE_OF_DECLARE(brcm_kona, "brcm,kona-timer", kona_timer_init); diff --git a/drivers/clocksource/cadence_ttc_timer.c b/drivers/clocksource/cadence_ttc_timer.c index 9be6018bd2b8..fbfbdec13b08 100644 --- a/drivers/clocksource/cadence_ttc_timer.c +++ b/drivers/clocksource/cadence_ttc_timer.c @@ -322,22 +322,22 @@ static int ttc_rate_change_clocksource_cb(struct notifier_block *nb, return NOTIFY_DONE; } -static void __init ttc_setup_clocksource(struct clk *clk, void __iomem *base, +static int __init ttc_setup_clocksource(struct clk *clk, void __iomem *base, u32 timer_width) { struct ttc_timer_clocksource *ttccs; int err; ttccs = kzalloc(sizeof(*ttccs), GFP_KERNEL); - if (WARN_ON(!ttccs)) - return; + if (!ttccs) + return -ENOMEM; ttccs->ttc.clk = clk; err = clk_prepare_enable(ttccs->ttc.clk); - if (WARN_ON(err)) { + if (err) { kfree(ttccs); - return; + return err; } ttccs->ttc.freq = clk_get_rate(ttccs->ttc.clk); @@ -345,8 +345,10 @@ static void __init ttc_setup_clocksource(struct clk *clk, void __iomem *base, ttccs->ttc.clk_rate_change_nb.notifier_call = ttc_rate_change_clocksource_cb; ttccs->ttc.clk_rate_change_nb.next = NULL; - if (clk_notifier_register(ttccs->ttc.clk, - &ttccs->ttc.clk_rate_change_nb)) + + err = clk_notifier_register(ttccs->ttc.clk, + &ttccs->ttc.clk_rate_change_nb); + if (err) pr_warn("Unable to register clock notifier.\n"); ttccs->ttc.base_addr = base; @@ -368,14 +370,16 @@ static void __init ttc_setup_clocksource(struct clk *clk, void __iomem *base, ttccs->ttc.base_addr + TTC_CNT_CNTRL_OFFSET); err = clocksource_register_hz(&ttccs->cs, ttccs->ttc.freq / PRESCALE); - if (WARN_ON(err)) { + if (err) { kfree(ttccs); - return; + return err; } ttc_sched_clock_val_reg = base + TTC_COUNT_VAL_OFFSET; sched_clock_register(ttc_sched_clock_read, timer_width, ttccs->ttc.freq / PRESCALE); + + return 0; } static int ttc_rate_change_clockevent_cb(struct notifier_block *nb, @@ -401,30 +405,35 @@ static int ttc_rate_change_clockevent_cb(struct notifier_block *nb, } } -static void __init ttc_setup_clockevent(struct clk *clk, - void __iomem *base, u32 irq) +static int __init ttc_setup_clockevent(struct clk *clk, + void __iomem *base, u32 irq) { struct ttc_timer_clockevent *ttcce; int err; ttcce = kzalloc(sizeof(*ttcce), GFP_KERNEL); - if (WARN_ON(!ttcce)) - return; + if (!ttcce) + return -ENOMEM; ttcce->ttc.clk = clk; err = clk_prepare_enable(ttcce->ttc.clk); - if (WARN_ON(err)) { + if (err) { kfree(ttcce); - return; + return err; } ttcce->ttc.clk_rate_change_nb.notifier_call = ttc_rate_change_clockevent_cb; ttcce->ttc.clk_rate_change_nb.next = NULL; - if (clk_notifier_register(ttcce->ttc.clk, - &ttcce->ttc.clk_rate_change_nb)) + + err = clk_notifier_register(ttcce->ttc.clk, + &ttcce->ttc.clk_rate_change_nb); + if (err) { pr_warn("Unable to register clock notifier.\n"); + return err; + } + ttcce->ttc.freq = clk_get_rate(ttcce->ttc.clk); ttcce->ttc.base_addr = base; @@ -451,13 +460,15 @@ static void __init ttc_setup_clockevent(struct clk *clk, err = request_irq(irq, ttc_clock_event_interrupt, IRQF_TIMER, ttcce->ce.name, ttcce); - if (WARN_ON(err)) { + if (err) { kfree(ttcce); - return; + return err; } clockevents_config_and_register(&ttcce->ce, ttcce->ttc.freq / PRESCALE, 1, 0xfffe); + + return 0; } /** @@ -466,17 +477,17 @@ static void __init ttc_setup_clockevent(struct clk *clk, * Initializes the timer hardware and register the clock source and clock event * timers with Linux kernal timer framework */ -static void __init ttc_timer_init(struct device_node *timer) +static int __init ttc_timer_init(struct device_node *timer) { unsigned int irq; void __iomem *timer_baseaddr; struct clk *clk_cs, *clk_ce; static int initialized; - int clksel; + int clksel, ret; u32 timer_width = 16; if (initialized) - return; + return 0; initialized = 1; @@ -488,13 +499,13 @@ static void __init ttc_timer_init(struct device_node *timer) timer_baseaddr = of_iomap(timer, 0); if (!timer_baseaddr) { pr_err("ERROR: invalid timer base address\n"); - BUG(); + return -ENXIO; } irq = irq_of_parse_and_map(timer, 1); if (irq <= 0) { pr_err("ERROR: invalid interrupt number\n"); - BUG(); + return -EINVAL; } of_property_read_u32(timer, "timer-width", &timer_width); @@ -504,7 +515,7 @@ static void __init ttc_timer_init(struct device_node *timer) clk_cs = of_clk_get(timer, clksel); if (IS_ERR(clk_cs)) { pr_err("ERROR: timer input clock not found\n"); - BUG(); + return PTR_ERR(clk_cs); } clksel = readl_relaxed(timer_baseaddr + 4 + TTC_CLK_CNTRL_OFFSET); @@ -512,13 +523,20 @@ static void __init ttc_timer_init(struct device_node *timer) clk_ce = of_clk_get(timer, clksel); if (IS_ERR(clk_ce)) { pr_err("ERROR: timer input clock not found\n"); - BUG(); + return PTR_ERR(clk_ce); } - ttc_setup_clocksource(clk_cs, timer_baseaddr, timer_width); - ttc_setup_clockevent(clk_ce, timer_baseaddr + 4, irq); + ret = ttc_setup_clocksource(clk_cs, timer_baseaddr, timer_width); + if (ret) + return ret; + + ret = ttc_setup_clockevent(clk_ce, timer_baseaddr + 4, irq); + if (ret) + return ret; pr_info("%s #0 at %p, irq=%d\n", timer->name, timer_baseaddr, irq); + + return 0; } CLOCKSOURCE_OF_DECLARE(ttc, "cdns,ttc", ttc_timer_init); diff --git a/drivers/clocksource/clksrc-dbx500-prcmu.c b/drivers/clocksource/clksrc-dbx500-prcmu.c index dfad6eb99662..77a365f573d7 100644 --- a/drivers/clocksource/clksrc-dbx500-prcmu.c +++ b/drivers/clocksource/clksrc-dbx500-prcmu.c @@ -64,7 +64,7 @@ static u64 notrace dbx500_prcmu_sched_clock_read(void) #endif -static void __init clksrc_dbx500_prcmu_init(struct device_node *node) +static int __init clksrc_dbx500_prcmu_init(struct device_node *node) { clksrc_dbx500_timer_base = of_iomap(node, 0); @@ -84,7 +84,7 @@ static void __init clksrc_dbx500_prcmu_init(struct device_node *node) #ifdef CONFIG_CLKSRC_DBX500_PRCMU_SCHED_CLOCK sched_clock_register(dbx500_prcmu_sched_clock_read, 32, RATE_32K); #endif - clocksource_register_hz(&clocksource_dbx500_prcmu, RATE_32K); + return clocksource_register_hz(&clocksource_dbx500_prcmu, RATE_32K); } CLOCKSOURCE_OF_DECLARE(dbx500_prcmu, "stericsson,db8500-prcmu-timer-4", clksrc_dbx500_prcmu_init); diff --git a/drivers/clocksource/clksrc-probe.c b/drivers/clocksource/clksrc-probe.c index 7cb6c923a836..bc62be97f0a8 100644 --- a/drivers/clocksource/clksrc-probe.c +++ b/drivers/clocksource/clksrc-probe.c @@ -28,15 +28,23 @@ void __init clocksource_probe(void) { struct device_node *np; const struct of_device_id *match; - of_init_fn_1 init_func; + of_init_fn_1_ret init_func_ret; unsigned clocksources = 0; + int ret; for_each_matching_node_and_match(np, __clksrc_of_table, &match) { if (!of_device_is_available(np)) continue; - init_func = match->data; - init_func(np); + init_func_ret = match->data; + + ret = init_func_ret(np); + if (ret) { + pr_err("Failed to initialize '%s': %d", + of_node_full_name(np), ret); + continue; + } + clocksources++; } diff --git a/drivers/clocksource/clksrc_st_lpc.c b/drivers/clocksource/clksrc_st_lpc.c index 65ec4674416d..03cc49217bb4 100644 --- a/drivers/clocksource/clksrc_st_lpc.c +++ b/drivers/clocksource/clksrc_st_lpc.c @@ -92,7 +92,7 @@ static int __init st_clksrc_setup_clk(struct device_node *np) return 0; } -static void __init st_clksrc_of_register(struct device_node *np) +static int __init st_clksrc_of_register(struct device_node *np) { int ret; uint32_t mode; @@ -100,32 +100,36 @@ static void __init st_clksrc_of_register(struct device_node *np) ret = of_property_read_u32(np, "st,lpc-mode", &mode); if (ret) { pr_err("clksrc-st-lpc: An LPC mode must be provided\n"); - return; + return ret; } /* LPC can either run as a Clocksource or in RTC or WDT mode */ if (mode != ST_LPC_MODE_CLKSRC) - return; + return 0; ddata.base = of_iomap(np, 0); if (!ddata.base) { pr_err("clksrc-st-lpc: Unable to map iomem\n"); - return; + return -ENXIO; } - if (st_clksrc_setup_clk(np)) { + ret = st_clksrc_setup_clk(np); + if (ret) { iounmap(ddata.base); - return; + return ret; } - if (st_clksrc_init()) { + ret = st_clksrc_init(); + if (ret) { clk_disable_unprepare(ddata.clk); clk_put(ddata.clk); iounmap(ddata.base); - return; + return ret; } pr_info("clksrc-st-lpc: clocksource initialised - running @ %luHz\n", clk_get_rate(ddata.clk)); + + return ret; } CLOCKSOURCE_OF_DECLARE(ddata, "st,stih407-lpc", st_clksrc_of_register); diff --git a/drivers/clocksource/clps711x-timer.c b/drivers/clocksource/clps711x-timer.c index cdd86e3525bb..84aed78261e4 100644 --- a/drivers/clocksource/clps711x-timer.c +++ b/drivers/clocksource/clps711x-timer.c @@ -104,7 +104,7 @@ void __init clps711x_clksrc_init(void __iomem *tc1_base, void __iomem *tc2_base, } #ifdef CONFIG_CLKSRC_OF -static void __init clps711x_timer_init(struct device_node *np) +static int __init clps711x_timer_init(struct device_node *np) { unsigned int irq = irq_of_parse_and_map(np, 0); struct clk *clock = of_clk_get(np, 0); @@ -112,13 +112,11 @@ static void __init clps711x_timer_init(struct device_node *np) switch (of_alias_get_id(np, "timer")) { case CLPS711X_CLKSRC_CLOCKSOURCE: - BUG_ON(_clps711x_clksrc_init(clock, base)); - break; + return _clps711x_clksrc_init(clock, base); case CLPS711X_CLKSRC_CLOCKEVENT: - BUG_ON(_clps711x_clkevt_init(clock, base, irq)); - break; + return _clps711x_clkevt_init(clock, base, irq); default: - break; + return -EINVAL; } } CLOCKSOURCE_OF_DECLARE(clps711x, "cirrus,clps711x-timer", clps711x_timer_init); diff --git a/drivers/clocksource/dw_apb_timer_of.c b/drivers/clocksource/dw_apb_timer_of.c index 860843cef572..aee6c0d39a7c 100644 --- a/drivers/clocksource/dw_apb_timer_of.c +++ b/drivers/clocksource/dw_apb_timer_of.c @@ -143,7 +143,7 @@ static struct delay_timer dw_apb_delay_timer = { #endif static int num_called; -static void __init dw_apb_timer_init(struct device_node *timer) +static int __init dw_apb_timer_init(struct device_node *timer) { switch (num_called) { case 0: @@ -164,6 +164,8 @@ static void __init dw_apb_timer_init(struct device_node *timer) } num_called++; + + return 0; } CLOCKSOURCE_OF_DECLARE(pc3x2_timer, "picochip,pc3x2-timer", dw_apb_timer_init); CLOCKSOURCE_OF_DECLARE(apb_timer_osc, "snps,dw-apb-timer-osc", dw_apb_timer_init); diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index be09bc0b5e26..0d18dd4b3bd2 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -232,7 +232,7 @@ static cycles_t exynos4_read_current_timer(void) return exynos4_read_count_32(); } -static void __init exynos4_clocksource_init(void) +static int __init exynos4_clocksource_init(void) { exynos4_mct_frc_start(); @@ -244,6 +244,8 @@ static void __init exynos4_clocksource_init(void) panic("%s: can't register clocksource\n", mct_frc.name); sched_clock_register(exynos4_read_sched_clock, 32, clk_rate); + + return 0; } static void exynos4_mct_comp0_stop(void) @@ -335,12 +337,14 @@ static struct irqaction mct_comp_event_irq = { .dev_id = &mct_comp_device, }; -static void exynos4_clockevent_init(void) +static int exynos4_clockevent_init(void) { mct_comp_device.cpumask = cpumask_of(0); clockevents_config_and_register(&mct_comp_device, clk_rate, 0xf, 0xffffffff); setup_irq(mct_irqs[MCT_G0_IRQ], &mct_comp_event_irq); + + return 0; } static DEFINE_PER_CPU(struct mct_clock_event_device, percpu_mct_tick); @@ -516,7 +520,7 @@ static struct notifier_block exynos4_mct_cpu_nb = { .notifier_call = exynos4_mct_cpu_notify, }; -static void __init exynos4_timer_resources(struct device_node *np, void __iomem *base) +static int __init exynos4_timer_resources(struct device_node *np, void __iomem *base) { int err, cpu; struct mct_clock_event_device *mevt = this_cpu_ptr(&percpu_mct_tick); @@ -572,15 +576,17 @@ static void __init exynos4_timer_resources(struct device_node *np, void __iomem /* Immediately configure the timer on the boot CPU */ exynos4_local_timer_setup(mevt); - return; + return 0; out_irq: free_percpu_irq(mct_irqs[MCT_L0_IRQ], &percpu_mct_tick); + return err; } -static void __init mct_init_dt(struct device_node *np, unsigned int int_type) +static int __init mct_init_dt(struct device_node *np, unsigned int int_type) { u32 nr_irqs, i; + int ret; mct_int_type = int_type; @@ -600,18 +606,24 @@ static void __init mct_init_dt(struct device_node *np, unsigned int int_type) for (i = MCT_L0_IRQ; i < nr_irqs; i++) mct_irqs[i] = irq_of_parse_and_map(np, i); - exynos4_timer_resources(np, of_iomap(np, 0)); - exynos4_clocksource_init(); - exynos4_clockevent_init(); + ret = exynos4_timer_resources(np, of_iomap(np, 0)); + if (ret) + return ret; + + ret = exynos4_clocksource_init(); + if (ret) + return ret; + + return exynos4_clockevent_init(); } -static void __init mct_init_spi(struct device_node *np) +static int __init mct_init_spi(struct device_node *np) { return mct_init_dt(np, MCT_INT_SPI); } -static void __init mct_init_ppi(struct device_node *np) +static int __init mct_init_ppi(struct device_node *np) { return mct_init_dt(np, MCT_INT_PPI); } diff --git a/drivers/clocksource/fsl_ftm_timer.c b/drivers/clocksource/fsl_ftm_timer.c index 517e1c7624d4..738515b89073 100644 --- a/drivers/clocksource/fsl_ftm_timer.c +++ b/drivers/clocksource/fsl_ftm_timer.c @@ -316,15 +316,16 @@ static int __init ftm_calc_closest_round_cyc(unsigned long freq) return 0; } -static void __init ftm_timer_init(struct device_node *np) +static int __init ftm_timer_init(struct device_node *np) { unsigned long freq; - int irq; + int ret, irq; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) - return; + return -ENOMEM; + ret = -ENXIO; priv->clkevt_base = of_iomap(np, 0); if (!priv->clkevt_base) { pr_err("ftm: unable to map event timer registers\n"); @@ -337,6 +338,7 @@ static void __init ftm_timer_init(struct device_node *np) goto err; } + ret = -EINVAL; irq = irq_of_parse_and_map(np, 0); if (irq <= 0) { pr_err("ftm: unable to get IRQ from DT, %d\n", irq); @@ -349,18 +351,22 @@ static void __init ftm_timer_init(struct device_node *np) if (!freq) goto err; - if (ftm_calc_closest_round_cyc(freq)) + ret = ftm_calc_closest_round_cyc(freq); + if (ret) goto err; - if (ftm_clocksource_init(freq)) + ret = ftm_clocksource_init(freq); + if (ret) goto err; - if (ftm_clockevent_init(freq, irq)) + ret = ftm_clockevent_init(freq, irq); + if (ret) goto err; - return; + return 0; err: kfree(priv); + return ret; } CLOCKSOURCE_OF_DECLARE(flextimer, "fsl,ftm-timer", ftm_timer_init); diff --git a/drivers/clocksource/h8300_timer16.c b/drivers/clocksource/h8300_timer16.c index 75c44079b345..07d9d5be9054 100644 --- a/drivers/clocksource/h8300_timer16.c +++ b/drivers/clocksource/h8300_timer16.c @@ -126,7 +126,7 @@ static struct timer16_priv timer16_priv = { #define REG_CH 0 #define REG_COMM 1 -static void __init h8300_16timer_init(struct device_node *node) +static int __init h8300_16timer_init(struct device_node *node) { void __iomem *base[2]; int ret, irq; @@ -136,9 +136,10 @@ static void __init h8300_16timer_init(struct device_node *node) clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("failed to get clock for clocksource\n"); - return; + return PTR_ERR(clk); } + ret = -ENXIO; base[REG_CH] = of_iomap(node, 0); if (!base[REG_CH]) { pr_err("failed to map registers for clocksource\n"); @@ -151,6 +152,7 @@ static void __init h8300_16timer_init(struct device_node *node) goto unmap_ch; } + ret = -EINVAL; irq = irq_of_parse_and_map(node, 0); if (!irq) { pr_err("failed to get irq for clockevent\n"); @@ -174,7 +176,7 @@ static void __init h8300_16timer_init(struct device_node *node) clocksource_register_hz(&timer16_priv.cs, clk_get_rate(clk) / 8); - return; + return 0; unmap_comm: iounmap(base[REG_COMM]); @@ -182,6 +184,8 @@ unmap_ch: iounmap(base[REG_CH]); free_clk: clk_put(clk); + return ret; } -CLOCKSOURCE_OF_DECLARE(h8300_16bit, "renesas,16bit-timer", h8300_16timer_init); +CLOCKSOURCE_OF_DECLARE(h8300_16bit, "renesas,16bit-timer", + h8300_16timer_init); diff --git a/drivers/clocksource/h8300_timer8.c b/drivers/clocksource/h8300_timer8.c index c151941e1956..546bb180f5a4 100644 --- a/drivers/clocksource/h8300_timer8.c +++ b/drivers/clocksource/h8300_timer8.c @@ -164,24 +164,26 @@ static struct timer8_priv timer8_priv = { }, }; -static void __init h8300_8timer_init(struct device_node *node) +static int __init h8300_8timer_init(struct device_node *node) { void __iomem *base; - int irq; + int irq, ret; struct clk *clk; clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("failed to get clock for clockevent\n"); - return; + return PTR_ERR(clk); } + ret = ENXIO; base = of_iomap(node, 0); if (!base) { pr_err("failed to map registers for clockevent\n"); goto free_clk; } + ret = -EINVAL; irq = irq_of_parse_and_map(node, 0); if (!irq) { pr_err("failed to get irq for clockevent\n"); @@ -205,11 +207,12 @@ static void __init h8300_8timer_init(struct device_node *node) clockevents_config_and_register(&timer8_priv.ced, timer8_priv.rate, 1, 0x0000ffff); - return; + return 0; unmap_reg: iounmap(base); free_clk: clk_put(clk); + return ret; } CLOCKSOURCE_OF_DECLARE(h8300_8bit, "renesas,8bit-timer", h8300_8timer_init); diff --git a/drivers/clocksource/h8300_tpu.c b/drivers/clocksource/h8300_tpu.c index d4c1a287c262..7bdf1991c847 100644 --- a/drivers/clocksource/h8300_tpu.c +++ b/drivers/clocksource/h8300_tpu.c @@ -119,15 +119,16 @@ static struct tpu_priv tpu_priv = { #define CH_L 0 #define CH_H 1 -static void __init h8300_tpu_init(struct device_node *node) +static int __init h8300_tpu_init(struct device_node *node) { void __iomem *base[2]; struct clk *clk; + int ret = -ENXIO; clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("failed to get clock for clocksource\n"); - return; + return PTR_ERR(clk); } base[CH_L] = of_iomap(node, CH_L); @@ -144,14 +145,13 @@ static void __init h8300_tpu_init(struct device_node *node) tpu_priv.mapbase1 = base[CH_L]; tpu_priv.mapbase2 = base[CH_H]; - clocksource_register_hz(&tpu_priv.cs, clk_get_rate(clk) / 64); - - return; + return clocksource_register_hz(&tpu_priv.cs, clk_get_rate(clk) / 64); unmap_L: iounmap(base[CH_H]); free_clk: clk_put(clk); + return ret; } CLOCKSOURCE_OF_DECLARE(h8300_tpu, "renesas,tpu", h8300_tpu_init); diff --git a/drivers/clocksource/meson6_timer.c b/drivers/clocksource/meson6_timer.c index 1fa22c4d2d49..52af591a9fc7 100644 --- a/drivers/clocksource/meson6_timer.c +++ b/drivers/clocksource/meson6_timer.c @@ -126,18 +126,22 @@ static struct irqaction meson6_timer_irq = { .dev_id = &meson6_clockevent, }; -static void __init meson6_timer_init(struct device_node *node) +static int __init meson6_timer_init(struct device_node *node) { u32 val; int ret, irq; timer_base = of_io_request_and_map(node, 0, "meson6-timer"); - if (IS_ERR(timer_base)) - panic("Can't map registers"); + if (IS_ERR(timer_base)) { + pr_err("Can't map registers"); + return -ENXIO; + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_err("Can't parse IRQ"); + return -EINVAL; + } /* Set 1us for timer E */ val = readl(timer_base + TIMER_ISA_MUX); @@ -158,14 +162,17 @@ static void __init meson6_timer_init(struct device_node *node) meson6_clkevt_time_stop(CED_ID); ret = setup_irq(irq, &meson6_timer_irq); - if (ret) + if (ret) { pr_warn("failed to setup irq %d\n", irq); + return ret; + } meson6_clockevent.cpumask = cpu_possible_mask; meson6_clockevent.irq = irq; clockevents_config_and_register(&meson6_clockevent, USEC_PER_SEC, 1, 0xfffe); + return 0; } CLOCKSOURCE_OF_DECLARE(meson6, "amlogic,meson6-timer", meson6_timer_init); diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c index 89d3e4d7900c..1572c7a778ab 100644 --- a/drivers/clocksource/mips-gic-timer.c +++ b/drivers/clocksource/mips-gic-timer.c @@ -146,7 +146,7 @@ static struct clocksource gic_clocksource = { .archdata = { .vdso_clock_mode = VDSO_CLOCK_GIC }, }; -static void __init __gic_clocksource_init(void) +static int __init __gic_clocksource_init(void) { int ret; @@ -159,6 +159,8 @@ static void __init __gic_clocksource_init(void) ret = clocksource_register_hz(&gic_clocksource, gic_frequency); if (ret < 0) pr_warn("GIC: Unable to register clocksource\n"); + + return ret; } void __init gic_clocksource_init(unsigned int frequency) @@ -179,31 +181,35 @@ static void __init gic_clocksource_of_init(struct device_node *node) struct clk *clk; int ret; - if (WARN_ON(!gic_present || !node->parent || - !of_device_is_compatible(node->parent, "mti,gic"))) - return; + if (!gic_present || !node->parent || + !of_device_is_compatible(node->parent, "mti,gic")) { + pr_warn("No DT definition for the mips gic driver"); + return -ENXIO; + } clk = of_clk_get(node, 0); if (!IS_ERR(clk)) { if (clk_prepare_enable(clk) < 0) { pr_err("GIC failed to enable clock\n"); clk_put(clk); - return; + return PTR_ERR(clk); } gic_frequency = clk_get_rate(clk); } else if (of_property_read_u32(node, "clock-frequency", &gic_frequency)) { pr_err("GIC frequency not specified.\n"); - return; + return -EINVAL;; } gic_timer_irq = irq_of_parse_and_map(node, 0); if (!gic_timer_irq) { pr_err("GIC timer IRQ not specified.\n"); - return; + return -EINVAL;; } - __gic_clocksource_init(); + ret = __gic_clocksource_init(); + if (ret) + return ret; ret = gic_clockevent_init(); if (!ret && !IS_ERR(clk)) { @@ -213,6 +219,8 @@ static void __init gic_clocksource_of_init(struct device_node *node) /* And finally start the counter */ gic_start_count(); + + return 0; } CLOCKSOURCE_OF_DECLARE(mips_gic_timer, "mti,gic-timer", gic_clocksource_of_init); diff --git a/drivers/clocksource/moxart_timer.c b/drivers/clocksource/moxart_timer.c index 19857af651c1..841454417acd 100644 --- a/drivers/clocksource/moxart_timer.c +++ b/drivers/clocksource/moxart_timer.c @@ -119,34 +119,45 @@ static struct irqaction moxart_timer_irq = { .dev_id = &moxart_clockevent, }; -static void __init moxart_timer_init(struct device_node *node) +static int __init moxart_timer_init(struct device_node *node) { int ret, irq; unsigned long pclk; struct clk *clk; base = of_iomap(node, 0); - if (!base) - panic("%s: of_iomap failed\n", node->full_name); + if (!base) { + pr_err("%s: of_iomap failed\n", node->full_name); + return -ENXIO; + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("%s: irq_of_parse_and_map failed\n", node->full_name); + if (irq <= 0) { + pr_err("%s: irq_of_parse_and_map failed\n", node->full_name); + return -EINVAL; + } ret = setup_irq(irq, &moxart_timer_irq); - if (ret) - panic("%s: setup_irq failed\n", node->full_name); + if (ret) { + pr_err("%s: setup_irq failed\n", node->full_name); + return ret; + } clk = of_clk_get(node, 0); - if (IS_ERR(clk)) - panic("%s: of_clk_get failed\n", node->full_name); + if (IS_ERR(clk)) { + pr_err("%s: of_clk_get failed\n", node->full_name); + return PTR_ERR(clk); + } pclk = clk_get_rate(clk); - if (clocksource_mmio_init(base + TIMER2_BASE + REG_COUNT, - "moxart_timer", pclk, 200, 32, - clocksource_mmio_readl_down)) - panic("%s: clocksource_mmio_init failed\n", node->full_name); + ret = clocksource_mmio_init(base + TIMER2_BASE + REG_COUNT, + "moxart_timer", pclk, 200, 32, + clocksource_mmio_readl_down); + if (ret) { + pr_err("%s: clocksource_mmio_init failed\n", node->full_name); + return ret; + } clock_count_per_tick = DIV_ROUND_CLOSEST(pclk, HZ); @@ -164,5 +175,7 @@ static void __init moxart_timer_init(struct device_node *node) */ clockevents_config_and_register(&moxart_clockevent, pclk, 0x4, 0xfffffffe); + + return 0; } CLOCKSOURCE_OF_DECLARE(moxart, "moxa,moxart-timer", moxart_timer_init); diff --git a/drivers/clocksource/mps2-timer.c b/drivers/clocksource/mps2-timer.c index 3d33a5e23dee..3e4431ed9aa9 100644 --- a/drivers/clocksource/mps2-timer.c +++ b/drivers/clocksource/mps2-timer.c @@ -250,7 +250,7 @@ out: return ret; } -static void __init mps2_timer_init(struct device_node *np) +static int __init mps2_timer_init(struct device_node *np) { static int has_clocksource, has_clockevent; int ret; @@ -259,7 +259,7 @@ static void __init mps2_timer_init(struct device_node *np) ret = mps2_clocksource_init(np); if (!ret) { has_clocksource = 1; - return; + return 0; } } @@ -267,9 +267,11 @@ static void __init mps2_timer_init(struct device_node *np) ret = mps2_clockevent_init(np); if (!ret) { has_clockevent = 1; - return; + return 0; } } + + return 0; } CLOCKSOURCE_OF_DECLARE(mps2_timer, "arm,mps2-timer", mps2_timer_init); diff --git a/drivers/clocksource/mtk_timer.c b/drivers/clocksource/mtk_timer.c index 7e583f8ea5f4..90659493c59c 100644 --- a/drivers/clocksource/mtk_timer.c +++ b/drivers/clocksource/mtk_timer.c @@ -181,7 +181,7 @@ static void mtk_timer_enable_irq(struct mtk_clock_event_device *evt, u8 timer) evt->gpt_base + GPT_IRQ_EN_REG); } -static void __init mtk_timer_init(struct device_node *node) +static int __init mtk_timer_init(struct device_node *node) { struct mtk_clock_event_device *evt; struct resource res; @@ -190,7 +190,7 @@ static void __init mtk_timer_init(struct device_node *node) evt = kzalloc(sizeof(*evt), GFP_KERNEL); if (!evt) - return; + return -ENOMEM; evt->dev.name = "mtk_tick"; evt->dev.rating = 300; @@ -248,7 +248,7 @@ static void __init mtk_timer_init(struct device_node *node) mtk_timer_enable_irq(evt, GPT_CLK_EVT); - return; + return 0; err_clk_disable: clk_disable_unprepare(clk); @@ -262,5 +262,7 @@ err_mem: release_mem_region(res.start, resource_size(&res)); err_kzalloc: kfree(evt); + + return -EINVAL; } CLOCKSOURCE_OF_DECLARE(mtk_mt6577, "mediatek,mt6577-timer", mtk_timer_init); diff --git a/drivers/clocksource/mxs_timer.c b/drivers/clocksource/mxs_timer.c index f5ce2961c0d6..0ba0a913b41d 100644 --- a/drivers/clocksource/mxs_timer.c +++ b/drivers/clocksource/mxs_timer.c @@ -31,8 +31,6 @@ #include #include -#include - /* * There are 2 versions of the timrot on Freescale MXS-based SoCs. * The v1 on MX23 only gets 16 bits counter, while v2 on MX28 @@ -226,10 +224,10 @@ static int __init mxs_clocksource_init(struct clk *timer_clk) return 0; } -static void __init mxs_timer_init(struct device_node *np) +static int __init mxs_timer_init(struct device_node *np) { struct clk *timer_clk; - int irq; + int irq, ret; mxs_timrot_base = of_iomap(np, 0); WARN_ON(!mxs_timrot_base); @@ -237,10 +235,12 @@ static void __init mxs_timer_init(struct device_node *np) timer_clk = of_clk_get(np, 0); if (IS_ERR(timer_clk)) { pr_err("%s: failed to get clk\n", __func__); - return; + return PTR_ERR(timer_clk); } - clk_prepare_enable(timer_clk); + ret = clk_prepare_enable(timer_clk); + if (ret) + return ret; /* * Initialize timers to a known state @@ -278,11 +278,19 @@ static void __init mxs_timer_init(struct device_node *np) mxs_timrot_base + HW_TIMROT_FIXED_COUNTn(1)); /* init and register the timer to the framework */ - mxs_clocksource_init(timer_clk); - mxs_clockevent_init(timer_clk); + ret = mxs_clocksource_init(timer_clk); + if (ret) + return ret; + + ret = mxs_clockevent_init(timer_clk); + if (ret) + return ret; /* Make irqs happen */ irq = irq_of_parse_and_map(np, 0); - setup_irq(irq, &mxs_timer_irq); + if (irq <= 0) + return -EINVAL; + + return setup_irq(irq, &mxs_timer_irq); } CLOCKSOURCE_OF_DECLARE(mxs, "fsl,timrot", mxs_timer_init); diff --git a/drivers/clocksource/nomadik-mtu.c b/drivers/clocksource/nomadik-mtu.c index bc8dd443c727..3c124d1ca600 100644 --- a/drivers/clocksource/nomadik-mtu.c +++ b/drivers/clocksource/nomadik-mtu.c @@ -193,10 +193,11 @@ static struct irqaction nmdk_timer_irq = { .dev_id = &nmdk_clkevt, }; -static void __init nmdk_timer_init(void __iomem *base, int irq, +static int __init nmdk_timer_init(void __iomem *base, int irq, struct clk *pclk, struct clk *clk) { unsigned long rate; + int ret; mtu_base = base; @@ -226,10 +227,12 @@ static void __init nmdk_timer_init(void __iomem *base, int irq, /* Timer 0 is the free running clocksource */ nmdk_clksrc_reset(); - if (clocksource_mmio_init(mtu_base + MTU_VAL(0), "mtu_0", - rate, 200, 32, clocksource_mmio_readl_down)) - pr_err("timer: failed to initialize clock source %s\n", - "mtu_0"); + ret = clocksource_mmio_init(mtu_base + MTU_VAL(0), "mtu_0", + rate, 200, 32, clocksource_mmio_readl_down); + if (ret) { + pr_err("timer: failed to initialize clock source %s\n", "mtu_0"); + return ret; + } #ifdef CONFIG_CLKSRC_NOMADIK_MTU_SCHED_CLOCK sched_clock_register(nomadik_read_sched_clock, 32, rate); @@ -244,9 +247,11 @@ static void __init nmdk_timer_init(void __iomem *base, int irq, mtu_delay_timer.read_current_timer = &nmdk_timer_read_current_timer; mtu_delay_timer.freq = rate; register_current_timer_delay(&mtu_delay_timer); + + return 0; } -static void __init nmdk_timer_of_init(struct device_node *node) +static int __init nmdk_timer_of_init(struct device_node *node) { struct clk *pclk; struct clk *clk; @@ -254,22 +259,30 @@ static void __init nmdk_timer_of_init(struct device_node *node) int irq; base = of_iomap(node, 0); - if (!base) - panic("Can't remap registers"); + if (!base) { + pr_err("Can't remap registers"); + return -ENXIO; + } pclk = of_clk_get_by_name(node, "apb_pclk"); - if (IS_ERR(pclk)) - panic("could not get apb_pclk"); + if (IS_ERR(pclk)) { + pr_err("could not get apb_pclk"); + return PTR_ERR(pclk); + } clk = of_clk_get_by_name(node, "timclk"); - if (IS_ERR(clk)) - panic("could not get timclk"); + if (IS_ERR(clk)) { + pr_err("could not get timclk"); + return PTR_ERR(clk); + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_err("Can't parse IRQ"); + return -EINVAL; + } - nmdk_timer_init(base, irq, pclk, clk); + return nmdk_timer_init(base, irq, pclk, clk); } CLOCKSOURCE_OF_DECLARE(nomadik_mtu, "st,nomadik-mtu", nmdk_timer_of_init); diff --git a/drivers/clocksource/pxa_timer.c b/drivers/clocksource/pxa_timer.c index 45b6a4999713..937e10b84d58 100644 --- a/drivers/clocksource/pxa_timer.c +++ b/drivers/clocksource/pxa_timer.c @@ -150,8 +150,10 @@ static struct irqaction pxa_ost0_irq = { .dev_id = &ckevt_pxa_osmr0, }; -static void __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) +static int __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) { + int ret; + timer_writel(0, OIER); timer_writel(OSSR_M0 | OSSR_M1 | OSSR_M2 | OSSR_M3, OSSR); @@ -159,39 +161,57 @@ static void __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) ckevt_pxa_osmr0.cpumask = cpumask_of(0); - setup_irq(irq, &pxa_ost0_irq); + ret = setup_irq(irq, &pxa_ost0_irq); + if (ret) { + pr_err("Failed to setup irq"); + return ret; + } + + ret = clocksource_mmio_init(timer_base + OSCR, "oscr0", clock_tick_rate, 200, + 32, clocksource_mmio_readl_up); + if (ret) { + pr_err("Failed to init clocksource"); + return ret; + } - clocksource_mmio_init(timer_base + OSCR, "oscr0", clock_tick_rate, 200, - 32, clocksource_mmio_readl_up); clockevents_config_and_register(&ckevt_pxa_osmr0, clock_tick_rate, MIN_OSCR_DELTA * 2, 0x7fffffff); + + return 0; } -static void __init pxa_timer_dt_init(struct device_node *np) +static int __init pxa_timer_dt_init(struct device_node *np) { struct clk *clk; - int irq; + int irq, ret; /* timer registers are shared with watchdog timer */ timer_base = of_iomap(np, 0); - if (!timer_base) - panic("%s: unable to map resource\n", np->name); + if (!timer_base) { + pr_err("%s: unable to map resource\n", np->name); + return -ENXIO; + } clk = of_clk_get(np, 0); if (IS_ERR(clk)) { pr_crit("%s: unable to get clk\n", np->name); - return; + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) { + pr_crit("Failed to prepare clock"); + return ret; } - clk_prepare_enable(clk); /* we are only interested in OS-timer0 irq */ irq = irq_of_parse_and_map(np, 0); if (irq <= 0) { pr_crit("%s: unable to parse OS-timer0 irq\n", np->name); - return; + return -EINVAL; } - pxa_timer_common_init(irq, clk_get_rate(clk)); + return pxa_timer_common_init(irq, clk_get_rate(clk)); } CLOCKSOURCE_OF_DECLARE(pxa_timer, "marvell,pxa-timer", pxa_timer_dt_init); diff --git a/drivers/clocksource/qcom-timer.c b/drivers/clocksource/qcom-timer.c index f8e09f923651..662576339049 100644 --- a/drivers/clocksource/qcom-timer.c +++ b/drivers/clocksource/qcom-timer.c @@ -178,7 +178,7 @@ static struct delay_timer msm_delay_timer = { .read_current_timer = msm_read_current_timer, }; -static void __init msm_timer_init(u32 dgt_hz, int sched_bits, int irq, +static int __init msm_timer_init(u32 dgt_hz, int sched_bits, int irq, bool percpu) { struct clocksource *cs = &msm_clocksource; @@ -218,12 +218,14 @@ err: sched_clock_register(msm_sched_clock_read, sched_bits, dgt_hz); msm_delay_timer.freq = dgt_hz; register_current_timer_delay(&msm_delay_timer); + + return res; } -static void __init msm_dt_timer_init(struct device_node *np) +static int __init msm_dt_timer_init(struct device_node *np) { u32 freq; - int irq; + int irq, ret; struct resource res; u32 percpu_offset; void __iomem *base; @@ -232,34 +234,35 @@ static void __init msm_dt_timer_init(struct device_node *np) base = of_iomap(np, 0); if (!base) { pr_err("Failed to map event base\n"); - return; + return -ENXIO; } /* We use GPT0 for the clockevent */ irq = irq_of_parse_and_map(np, 1); if (irq <= 0) { pr_err("Can't get irq\n"); - return; + return -EINVAL; } /* We use CPU0's DGT for the clocksource */ if (of_property_read_u32(np, "cpu-offset", &percpu_offset)) percpu_offset = 0; - if (of_address_to_resource(np, 0, &res)) { + ret = of_address_to_resource(np, 0, &res); + if (ret) { pr_err("Failed to parse DGT resource\n"); - return; + return ret; } cpu0_base = ioremap(res.start + percpu_offset, resource_size(&res)); if (!cpu0_base) { pr_err("Failed to map source base\n"); - return; + return -EINVAL; } if (of_property_read_u32(np, "clock-frequency", &freq)) { pr_err("Unknown frequency\n"); - return; + return -EINVAL; } event_base = base + 0x4; @@ -268,7 +271,7 @@ static void __init msm_dt_timer_init(struct device_node *np) freq /= 4; writel_relaxed(DGT_CLK_CTL_DIV_4, source_base + DGT_CLK_CTL); - msm_timer_init(freq, 32, irq, !!percpu_offset); + return msm_timer_init(freq, 32, irq, !!percpu_offset); } CLOCKSOURCE_OF_DECLARE(kpss_timer, "qcom,kpss-timer", msm_dt_timer_init); CLOCKSOURCE_OF_DECLARE(scss_timer, "qcom,scss-timer", msm_dt_timer_init); diff --git a/drivers/clocksource/rockchip_timer.c b/drivers/clocksource/rockchip_timer.c index b991b288c803..23e267acba25 100644 --- a/drivers/clocksource/rockchip_timer.c +++ b/drivers/clocksource/rockchip_timer.c @@ -19,7 +19,8 @@ #define TIMER_LOAD_COUNT0 0x00 #define TIMER_LOAD_COUNT1 0x04 -#define TIMER_CONTROL_REG 0x10 +#define TIMER_CONTROL_REG3288 0x10 +#define TIMER_CONTROL_REG3399 0x1c #define TIMER_INT_STATUS 0x18 #define TIMER_DISABLE 0x0 @@ -31,6 +32,7 @@ struct bc_timer { struct clock_event_device ce; void __iomem *base; + void __iomem *ctrl; u32 freq; }; @@ -46,15 +48,20 @@ static inline void __iomem *rk_base(struct clock_event_device *ce) return rk_timer(ce)->base; } +static inline void __iomem *rk_ctrl(struct clock_event_device *ce) +{ + return rk_timer(ce)->ctrl; +} + static inline void rk_timer_disable(struct clock_event_device *ce) { - writel_relaxed(TIMER_DISABLE, rk_base(ce) + TIMER_CONTROL_REG); + writel_relaxed(TIMER_DISABLE, rk_ctrl(ce)); } static inline void rk_timer_enable(struct clock_event_device *ce, u32 flags) { writel_relaxed(TIMER_ENABLE | TIMER_INT_UNMASK | flags, - rk_base(ce) + TIMER_CONTROL_REG); + rk_ctrl(ce)); } static void rk_timer_update_counter(unsigned long cycles, @@ -106,37 +113,42 @@ static irqreturn_t rk_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static void __init rk_timer_init(struct device_node *np) +static int __init rk_timer_init(struct device_node *np, u32 ctrl_reg) { struct clock_event_device *ce = &bc_timer.ce; struct clk *timer_clk; struct clk *pclk; - int ret, irq; + int ret = -EINVAL, irq; bc_timer.base = of_iomap(np, 0); if (!bc_timer.base) { pr_err("Failed to get base address for '%s'\n", TIMER_NAME); - return; + return -ENXIO; } + bc_timer.ctrl = bc_timer.base + ctrl_reg; pclk = of_clk_get_by_name(np, "pclk"); if (IS_ERR(pclk)) { + ret = PTR_ERR(pclk); pr_err("Failed to get pclk for '%s'\n", TIMER_NAME); goto out_unmap; } - if (clk_prepare_enable(pclk)) { + ret = clk_prepare_enable(pclk); + if (ret) { pr_err("Failed to enable pclk for '%s'\n", TIMER_NAME); goto out_unmap; } timer_clk = of_clk_get_by_name(np, "timer"); if (IS_ERR(timer_clk)) { + ret = PTR_ERR(timer_clk); pr_err("Failed to get timer clock for '%s'\n", TIMER_NAME); goto out_timer_clk; } - if (clk_prepare_enable(timer_clk)) { + ret = clk_prepare_enable(timer_clk); + if (ret) { pr_err("Failed to enable timer clock\n"); goto out_timer_clk; } @@ -145,17 +157,19 @@ static void __init rk_timer_init(struct device_node *np) irq = irq_of_parse_and_map(np, 0); if (!irq) { + ret = -EINVAL; pr_err("Failed to map interrupts for '%s'\n", TIMER_NAME); goto out_irq; } ce->name = TIMER_NAME; - ce->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; + ce->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_DYNIRQ; ce->set_next_event = rk_timer_set_next_event; ce->set_state_shutdown = rk_timer_shutdown; ce->set_state_periodic = rk_timer_set_periodic; ce->irq = irq; - ce->cpumask = cpumask_of(0); + ce->cpumask = cpu_possible_mask; ce->rating = 250; rk_timer_interrupt_clear(ce); @@ -169,7 +183,7 @@ static void __init rk_timer_init(struct device_node *np) clockevents_config_and_register(ce, bc_timer.freq, 1, UINT_MAX); - return; + return 0; out_irq: clk_disable_unprepare(timer_clk); @@ -177,6 +191,21 @@ out_timer_clk: clk_disable_unprepare(pclk); out_unmap: iounmap(bc_timer.base); + + return ret; +} + +static int __init rk3288_timer_init(struct device_node *np) +{ + return rk_timer_init(np, TIMER_CONTROL_REG3288); +} + +static int __init rk3399_timer_init(struct device_node *np) +{ + return rk_timer_init(np, TIMER_CONTROL_REG3399); } -CLOCKSOURCE_OF_DECLARE(rk_timer, "rockchip,rk3288-timer", rk_timer_init); +CLOCKSOURCE_OF_DECLARE(rk3288_timer, "rockchip,rk3288-timer", + rk3288_timer_init); +CLOCKSOURCE_OF_DECLARE(rk3399_timer, "rockchip,rk3399-timer", + rk3399_timer_init); diff --git a/drivers/clocksource/samsung_pwm_timer.c b/drivers/clocksource/samsung_pwm_timer.c index 9502bc4c3f6d..54565bd0093b 100644 --- a/drivers/clocksource/samsung_pwm_timer.c +++ b/drivers/clocksource/samsung_pwm_timer.c @@ -130,9 +130,9 @@ static void samsung_time_stop(unsigned int channel) spin_lock_irqsave(&samsung_pwm_lock, flags); - tcon = __raw_readl(pwm.base + REG_TCON); + tcon = readl_relaxed(pwm.base + REG_TCON); tcon &= ~TCON_START(channel); - __raw_writel(tcon, pwm.base + REG_TCON); + writel_relaxed(tcon, pwm.base + REG_TCON); spin_unlock_irqrestore(&samsung_pwm_lock, flags); } @@ -148,14 +148,14 @@ static void samsung_time_setup(unsigned int channel, unsigned long tcnt) spin_lock_irqsave(&samsung_pwm_lock, flags); - tcon = __raw_readl(pwm.base + REG_TCON); + tcon = readl_relaxed(pwm.base + REG_TCON); tcon &= ~(TCON_START(tcon_chan) | TCON_AUTORELOAD(tcon_chan)); tcon |= TCON_MANUALUPDATE(tcon_chan); - __raw_writel(tcnt, pwm.base + REG_TCNTB(channel)); - __raw_writel(tcnt, pwm.base + REG_TCMPB(channel)); - __raw_writel(tcon, pwm.base + REG_TCON); + writel_relaxed(tcnt, pwm.base + REG_TCNTB(channel)); + writel_relaxed(tcnt, pwm.base + REG_TCMPB(channel)); + writel_relaxed(tcon, pwm.base + REG_TCON); spin_unlock_irqrestore(&samsung_pwm_lock, flags); } @@ -170,7 +170,7 @@ static void samsung_time_start(unsigned int channel, bool periodic) spin_lock_irqsave(&samsung_pwm_lock, flags); - tcon = __raw_readl(pwm.base + REG_TCON); + tcon = readl_relaxed(pwm.base + REG_TCON); tcon &= ~TCON_MANUALUPDATE(channel); tcon |= TCON_START(channel); @@ -180,7 +180,7 @@ static void samsung_time_start(unsigned int channel, bool periodic) else tcon &= ~TCON_AUTORELOAD(channel); - __raw_writel(tcon, pwm.base + REG_TCON); + writel_relaxed(tcon, pwm.base + REG_TCON); spin_unlock_irqrestore(&samsung_pwm_lock, flags); } @@ -333,11 +333,10 @@ static u64 notrace samsung_read_sched_clock(void) return samsung_clocksource_read(NULL); } -static void __init samsung_clocksource_init(void) +static int __init samsung_clocksource_init(void) { unsigned long pclk; unsigned long clock_rate; - int ret; pclk = clk_get_rate(pwm.timerclk); @@ -358,9 +357,7 @@ static void __init samsung_clocksource_init(void) pwm.variant.bits, clock_rate); samsung_clocksource.mask = CLOCKSOURCE_MASK(pwm.variant.bits); - ret = clocksource_register_hz(&samsung_clocksource, clock_rate); - if (ret) - panic("samsung_clocksource_timer: can't register clocksource\n"); + return clocksource_register_hz(&samsung_clocksource, clock_rate); } static void __init samsung_timer_resources(void) @@ -380,26 +377,31 @@ static void __init samsung_timer_resources(void) /* * PWM master driver */ -static void __init _samsung_pwm_clocksource_init(void) +static int __init _samsung_pwm_clocksource_init(void) { u8 mask; int channel; mask = ~pwm.variant.output_mask & ((1 << SAMSUNG_PWM_NUM) - 1); channel = fls(mask) - 1; - if (channel < 0) - panic("failed to find PWM channel for clocksource"); + if (channel < 0) { + pr_crit("failed to find PWM channel for clocksource"); + return -EINVAL; + } pwm.source_id = channel; mask &= ~(1 << channel); channel = fls(mask) - 1; - if (channel < 0) - panic("failed to find PWM channel for clock event"); + if (channel < 0) { + pr_crit("failed to find PWM channel for clock event"); + return -EINVAL; + } pwm.event_id = channel; samsung_timer_resources(); samsung_clockevent_init(); - samsung_clocksource_init(); + + return samsung_clocksource_init(); } void __init samsung_pwm_clocksource_init(void __iomem *base, @@ -417,8 +419,8 @@ void __init samsung_pwm_clocksource_init(void __iomem *base, } #ifdef CONFIG_CLKSRC_OF -static void __init samsung_pwm_alloc(struct device_node *np, - const struct samsung_pwm_variant *variant) +static int __init samsung_pwm_alloc(struct device_node *np, + const struct samsung_pwm_variant *variant) { struct property *prop; const __be32 *cur; @@ -441,14 +443,16 @@ static void __init samsung_pwm_alloc(struct device_node *np, pwm.base = of_iomap(np, 0); if (!pwm.base) { pr_err("%s: failed to map PWM registers\n", __func__); - return; + return -ENXIO; } pwm.timerclk = of_clk_get_by_name(np, "timers"); - if (IS_ERR(pwm.timerclk)) - panic("failed to get timers clock for timer"); + if (IS_ERR(pwm.timerclk)) { + pr_crit("failed to get timers clock for timer"); + return PTR_ERR(pwm.timerclk); + } - _samsung_pwm_clocksource_init(); + return _samsung_pwm_clocksource_init(); } static const struct samsung_pwm_variant s3c24xx_variant = { @@ -458,9 +462,9 @@ static const struct samsung_pwm_variant s3c24xx_variant = { .tclk_mask = (1 << 4), }; -static void __init s3c2410_pwm_clocksource_init(struct device_node *np) +static int __init s3c2410_pwm_clocksource_init(struct device_node *np) { - samsung_pwm_alloc(np, &s3c24xx_variant); + return samsung_pwm_alloc(np, &s3c24xx_variant); } CLOCKSOURCE_OF_DECLARE(s3c2410_pwm, "samsung,s3c2410-pwm", s3c2410_pwm_clocksource_init); @@ -471,9 +475,9 @@ static const struct samsung_pwm_variant s3c64xx_variant = { .tclk_mask = (1 << 7) | (1 << 6) | (1 << 5), }; -static void __init s3c64xx_pwm_clocksource_init(struct device_node *np) +static int __init s3c64xx_pwm_clocksource_init(struct device_node *np) { - samsung_pwm_alloc(np, &s3c64xx_variant); + return samsung_pwm_alloc(np, &s3c64xx_variant); } CLOCKSOURCE_OF_DECLARE(s3c6400_pwm, "samsung,s3c6400-pwm", s3c64xx_pwm_clocksource_init); @@ -484,9 +488,9 @@ static const struct samsung_pwm_variant s5p64x0_variant = { .tclk_mask = 0, }; -static void __init s5p64x0_pwm_clocksource_init(struct device_node *np) +static int __init s5p64x0_pwm_clocksource_init(struct device_node *np) { - samsung_pwm_alloc(np, &s5p64x0_variant); + return samsung_pwm_alloc(np, &s5p64x0_variant); } CLOCKSOURCE_OF_DECLARE(s5p6440_pwm, "samsung,s5p6440-pwm", s5p64x0_pwm_clocksource_init); @@ -497,9 +501,9 @@ static const struct samsung_pwm_variant s5p_variant = { .tclk_mask = (1 << 5), }; -static void __init s5p_pwm_clocksource_init(struct device_node *np) +static int __init s5p_pwm_clocksource_init(struct device_node *np) { - samsung_pwm_alloc(np, &s5p_variant); + return samsung_pwm_alloc(np, &s5p_variant); } CLOCKSOURCE_OF_DECLARE(s5pc100_pwm, "samsung,s5pc100-pwm", s5p_pwm_clocksource_init); #endif diff --git a/drivers/clocksource/sun4i_timer.c b/drivers/clocksource/sun4i_timer.c index 6f3719d73390..97669ee4df2a 100644 --- a/drivers/clocksource/sun4i_timer.c +++ b/drivers/clocksource/sun4i_timer.c @@ -146,7 +146,7 @@ static u64 notrace sun4i_timer_sched_read(void) return ~readl(timer_base + TIMER_CNTVAL_REG(1)); } -static void __init sun4i_timer_init(struct device_node *node) +static int __init sun4i_timer_init(struct device_node *node) { unsigned long rate = 0; struct clk *clk; @@ -154,17 +154,28 @@ static void __init sun4i_timer_init(struct device_node *node) u32 val; timer_base = of_iomap(node, 0); - if (!timer_base) - panic("Can't map registers"); + if (!timer_base) { + pr_crit("Can't map registers"); + return -ENXIO; + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_crit("Can't parse IRQ"); + return -EINVAL; + } clk = of_clk_get(node, 0); - if (IS_ERR(clk)) - panic("Can't get timer clock"); - clk_prepare_enable(clk); + if (IS_ERR(clk)) { + pr_crit("Can't get timer clock"); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) { + pr_err("Failed to prepare clock"); + return ret; + } rate = clk_get_rate(clk); @@ -182,8 +193,12 @@ static void __init sun4i_timer_init(struct device_node *node) of_machine_is_compatible("allwinner,sun5i-a10s")) sched_clock_register(sun4i_timer_sched_read, 32, rate); - clocksource_mmio_init(timer_base + TIMER_CNTVAL_REG(1), node->name, - rate, 350, 32, clocksource_mmio_readl_down); + ret = clocksource_mmio_init(timer_base + TIMER_CNTVAL_REG(1), node->name, + rate, 350, 32, clocksource_mmio_readl_down); + if (ret) { + pr_err("Failed to register clocksource"); + return ret; + } ticks_per_jiffy = DIV_ROUND_UP(rate, HZ); @@ -200,12 +215,16 @@ static void __init sun4i_timer_init(struct device_node *node) TIMER_SYNC_TICKS, 0xffffffff); ret = setup_irq(irq, &sun4i_timer_irq); - if (ret) - pr_warn("failed to setup irq %d\n", irq); + if (ret) { + pr_err("failed to setup irq %d\n", irq); + return ret; + } /* Enable timer0 interrupt */ val = readl(timer_base + TIMER_IRQ_EN_REG); writel(val | TIMER_IRQ_EN(0), timer_base + TIMER_IRQ_EN_REG); + + return ret; } CLOCKSOURCE_OF_DECLARE(sun4i, "allwinner,sun4i-a10-timer", sun4i_timer_init); diff --git a/drivers/clocksource/tango_xtal.c b/drivers/clocksource/tango_xtal.c index c407c47a3232..12fcef8cf2d3 100644 --- a/drivers/clocksource/tango_xtal.c +++ b/drivers/clocksource/tango_xtal.c @@ -19,7 +19,7 @@ static u64 notrace read_sched_clock(void) return read_xtal_counter(); } -static void __init tango_clocksource_init(struct device_node *np) +static int __init tango_clocksource_init(struct device_node *np) { struct clk *clk; int xtal_freq, ret; @@ -27,13 +27,13 @@ static void __init tango_clocksource_init(struct device_node *np) xtal_in_cnt = of_iomap(np, 0); if (xtal_in_cnt == NULL) { pr_err("%s: invalid address\n", np->full_name); - return; + return -ENXIO; } clk = of_clk_get(np, 0); if (IS_ERR(clk)) { pr_err("%s: invalid clock\n", np->full_name); - return; + return PTR_ERR(clk); } xtal_freq = clk_get_rate(clk); @@ -44,11 +44,13 @@ static void __init tango_clocksource_init(struct device_node *np) 32, clocksource_mmio_readl_up); if (ret) { pr_err("%s: registration failed\n", np->full_name); - return; + return ret; } sched_clock_register(read_sched_clock, 32, xtal_freq); register_current_timer_delay(&delay_timer); + + return 0; } CLOCKSOURCE_OF_DECLARE(tango, "sigma,tick-counter", tango_clocksource_init); diff --git a/drivers/clocksource/tegra20_timer.c b/drivers/clocksource/tegra20_timer.c index 7b94ad2ab278..f960891aa04e 100644 --- a/drivers/clocksource/tegra20_timer.c +++ b/drivers/clocksource/tegra20_timer.c @@ -165,7 +165,7 @@ static struct irqaction tegra_timer_irq = { .dev_id = &tegra_clockevent, }; -static void __init tegra20_init_timer(struct device_node *np) +static int __init tegra20_init_timer(struct device_node *np) { struct clk *clk; unsigned long rate; @@ -174,13 +174,13 @@ static void __init tegra20_init_timer(struct device_node *np) timer_reg_base = of_iomap(np, 0); if (!timer_reg_base) { pr_err("Can't map timer registers\n"); - BUG(); + return -ENXIO; } tegra_timer_irq.irq = irq_of_parse_and_map(np, 2); if (tegra_timer_irq.irq <= 0) { pr_err("Failed to map timer IRQ\n"); - BUG(); + return -EINVAL; } clk = of_clk_get(np, 0); @@ -211,10 +211,12 @@ static void __init tegra20_init_timer(struct device_node *np) sched_clock_register(tegra_read_sched_clock, 32, 1000000); - if (clocksource_mmio_init(timer_reg_base + TIMERUS_CNTR_1US, - "timer_us", 1000000, 300, 32, clocksource_mmio_readl_up)) { + ret = clocksource_mmio_init(timer_reg_base + TIMERUS_CNTR_1US, + "timer_us", 1000000, 300, 32, + clocksource_mmio_readl_up); + if (ret) { pr_err("Failed to register clocksource\n"); - BUG(); + return ret; } tegra_delay_timer.read_current_timer = @@ -225,24 +227,26 @@ static void __init tegra20_init_timer(struct device_node *np) ret = setup_irq(tegra_timer_irq.irq, &tegra_timer_irq); if (ret) { pr_err("Failed to register timer IRQ: %d\n", ret); - BUG(); + return ret; } tegra_clockevent.cpumask = cpu_all_mask; tegra_clockevent.irq = tegra_timer_irq.irq; clockevents_config_and_register(&tegra_clockevent, 1000000, 0x1, 0x1fffffff); + + return 0; } CLOCKSOURCE_OF_DECLARE(tegra20_timer, "nvidia,tegra20-timer", tegra20_init_timer); -static void __init tegra20_init_rtc(struct device_node *np) +static int __init tegra20_init_rtc(struct device_node *np) { struct clk *clk; rtc_base = of_iomap(np, 0); if (!rtc_base) { pr_err("Can't map RTC registers"); - BUG(); + return -ENXIO; } /* @@ -255,6 +259,6 @@ static void __init tegra20_init_rtc(struct device_node *np) else clk_prepare_enable(clk); - register_persistent_clock(NULL, tegra_read_persistent_clock64); + return register_persistent_clock(NULL, tegra_read_persistent_clock64); } CLOCKSOURCE_OF_DECLARE(tegra20_rtc, "nvidia,tegra20-rtc", tegra20_init_rtc); diff --git a/drivers/clocksource/time-armada-370-xp.c b/drivers/clocksource/time-armada-370-xp.c index d93ec3c4f139..20ec066481fe 100644 --- a/drivers/clocksource/time-armada-370-xp.c +++ b/drivers/clocksource/time-armada-370-xp.c @@ -246,7 +246,7 @@ static void armada_370_xp_timer_resume(void) writel(timer0_local_ctrl_reg, local_base + TIMER_CTRL_OFF); } -struct syscore_ops armada_370_xp_timer_syscore_ops = { +static struct syscore_ops armada_370_xp_timer_syscore_ops = { .suspend = armada_370_xp_timer_suspend, .resume = armada_370_xp_timer_resume, }; @@ -260,14 +260,22 @@ static struct delay_timer armada_370_delay_timer = { .read_current_timer = armada_370_delay_timer_read, }; -static void __init armada_370_xp_timer_common_init(struct device_node *np) +static int __init armada_370_xp_timer_common_init(struct device_node *np) { u32 clr = 0, set = 0; int res; timer_base = of_iomap(np, 0); - WARN_ON(!timer_base); + if (!timer_base) { + pr_err("Failed to iomap"); + return -ENXIO; + } + local_base = of_iomap(np, 1); + if (!local_base) { + pr_err("Failed to iomap"); + return -ENXIO; + } if (timer25Mhz) { set = TIMER0_25MHZ; @@ -306,14 +314,19 @@ static void __init armada_370_xp_timer_common_init(struct device_node *np) */ sched_clock_register(armada_370_xp_read_sched_clock, 32, timer_clk); - clocksource_mmio_init(timer_base + TIMER0_VAL_OFF, - "armada_370_xp_clocksource", - timer_clk, 300, 32, clocksource_mmio_readl_down); + res = clocksource_mmio_init(timer_base + TIMER0_VAL_OFF, + "armada_370_xp_clocksource", + timer_clk, 300, 32, clocksource_mmio_readl_down); + if (res) { + pr_err("Failed to initialize clocksource mmio"); + return res; + } register_cpu_notifier(&armada_370_xp_timer_cpu_nb); armada_370_xp_evt = alloc_percpu(struct clock_event_device); - + if (!armada_370_xp_evt) + return -ENOMEM; /* * Setup clockevent timer (interrupt-driven). @@ -323,33 +336,54 @@ static void __init armada_370_xp_timer_common_init(struct device_node *np) "armada_370_xp_per_cpu_tick", armada_370_xp_evt); /* Immediately configure the timer on the boot CPU */ - if (!res) - armada_370_xp_timer_setup(this_cpu_ptr(armada_370_xp_evt)); + if (res) { + pr_err("Failed to request percpu irq"); + return res; + } + + res = armada_370_xp_timer_setup(this_cpu_ptr(armada_370_xp_evt)); + if (res) { + pr_err("Failed to setup timer"); + return res; + } register_syscore_ops(&armada_370_xp_timer_syscore_ops); + + return 0; } -static void __init armada_xp_timer_init(struct device_node *np) +static int __init armada_xp_timer_init(struct device_node *np) { struct clk *clk = of_clk_get_by_name(np, "fixed"); + int ret; + + clk = of_clk_get(np, 0); + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) + return ret; - /* The 25Mhz fixed clock is mandatory, and must always be available */ - BUG_ON(IS_ERR(clk)); - clk_prepare_enable(clk); timer_clk = clk_get_rate(clk); - armada_370_xp_timer_common_init(np); + return armada_370_xp_timer_common_init(np); } CLOCKSOURCE_OF_DECLARE(armada_xp, "marvell,armada-xp-timer", armada_xp_timer_init); -static void __init armada_375_timer_init(struct device_node *np) +static int __init armada_375_timer_init(struct device_node *np) { struct clk *clk; + int ret; clk = of_clk_get_by_name(np, "fixed"); if (!IS_ERR(clk)) { - clk_prepare_enable(clk); + ret = clk_prepare_enable(clk); + if (ret) + return ret; timer_clk = clk_get_rate(clk); } else { @@ -360,27 +394,43 @@ static void __init armada_375_timer_init(struct device_node *np) clk = of_clk_get(np, 0); /* Must have at least a clock */ - BUG_ON(IS_ERR(clk)); - clk_prepare_enable(clk); + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) + return ret; + timer_clk = clk_get_rate(clk) / TIMER_DIVIDER; timer25Mhz = false; } - armada_370_xp_timer_common_init(np); + return armada_370_xp_timer_common_init(np); } CLOCKSOURCE_OF_DECLARE(armada_375, "marvell,armada-375-timer", armada_375_timer_init); -static void __init armada_370_timer_init(struct device_node *np) +static int __init armada_370_timer_init(struct device_node *np) { - struct clk *clk = of_clk_get(np, 0); + struct clk *clk; + int ret; + + clk = of_clk_get(np, 0); + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) + return ret; - BUG_ON(IS_ERR(clk)); - clk_prepare_enable(clk); timer_clk = clk_get_rate(clk) / TIMER_DIVIDER; timer25Mhz = false; - armada_370_xp_timer_common_init(np); + return armada_370_xp_timer_common_init(np); } CLOCKSOURCE_OF_DECLARE(armada_370, "marvell,armada-370-timer", armada_370_timer_init); diff --git a/drivers/clocksource/time-efm32.c b/drivers/clocksource/time-efm32.c index b06e4c2be406..5ac344b383e1 100644 --- a/drivers/clocksource/time-efm32.c +++ b/drivers/clocksource/time-efm32.c @@ -233,10 +233,15 @@ static int __init efm32_clockevent_init(struct device_node *np) DIV_ROUND_CLOSEST(rate, 1024), 0xf, 0xffff); - setup_irq(irq, &efm32_clock_event_irq); + ret = setup_irq(irq, &efm32_clock_event_irq); + if (ret) { + pr_err("Failed setup irq"); + goto err_setup_irq; + } return 0; +err_setup_irq: err_get_irq: iounmap(base); @@ -255,16 +260,16 @@ err_clk_get: * This function asserts that we have exactly one clocksource and one * clock_event_device in the end. */ -static void __init efm32_timer_init(struct device_node *np) +static int __init efm32_timer_init(struct device_node *np) { static int has_clocksource, has_clockevent; - int ret; + int ret = 0; if (!has_clocksource) { ret = efm32_clocksource_init(np); if (!ret) { has_clocksource = 1; - return; + return 0; } } @@ -272,9 +277,11 @@ static void __init efm32_timer_init(struct device_node *np) ret = efm32_clockevent_init(np); if (!ret) { has_clockevent = 1; - return; + return 0; } } + + return ret; } CLOCKSOURCE_OF_DECLARE(efm32compat, "efm32,timer", efm32_timer_init); CLOCKSOURCE_OF_DECLARE(efm32, "energymicro,efm32-timer", efm32_timer_init); diff --git a/drivers/clocksource/time-lpc32xx.c b/drivers/clocksource/time-lpc32xx.c index daae61e8c820..9649cfdb9213 100644 --- a/drivers/clocksource/time-lpc32xx.c +++ b/drivers/clocksource/time-lpc32xx.c @@ -288,16 +288,16 @@ err_clk_enable: * This function asserts that we have exactly one clocksource and one * clock_event_device in the end. */ -static void __init lpc32xx_timer_init(struct device_node *np) +static int __init lpc32xx_timer_init(struct device_node *np) { static int has_clocksource, has_clockevent; - int ret; + int ret = 0; if (!has_clocksource) { ret = lpc32xx_clocksource_init(np); if (!ret) { has_clocksource = 1; - return; + return 0; } } @@ -305,8 +305,10 @@ static void __init lpc32xx_timer_init(struct device_node *np) ret = lpc32xx_clockevent_init(np); if (!ret) { has_clockevent = 1; - return; + return 0; } } + + return ret; } CLOCKSOURCE_OF_DECLARE(lpc32xx_timer, "nxp,lpc3220-timer", lpc32xx_timer_init); diff --git a/drivers/clocksource/time-orion.c b/drivers/clocksource/time-orion.c index 0ece7427b497..a28f496e97cf 100644 --- a/drivers/clocksource/time-orion.c +++ b/drivers/clocksource/time-orion.c @@ -104,25 +104,36 @@ static struct irqaction orion_clkevt_irq = { .handler = orion_clkevt_irq_handler, }; -static void __init orion_timer_init(struct device_node *np) +static int __init orion_timer_init(struct device_node *np) { struct clk *clk; - int irq; + int irq, ret; /* timer registers are shared with watchdog timer */ timer_base = of_iomap(np, 0); - if (!timer_base) - panic("%s: unable to map resource\n", np->name); + if (!timer_base) { + pr_err("%s: unable to map resource\n", np->name); + return -ENXIO; + } clk = of_clk_get(np, 0); - if (IS_ERR(clk)) - panic("%s: unable to get clk\n", np->name); - clk_prepare_enable(clk); + if (IS_ERR(clk)) { + pr_err("%s: unable to get clk\n", np->name); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) { + pr_err("Failed to prepare clock"); + return ret; + } /* we are only interested in timer1 irq */ irq = irq_of_parse_and_map(np, 1); - if (irq <= 0) - panic("%s: unable to parse timer1 irq\n", np->name); + if (irq <= 0) { + pr_err("%s: unable to parse timer1 irq\n", np->name); + return -EINVAL; + } /* setup timer0 as free-running clocksource */ writel(~0, timer_base + TIMER0_VAL); @@ -130,19 +141,30 @@ static void __init orion_timer_init(struct device_node *np) atomic_io_modify(timer_base + TIMER_CTRL, TIMER0_RELOAD_EN | TIMER0_EN, TIMER0_RELOAD_EN | TIMER0_EN); - clocksource_mmio_init(timer_base + TIMER0_VAL, "orion_clocksource", - clk_get_rate(clk), 300, 32, - clocksource_mmio_readl_down); + + ret = clocksource_mmio_init(timer_base + TIMER0_VAL, "orion_clocksource", + clk_get_rate(clk), 300, 32, + clocksource_mmio_readl_down); + if (ret) { + pr_err("Failed to initialize mmio timer"); + return ret; + } + sched_clock_register(orion_read_sched_clock, 32, clk_get_rate(clk)); /* setup timer1 as clockevent timer */ - if (setup_irq(irq, &orion_clkevt_irq)) - panic("%s: unable to setup irq\n", np->name); + ret = setup_irq(irq, &orion_clkevt_irq); + if (ret) { + pr_err("%s: unable to setup irq\n", np->name); + return ret; + } ticks_per_jiffy = (clk_get_rate(clk) + HZ/2) / HZ; orion_clkevt.cpumask = cpumask_of(0); orion_clkevt.irq = irq; clockevents_config_and_register(&orion_clkevt, clk_get_rate(clk), ORION_ONESHOT_MIN, ORION_ONESHOT_MAX); + + return 0; } CLOCKSOURCE_OF_DECLARE(orion_timer, "marvell,orion-timer", orion_timer_init); diff --git a/drivers/clocksource/time-pistachio.c b/drivers/clocksource/time-pistachio.c index 376e59bc5fa0..a7d9a08e4b0e 100644 --- a/drivers/clocksource/time-pistachio.c +++ b/drivers/clocksource/time-pistachio.c @@ -148,7 +148,7 @@ static struct pistachio_clocksource pcs_gpt = { }, }; -static void __init pistachio_clksrc_of_init(struct device_node *node) +static int __init pistachio_clksrc_of_init(struct device_node *node) { struct clk *sys_clk, *fast_clk; struct regmap *periph_regs; @@ -158,45 +158,45 @@ static void __init pistachio_clksrc_of_init(struct device_node *node) pcs_gpt.base = of_iomap(node, 0); if (!pcs_gpt.base) { pr_err("cannot iomap\n"); - return; + return -ENXIO; } periph_regs = syscon_regmap_lookup_by_phandle(node, "img,cr-periph"); if (IS_ERR(periph_regs)) { pr_err("cannot get peripheral regmap (%ld)\n", PTR_ERR(periph_regs)); - return; + return PTR_ERR(periph_regs); } /* Switch to using the fast counter clock */ ret = regmap_update_bits(periph_regs, PERIP_TIMER_CONTROL, 0xf, 0x0); if (ret) - return; + return ret; sys_clk = of_clk_get_by_name(node, "sys"); if (IS_ERR(sys_clk)) { pr_err("clock get failed (%ld)\n", PTR_ERR(sys_clk)); - return; + return PTR_ERR(sys_clk); } fast_clk = of_clk_get_by_name(node, "fast"); if (IS_ERR(fast_clk)) { pr_err("clock get failed (%lu)\n", PTR_ERR(fast_clk)); - return; + return PTR_ERR(fast_clk); } ret = clk_prepare_enable(sys_clk); if (ret < 0) { pr_err("failed to enable clock (%d)\n", ret); - return; + return ret; } ret = clk_prepare_enable(fast_clk); if (ret < 0) { pr_err("failed to enable clock (%d)\n", ret); clk_disable_unprepare(sys_clk); - return; + return ret; } rate = clk_get_rate(fast_clk); @@ -212,7 +212,7 @@ static void __init pistachio_clksrc_of_init(struct device_node *node) raw_spin_lock_init(&pcs_gpt.lock); sched_clock_register(pistachio_read_sched_clock, 32, rate); - clocksource_register_hz(&pcs_gpt.cs, rate); + return clocksource_register_hz(&pcs_gpt.cs, rate); } CLOCKSOURCE_OF_DECLARE(pistachio_gptimer, "img,pistachio-gptimer", pistachio_clksrc_of_init); diff --git a/drivers/clocksource/timer-atlas7.c b/drivers/clocksource/timer-atlas7.c index 27fa13680be1..90f8fbc154a4 100644 --- a/drivers/clocksource/timer-atlas7.c +++ b/drivers/clocksource/timer-atlas7.c @@ -238,7 +238,7 @@ static struct notifier_block sirfsoc_cpu_nb = { .notifier_call = sirfsoc_cpu_notify, }; -static void __init sirfsoc_clockevent_init(void) +static int __init sirfsoc_clockevent_init(void) { sirfsoc_clockevent = alloc_percpu(struct clock_event_device); BUG_ON(!sirfsoc_clockevent); @@ -246,11 +246,11 @@ static void __init sirfsoc_clockevent_init(void) BUG_ON(register_cpu_notifier(&sirfsoc_cpu_nb)); /* Immediately configure the timer on the boot CPU */ - sirfsoc_local_timer_setup(this_cpu_ptr(sirfsoc_clockevent)); + return sirfsoc_local_timer_setup(this_cpu_ptr(sirfsoc_clockevent)); } /* initialize the kernel jiffy timer source */ -static void __init sirfsoc_atlas7_timer_init(struct device_node *np) +static int __init sirfsoc_atlas7_timer_init(struct device_node *np) { struct clk *clk; @@ -279,23 +279,29 @@ static void __init sirfsoc_atlas7_timer_init(struct device_node *np) BUG_ON(clocksource_register_hz(&sirfsoc_clocksource, atlas7_timer_rate)); - sirfsoc_clockevent_init(); + return sirfsoc_clockevent_init(); } -static void __init sirfsoc_of_timer_init(struct device_node *np) +static int __init sirfsoc_of_timer_init(struct device_node *np) { sirfsoc_timer_base = of_iomap(np, 0); - if (!sirfsoc_timer_base) - panic("unable to map timer cpu registers\n"); + if (!sirfsoc_timer_base) { + pr_err("unable to map timer cpu registers\n"); + return -ENXIO; + } sirfsoc_timer_irq.irq = irq_of_parse_and_map(np, 0); - if (!sirfsoc_timer_irq.irq) - panic("No irq passed for timer0 via DT\n"); + if (!sirfsoc_timer_irq.irq) { + pr_err("No irq passed for timer0 via DT\n"); + return -EINVAL; + } sirfsoc_timer1_irq.irq = irq_of_parse_and_map(np, 1); - if (!sirfsoc_timer1_irq.irq) - panic("No irq passed for timer1 via DT\n"); + if (!sirfsoc_timer1_irq.irq) { + pr_err("No irq passed for timer1 via DT\n"); + return -EINVAL; + } - sirfsoc_atlas7_timer_init(np); + return sirfsoc_atlas7_timer_init(np); } CLOCKSOURCE_OF_DECLARE(sirfsoc_atlas7_timer, "sirf,atlas7-tick", sirfsoc_of_timer_init); diff --git a/drivers/clocksource/timer-atmel-pit.c b/drivers/clocksource/timer-atmel-pit.c index d911c5dca8f1..1ffac0cb0cb7 100644 --- a/drivers/clocksource/timer-atmel-pit.c +++ b/drivers/clocksource/timer-atmel-pit.c @@ -177,7 +177,7 @@ static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id) /* * Set up both clocksource and clockevent support. */ -static void __init at91sam926x_pit_common_init(struct pit_data *data) +static int __init at91sam926x_pit_common_init(struct pit_data *data) { unsigned long pit_rate; unsigned bits; @@ -204,14 +204,21 @@ static void __init at91sam926x_pit_common_init(struct pit_data *data) data->clksrc.rating = 175; data->clksrc.read = read_pit_clk; data->clksrc.flags = CLOCK_SOURCE_IS_CONTINUOUS; - clocksource_register_hz(&data->clksrc, pit_rate); + + ret = clocksource_register_hz(&data->clksrc, pit_rate); + if (ret) { + pr_err("Failed to register clocksource"); + return ret; + } /* Set up irq handler */ ret = request_irq(data->irq, at91sam926x_pit_interrupt, IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL, "at91_tick", data); - if (ret) - panic(pr_fmt("Unable to setup IRQ\n")); + if (ret) { + pr_err("Unable to setup IRQ\n"); + return ret; + } /* Set up and register clockevents */ data->clkevt.name = "pit"; @@ -226,34 +233,42 @@ static void __init at91sam926x_pit_common_init(struct pit_data *data) data->clkevt.resume = at91sam926x_pit_resume; data->clkevt.suspend = at91sam926x_pit_suspend; clockevents_register_device(&data->clkevt); + + return 0; } -static void __init at91sam926x_pit_dt_init(struct device_node *node) +static int __init at91sam926x_pit_dt_init(struct device_node *node) { struct pit_data *data; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) - panic(pr_fmt("Unable to allocate memory\n")); + return -ENOMEM; data->base = of_iomap(node, 0); - if (!data->base) - panic(pr_fmt("Could not map PIT address\n")); + if (!data->base) { + pr_err("Could not map PIT address\n"); + return -ENXIO; + } data->mck = of_clk_get(node, 0); if (IS_ERR(data->mck)) /* Fallback on clkdev for !CCF-based boards */ data->mck = clk_get(NULL, "mck"); - if (IS_ERR(data->mck)) - panic(pr_fmt("Unable to get mck clk\n")); + if (IS_ERR(data->mck)) { + pr_err("Unable to get mck clk\n"); + return PTR_ERR(data->mck); + } /* Get the interrupts property */ data->irq = irq_of_parse_and_map(node, 0); - if (!data->irq) - panic(pr_fmt("Unable to get IRQ from DT\n")); + if (!data->irq) { + pr_err("Unable to get IRQ from DT\n"); + return -EINVAL; + } - at91sam926x_pit_common_init(data); + return at91sam926x_pit_common_init(data); } CLOCKSOURCE_OF_DECLARE(at91sam926x_pit, "atmel,at91sam9260-pit", at91sam926x_pit_dt_init); diff --git a/drivers/clocksource/timer-atmel-st.c b/drivers/clocksource/timer-atmel-st.c index 29d21d68df5a..e90ab5b63a90 100644 --- a/drivers/clocksource/timer-atmel-st.c +++ b/drivers/clocksource/timer-atmel-st.c @@ -194,15 +194,17 @@ static struct clock_event_device clkevt = { /* * ST (system timer) module supports both clockevents and clocksource. */ -static void __init atmel_st_timer_init(struct device_node *node) +static int __init atmel_st_timer_init(struct device_node *node) { struct clk *sclk; unsigned int sclk_rate, val; int irq, ret; regmap_st = syscon_node_to_regmap(node); - if (IS_ERR(regmap_st)) - panic(pr_fmt("Unable to get regmap\n")); + if (IS_ERR(regmap_st)) { + pr_err("Unable to get regmap\n"); + return PTR_ERR(regmap_st); + } /* Disable all timer interrupts, and clear any pending ones */ regmap_write(regmap_st, AT91_ST_IDR, @@ -211,27 +213,37 @@ static void __init atmel_st_timer_init(struct device_node *node) /* Get the interrupts property */ irq = irq_of_parse_and_map(node, 0); - if (!irq) - panic(pr_fmt("Unable to get IRQ from DT\n")); + if (!irq) { + pr_err("Unable to get IRQ from DT\n"); + return -EINVAL; + } /* Make IRQs happen for the system timer */ ret = request_irq(irq, at91rm9200_timer_interrupt, IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL, "at91_tick", regmap_st); - if (ret) - panic(pr_fmt("Unable to setup IRQ\n")); + if (ret) { + pr_err("Unable to setup IRQ\n"); + return ret; + } sclk = of_clk_get(node, 0); - if (IS_ERR(sclk)) - panic(pr_fmt("Unable to get slow clock\n")); + if (IS_ERR(sclk)) { + pr_err("Unable to get slow clock\n"); + return PTR_ERR(sclk); + } - clk_prepare_enable(sclk); - if (ret) - panic(pr_fmt("Could not enable slow clock\n")); + ret = clk_prepare_enable(sclk); + if (ret) { + pr_err("Could not enable slow clock\n"); + return ret; + } sclk_rate = clk_get_rate(sclk); - if (!sclk_rate) - panic(pr_fmt("Invalid slow clock rate\n")); + if (!sclk_rate) { + pr_err("Invalid slow clock rate\n"); + return -EINVAL; + } timer_latch = (sclk_rate + HZ / 2) / HZ; /* The 32KiHz "Slow Clock" (tick every 30517.58 nanoseconds) is used @@ -246,7 +258,7 @@ static void __init atmel_st_timer_init(struct device_node *node) 2, AT91_ST_ALMV); /* register clocksource */ - clocksource_register_hz(&clk32k, sclk_rate); + return clocksource_register_hz(&clk32k, sclk_rate); } CLOCKSOURCE_OF_DECLARE(atmel_st_timer, "atmel,at91rm9200-st", atmel_st_timer_init); diff --git a/drivers/clocksource/timer-digicolor.c b/drivers/clocksource/timer-digicolor.c index a536eeb634d8..10318cc99c0e 100644 --- a/drivers/clocksource/timer-digicolor.c +++ b/drivers/clocksource/timer-digicolor.c @@ -63,7 +63,7 @@ struct digicolor_timer { int timer_id; /* one of TIMER_* */ }; -struct digicolor_timer *dc_timer(struct clock_event_device *ce) +static struct digicolor_timer *dc_timer(struct clock_event_device *ce) { return container_of(ce, struct digicolor_timer, ce); } @@ -148,7 +148,7 @@ static u64 notrace digicolor_timer_sched_read(void) return ~readl(dc_timer_dev.base + COUNT(TIMER_B)); } -static void __init digicolor_timer_init(struct device_node *node) +static int __init digicolor_timer_init(struct device_node *node) { unsigned long rate; struct clk *clk; @@ -161,19 +161,19 @@ static void __init digicolor_timer_init(struct device_node *node) dc_timer_dev.base = of_iomap(node, 0); if (!dc_timer_dev.base) { pr_err("Can't map registers"); - return; + return -ENXIO; } irq = irq_of_parse_and_map(node, dc_timer_dev.timer_id); if (irq <= 0) { pr_err("Can't parse IRQ"); - return; + return -EINVAL; } clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("Can't get timer clock"); - return; + return PTR_ERR(clk); } clk_prepare_enable(clk); rate = clk_get_rate(clk); @@ -190,13 +190,17 @@ static void __init digicolor_timer_init(struct device_node *node) ret = request_irq(irq, digicolor_timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL, "digicolor_timerC", &dc_timer_dev.ce); - if (ret) + if (ret) { pr_warn("request of timer irq %d failed (%d)\n", irq, ret); + return ret; + } dc_timer_dev.ce.cpumask = cpu_possible_mask; dc_timer_dev.ce.irq = irq; clockevents_config_and_register(&dc_timer_dev.ce, rate, 0, 0xffffffff); + + return 0; } CLOCKSOURCE_OF_DECLARE(conexant_digicolor, "cnxt,cx92755-timer", digicolor_timer_init); diff --git a/drivers/clocksource/timer-imx-gpt.c b/drivers/clocksource/timer-imx-gpt.c index 99ec96769dda..f595460bfc58 100644 --- a/drivers/clocksource/timer-imx-gpt.c +++ b/drivers/clocksource/timer-imx-gpt.c @@ -407,8 +407,10 @@ static const struct imx_gpt_data imx6dl_gpt_data = { .set_next_event = v2_set_next_event, }; -static void __init _mxc_timer_init(struct imx_timer *imxtm) +static int __init _mxc_timer_init(struct imx_timer *imxtm) { + int ret; + switch (imxtm->type) { case GPT_TYPE_IMX1: imxtm->gpt = &imx1_gpt_data; @@ -423,12 +425,12 @@ static void __init _mxc_timer_init(struct imx_timer *imxtm) imxtm->gpt = &imx6dl_gpt_data; break; default: - BUG(); + return -EINVAL; } if (IS_ERR(imxtm->clk_per)) { pr_err("i.MX timer: unable to get clk\n"); - return; + return PTR_ERR(imxtm->clk_per); } if (!IS_ERR(imxtm->clk_ipg)) @@ -446,8 +448,11 @@ static void __init _mxc_timer_init(struct imx_timer *imxtm) imxtm->gpt->gpt_setup_tctl(imxtm); /* init and register the timer to the framework */ - mxc_clocksource_init(imxtm); - mxc_clockevent_init(imxtm); + ret = mxc_clocksource_init(imxtm); + if (ret) + return ret; + + return mxc_clockevent_init(imxtm); } void __init mxc_timer_init(unsigned long pbase, int irq, enum imx_gpt_type type) @@ -469,21 +474,27 @@ void __init mxc_timer_init(unsigned long pbase, int irq, enum imx_gpt_type type) _mxc_timer_init(imxtm); } -static void __init mxc_timer_init_dt(struct device_node *np, enum imx_gpt_type type) +static int __init mxc_timer_init_dt(struct device_node *np, enum imx_gpt_type type) { struct imx_timer *imxtm; static int initialized; + int ret; /* Support one instance only */ if (initialized) - return; + return 0; imxtm = kzalloc(sizeof(*imxtm), GFP_KERNEL); - BUG_ON(!imxtm); + if (!imxtm) + return -ENOMEM; imxtm->base = of_iomap(np, 0); - WARN_ON(!imxtm->base); + if (!imxtm->base) + return -ENXIO; + imxtm->irq = irq_of_parse_and_map(np, 0); + if (imxtm->irq <= 0) + return -EINVAL; imxtm->clk_ipg = of_clk_get_by_name(np, "ipg"); @@ -494,22 +505,26 @@ static void __init mxc_timer_init_dt(struct device_node *np, enum imx_gpt_type imxtm->type = type; - _mxc_timer_init(imxtm); + ret = _mxc_timer_init(imxtm); + if (ret) + return ret; initialized = 1; + + return 0; } -static void __init imx1_timer_init_dt(struct device_node *np) +static int __init imx1_timer_init_dt(struct device_node *np) { - mxc_timer_init_dt(np, GPT_TYPE_IMX1); + return mxc_timer_init_dt(np, GPT_TYPE_IMX1); } -static void __init imx21_timer_init_dt(struct device_node *np) +static int __init imx21_timer_init_dt(struct device_node *np) { - mxc_timer_init_dt(np, GPT_TYPE_IMX21); + return mxc_timer_init_dt(np, GPT_TYPE_IMX21); } -static void __init imx31_timer_init_dt(struct device_node *np) +static int __init imx31_timer_init_dt(struct device_node *np) { enum imx_gpt_type type = GPT_TYPE_IMX31; @@ -522,12 +537,12 @@ static void __init imx31_timer_init_dt(struct device_node *np) if (of_machine_is_compatible("fsl,imx6dl")) type = GPT_TYPE_IMX6DL; - mxc_timer_init_dt(np, type); + return mxc_timer_init_dt(np, type); } -static void __init imx6dl_timer_init_dt(struct device_node *np) +static int __init imx6dl_timer_init_dt(struct device_node *np) { - mxc_timer_init_dt(np, GPT_TYPE_IMX6DL); + return mxc_timer_init_dt(np, GPT_TYPE_IMX6DL); } CLOCKSOURCE_OF_DECLARE(imx1_timer, "fsl,imx1-gpt", imx1_timer_init_dt); diff --git a/drivers/clocksource/timer-integrator-ap.c b/drivers/clocksource/timer-integrator-ap.c index 3f59ac2180dc..df6e672afc04 100644 --- a/drivers/clocksource/timer-integrator-ap.c +++ b/drivers/clocksource/timer-integrator-ap.c @@ -36,11 +36,12 @@ static u64 notrace integrator_read_sched_clock(void) return -readl(sched_clk_base + TIMER_VALUE); } -static void integrator_clocksource_init(unsigned long inrate, - void __iomem *base) +static int integrator_clocksource_init(unsigned long inrate, + void __iomem *base) { u32 ctrl = TIMER_CTRL_ENABLE | TIMER_CTRL_PERIODIC; unsigned long rate = inrate; + int ret; if (rate >= 1500000) { rate /= 16; @@ -50,11 +51,15 @@ static void integrator_clocksource_init(unsigned long inrate, writel(0xffff, base + TIMER_LOAD); writel(ctrl, base + TIMER_CTRL); - clocksource_mmio_init(base + TIMER_VALUE, "timer2", - rate, 200, 16, clocksource_mmio_readl_down); + ret = clocksource_mmio_init(base + TIMER_VALUE, "timer2", + rate, 200, 16, clocksource_mmio_readl_down); + if (ret) + return ret; sched_clk_base = base; sched_clock_register(integrator_read_sched_clock, 16, rate); + + return 0; } static unsigned long timer_reload; @@ -138,11 +143,12 @@ static struct irqaction integrator_timer_irq = { .dev_id = &integrator_clockevent, }; -static void integrator_clockevent_init(unsigned long inrate, - void __iomem *base, int irq) +static int integrator_clockevent_init(unsigned long inrate, + void __iomem *base, int irq) { unsigned long rate = inrate; unsigned int ctrl = 0; + int ret; clkevt_base = base; /* Calculate and program a divisor */ @@ -156,14 +162,18 @@ static void integrator_clockevent_init(unsigned long inrate, timer_reload = rate / HZ; writel(ctrl, clkevt_base + TIMER_CTRL); - setup_irq(irq, &integrator_timer_irq); + ret = setup_irq(irq, &integrator_timer_irq); + if (ret) + return ret; + clockevents_config_and_register(&integrator_clockevent, rate, 1, 0xffffU); + return 0; } -static void __init integrator_ap_timer_init_of(struct device_node *node) +static int __init integrator_ap_timer_init_of(struct device_node *node) { const char *path; void __iomem *base; @@ -176,12 +186,12 @@ static void __init integrator_ap_timer_init_of(struct device_node *node) base = of_io_request_and_map(node, 0, "integrator-timer"); if (IS_ERR(base)) - return; + return PTR_ERR(base); clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("No clock for %s\n", node->name); - return; + return PTR_ERR(clk); } clk_prepare_enable(clk); rate = clk_get_rate(clk); @@ -189,30 +199,37 @@ static void __init integrator_ap_timer_init_of(struct device_node *node) err = of_property_read_string(of_aliases, "arm,timer-primary", &path); - if (WARN_ON(err)) - return; + if (err) { + pr_warn("Failed to read property"); + return err; + } + pri_node = of_find_node_by_path(path); + err = of_property_read_string(of_aliases, "arm,timer-secondary", &path); - if (WARN_ON(err)) - return; + if (err) { + pr_warn("Failed to read property"); + return err; + } + + sec_node = of_find_node_by_path(path); - if (node == pri_node) { + if (node == pri_node) /* The primary timer lacks IRQ, use as clocksource */ - integrator_clocksource_init(rate, base); - return; - } + return integrator_clocksource_init(rate, base); if (node == sec_node) { /* The secondary timer will drive the clock event */ irq = irq_of_parse_and_map(node, 0); - integrator_clockevent_init(rate, base, irq); - return; + return integrator_clockevent_init(rate, base, irq); } pr_info("Timer @%p unused\n", base); clk_disable_unprepare(clk); + + return 0; } CLOCKSOURCE_OF_DECLARE(integrator_ap_timer, "arm,integrator-timer", diff --git a/drivers/clocksource/timer-keystone.c b/drivers/clocksource/timer-keystone.c index 1cea08cf603e..ab68a47ab3b4 100644 --- a/drivers/clocksource/timer-keystone.c +++ b/drivers/clocksource/timer-keystone.c @@ -144,7 +144,7 @@ static int keystone_set_periodic(struct clock_event_device *evt) return 0; } -static void __init keystone_timer_init(struct device_node *np) +static int __init keystone_timer_init(struct device_node *np) { struct clock_event_device *event_dev = &timer.event_dev; unsigned long rate; @@ -154,20 +154,20 @@ static void __init keystone_timer_init(struct device_node *np) irq = irq_of_parse_and_map(np, 0); if (!irq) { pr_err("%s: failed to map interrupts\n", __func__); - return; + return -EINVAL; } timer.base = of_iomap(np, 0); if (!timer.base) { pr_err("%s: failed to map registers\n", __func__); - return; + return -ENXIO; } clk = of_clk_get(np, 0); if (IS_ERR(clk)) { pr_err("%s: failed to get clock\n", __func__); iounmap(timer.base); - return; + return PTR_ERR(clk); } error = clk_prepare_enable(clk); @@ -219,11 +219,12 @@ static void __init keystone_timer_init(struct device_node *np) clockevents_config_and_register(event_dev, rate, 1, ULONG_MAX); pr_info("keystone timer clock @%lu Hz\n", rate); - return; + return 0; err: clk_put(clk); iounmap(timer.base); + return error; } CLOCKSOURCE_OF_DECLARE(keystone_timer, "ti,keystone-timer", - keystone_timer_init); + keystone_timer_init); diff --git a/drivers/clocksource/timer-nps.c b/drivers/clocksource/timer-nps.c index d46108920b2c..70c149af8ee0 100644 --- a/drivers/clocksource/timer-nps.c +++ b/drivers/clocksource/timer-nps.c @@ -55,8 +55,8 @@ static cycle_t nps_clksrc_read(struct clocksource *clksrc) return (cycle_t)ioread32be(nps_msu_reg_low_addr[cluster]); } -static void __init nps_setup_clocksource(struct device_node *node, - struct clk *clk) +static int __init nps_setup_clocksource(struct device_node *node, + struct clk *clk) { int ret, cluster; @@ -68,7 +68,7 @@ static void __init nps_setup_clocksource(struct device_node *node, ret = clk_prepare_enable(clk); if (ret) { pr_err("Couldn't enable parent clock\n"); - return; + return ret; } nps_timer_rate = clk_get_rate(clk); @@ -79,19 +79,21 @@ static void __init nps_setup_clocksource(struct device_node *node, pr_err("Couldn't register clock source.\n"); clk_disable_unprepare(clk); } + + return ret; } -static void __init nps_timer_init(struct device_node *node) +static int __init nps_timer_init(struct device_node *node) { struct clk *clk; clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("Can't get timer clock.\n"); - return; + return PTR_ERR(clk); } - nps_setup_clocksource(node, clk); + return nps_setup_clocksource(node, clk); } CLOCKSOURCE_OF_DECLARE(ezchip_nps400_clksrc, "ezchip,nps400-timer", diff --git a/drivers/clocksource/timer-oxnas-rps.c b/drivers/clocksource/timer-oxnas-rps.c new file mode 100644 index 000000000000..bd887e2a8cf8 --- /dev/null +++ b/drivers/clocksource/timer-oxnas-rps.c @@ -0,0 +1,297 @@ +/* + * drivers/clocksource/timer-oxnas-rps.c + * + * Copyright (C) 2009 Oxford Semiconductor Ltd + * Copyright (C) 2013 Ma Haijun + * Copyright (C) 2016 Neil Armstrong + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* TIMER1 used as tick + * TIMER2 used as clocksource + */ + +/* Registers definitions */ + +#define TIMER_LOAD_REG 0x0 +#define TIMER_CURR_REG 0x4 +#define TIMER_CTRL_REG 0x8 +#define TIMER_CLRINT_REG 0xC + +#define TIMER_BITS 24 + +#define TIMER_MAX_VAL (BIT(TIMER_BITS) - 1) + +#define TIMER_PERIODIC BIT(6) +#define TIMER_ENABLE BIT(7) + +#define TIMER_DIV1 (0) +#define TIMER_DIV16 (1 << 2) +#define TIMER_DIV256 (2 << 2) + +#define TIMER1_REG_OFFSET 0 +#define TIMER2_REG_OFFSET 0x20 + +/* Clockevent & Clocksource data */ + +struct oxnas_rps_timer { + struct clock_event_device clkevent; + void __iomem *clksrc_base; + void __iomem *clkevt_base; + unsigned long timer_period; + unsigned int timer_prescaler; + struct clk *clk; + int irq; +}; + +static irqreturn_t oxnas_rps_timer_irq(int irq, void *dev_id) +{ + struct oxnas_rps_timer *rps = dev_id; + + writel_relaxed(0, rps->clkevt_base + TIMER_CLRINT_REG); + + rps->clkevent.event_handler(&rps->clkevent); + + return IRQ_HANDLED; +} + +static void oxnas_rps_timer_config(struct oxnas_rps_timer *rps, + unsigned long period, + unsigned int periodic) +{ + uint32_t cfg = rps->timer_prescaler; + + if (period) + cfg |= TIMER_ENABLE; + + if (periodic) + cfg |= TIMER_PERIODIC; + + writel_relaxed(period, rps->clkevt_base + TIMER_LOAD_REG); + writel_relaxed(cfg, rps->clkevt_base + TIMER_CTRL_REG); +} + +static int oxnas_rps_timer_shutdown(struct clock_event_device *evt) +{ + struct oxnas_rps_timer *rps = + container_of(evt, struct oxnas_rps_timer, clkevent); + + oxnas_rps_timer_config(rps, 0, 0); + + return 0; +} + +static int oxnas_rps_timer_set_periodic(struct clock_event_device *evt) +{ + struct oxnas_rps_timer *rps = + container_of(evt, struct oxnas_rps_timer, clkevent); + + oxnas_rps_timer_config(rps, rps->timer_period, 1); + + return 0; +} + +static int oxnas_rps_timer_set_oneshot(struct clock_event_device *evt) +{ + struct oxnas_rps_timer *rps = + container_of(evt, struct oxnas_rps_timer, clkevent); + + oxnas_rps_timer_config(rps, rps->timer_period, 0); + + return 0; +} + +static int oxnas_rps_timer_next_event(unsigned long delta, + struct clock_event_device *evt) +{ + struct oxnas_rps_timer *rps = + container_of(evt, struct oxnas_rps_timer, clkevent); + + oxnas_rps_timer_config(rps, delta, 0); + + return 0; +} + +static int __init oxnas_rps_clockevent_init(struct oxnas_rps_timer *rps) +{ + ulong clk_rate = clk_get_rate(rps->clk); + ulong timer_rate; + + /* Start with prescaler 1 */ + rps->timer_prescaler = TIMER_DIV1; + rps->timer_period = DIV_ROUND_UP(clk_rate, HZ); + timer_rate = clk_rate; + + if (rps->timer_period > TIMER_MAX_VAL) { + rps->timer_prescaler = TIMER_DIV16; + timer_rate = clk_rate / 16; + rps->timer_period = DIV_ROUND_UP(timer_rate, HZ); + } + if (rps->timer_period > TIMER_MAX_VAL) { + rps->timer_prescaler = TIMER_DIV256; + timer_rate = clk_rate / 256; + rps->timer_period = DIV_ROUND_UP(timer_rate, HZ); + } + + rps->clkevent.name = "oxnas-rps"; + rps->clkevent.features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_DYNIRQ; + rps->clkevent.tick_resume = oxnas_rps_timer_shutdown; + rps->clkevent.set_state_shutdown = oxnas_rps_timer_shutdown; + rps->clkevent.set_state_periodic = oxnas_rps_timer_set_periodic; + rps->clkevent.set_state_oneshot = oxnas_rps_timer_set_oneshot; + rps->clkevent.set_next_event = oxnas_rps_timer_next_event; + rps->clkevent.rating = 200; + rps->clkevent.cpumask = cpu_possible_mask; + rps->clkevent.irq = rps->irq; + clockevents_config_and_register(&rps->clkevent, + timer_rate, + 1, + TIMER_MAX_VAL); + + pr_info("Registered clock event rate %luHz prescaler %x period %lu\n", + clk_rate, + rps->timer_prescaler, + rps->timer_period); + + return 0; +} + +/* Clocksource */ + +static void __iomem *timer_sched_base; + +static u64 notrace oxnas_rps_read_sched_clock(void) +{ + return ~readl_relaxed(timer_sched_base); +} + +static int __init oxnas_rps_clocksource_init(struct oxnas_rps_timer *rps) +{ + ulong clk_rate = clk_get_rate(rps->clk); + int ret; + + /* use prescale 16 */ + clk_rate = clk_rate / 16; + + writel_relaxed(TIMER_MAX_VAL, rps->clksrc_base + TIMER_LOAD_REG); + writel_relaxed(TIMER_PERIODIC | TIMER_ENABLE | TIMER_DIV16, + rps->clksrc_base + TIMER_CTRL_REG); + + timer_sched_base = rps->clksrc_base + TIMER_CURR_REG; + sched_clock_register(oxnas_rps_read_sched_clock, + TIMER_BITS, clk_rate); + ret = clocksource_mmio_init(timer_sched_base, + "oxnas_rps_clocksource_timer", + clk_rate, 250, TIMER_BITS, + clocksource_mmio_readl_down); + if (WARN_ON(ret)) { + pr_err("can't register clocksource\n"); + return ret; + } + + pr_info("Registered clocksource rate %luHz\n", clk_rate); + + return 0; +} + +static int __init oxnas_rps_timer_init(struct device_node *np) +{ + struct oxnas_rps_timer *rps; + void __iomem *base; + int ret; + + rps = kzalloc(sizeof(*rps), GFP_KERNEL); + if (!rps) + return -ENOMEM; + + rps->clk = of_clk_get(np, 0); + if (IS_ERR(rps->clk)) { + ret = PTR_ERR(rps->clk); + goto err_alloc; + } + + ret = clk_prepare_enable(rps->clk); + if (ret) + goto err_clk; + + base = of_iomap(np, 0); + if (!base) { + ret = -ENXIO; + goto err_clk_prepare; + } + + rps->irq = irq_of_parse_and_map(np, 0); + if (rps->irq < 0) { + ret = -EINVAL; + goto err_iomap; + } + + rps->clkevt_base = base + TIMER1_REG_OFFSET; + rps->clksrc_base = base + TIMER2_REG_OFFSET; + + /* Disable timers */ + writel_relaxed(0, rps->clkevt_base + TIMER_CTRL_REG); + writel_relaxed(0, rps->clksrc_base + TIMER_CTRL_REG); + writel_relaxed(0, rps->clkevt_base + TIMER_LOAD_REG); + writel_relaxed(0, rps->clksrc_base + TIMER_LOAD_REG); + writel_relaxed(0, rps->clkevt_base + TIMER_CLRINT_REG); + writel_relaxed(0, rps->clksrc_base + TIMER_CLRINT_REG); + + ret = request_irq(rps->irq, oxnas_rps_timer_irq, + IRQF_TIMER | IRQF_IRQPOLL, + "rps-timer", rps); + if (ret) + goto err_iomap; + + ret = oxnas_rps_clocksource_init(rps); + if (ret) + goto err_irqreq; + + ret = oxnas_rps_clockevent_init(rps); + if (ret) + goto err_irqreq; + + return 0; + +err_irqreq: + free_irq(rps->irq, rps); +err_iomap: + iounmap(base); +err_clk_prepare: + clk_disable_unprepare(rps->clk); +err_clk: + clk_put(rps->clk); +err_alloc: + kfree(rps); + + return ret; +} + +CLOCKSOURCE_OF_DECLARE(ox810se_rps, + "oxsemi,ox810se-rps-timer", oxnas_rps_timer_init); diff --git a/drivers/clocksource/timer-prima2.c b/drivers/clocksource/timer-prima2.c index 2854c663e8b5..c32148ec7a38 100644 --- a/drivers/clocksource/timer-prima2.c +++ b/drivers/clocksource/timer-prima2.c @@ -19,7 +19,6 @@ #include #include #include -#include #define PRIMA2_CLOCK_FREQ 1000000 @@ -189,24 +188,36 @@ static void __init sirfsoc_clockevent_init(void) } /* initialize the kernel jiffy timer source */ -static void __init sirfsoc_prima2_timer_init(struct device_node *np) +static int __init sirfsoc_prima2_timer_init(struct device_node *np) { unsigned long rate; struct clk *clk; + int ret; clk = of_clk_get(np, 0); - BUG_ON(IS_ERR(clk)); + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } - BUG_ON(clk_prepare_enable(clk)); + ret = clk_prepare_enable(clk); + if (ret) { + pr_err("Failed to enable clock"); + return ret; + } rate = clk_get_rate(clk); - BUG_ON(rate < PRIMA2_CLOCK_FREQ); - BUG_ON(rate % PRIMA2_CLOCK_FREQ); + if (rate < PRIMA2_CLOCK_FREQ || rate % PRIMA2_CLOCK_FREQ) { + pr_err("Invalid clock rate"); + return -EINVAL; + } sirfsoc_timer_base = of_iomap(np, 0); - if (!sirfsoc_timer_base) - panic("unable to map timer cpu registers\n"); + if (!sirfsoc_timer_base) { + pr_err("unable to map timer cpu registers\n"); + return -ENXIO; + } sirfsoc_timer_irq.irq = irq_of_parse_and_map(np, 0); @@ -216,14 +227,23 @@ static void __init sirfsoc_prima2_timer_init(struct device_node *np) writel_relaxed(0, sirfsoc_timer_base + SIRFSOC_TIMER_COUNTER_HI); writel_relaxed(BIT(0), sirfsoc_timer_base + SIRFSOC_TIMER_STATUS); - BUG_ON(clocksource_register_hz(&sirfsoc_clocksource, - PRIMA2_CLOCK_FREQ)); + ret = clocksource_register_hz(&sirfsoc_clocksource, PRIMA2_CLOCK_FREQ); + if (ret) { + pr_err("Failed to register clocksource"); + return ret; + } sched_clock_register(sirfsoc_read_sched_clock, 64, PRIMA2_CLOCK_FREQ); - BUG_ON(setup_irq(sirfsoc_timer_irq.irq, &sirfsoc_timer_irq)); + ret = setup_irq(sirfsoc_timer_irq.irq, &sirfsoc_timer_irq); + if (ret) { + pr_err("Failed to setup irq"); + return ret; + } sirfsoc_clockevent_init(); + + return 0; } CLOCKSOURCE_OF_DECLARE(sirfsoc_prima2_timer, "sirf,prima2-tick", sirfsoc_prima2_timer_init); diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c index 5f45b9adef60..d07863388e05 100644 --- a/drivers/clocksource/timer-sp804.c +++ b/drivers/clocksource/timer-sp804.c @@ -77,7 +77,7 @@ void __init sp804_timer_disable(void __iomem *base) writel(0, base + TIMER_CTRL); } -void __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, +int __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, const char *name, struct clk *clk, int use_sched_clock) @@ -89,14 +89,13 @@ void __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, if (IS_ERR(clk)) { pr_err("sp804: clock not found: %d\n", (int)PTR_ERR(clk)); - return; + return PTR_ERR(clk); } } rate = sp804_get_clock_rate(clk); - if (rate < 0) - return; + return -EINVAL; /* setup timer 0 as free-running clocksource */ writel(0, base + TIMER_CTRL); @@ -112,6 +111,8 @@ void __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, sched_clock_base = base; sched_clock_register(sp804_read, 32, rate); } + + return 0; } @@ -186,7 +187,7 @@ static struct irqaction sp804_timer_irq = { .dev_id = &sp804_clockevent, }; -void __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struct clk *clk, const char *name) +int __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struct clk *clk, const char *name) { struct clock_event_device *evt = &sp804_clockevent; long rate; @@ -196,12 +197,12 @@ void __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struc if (IS_ERR(clk)) { pr_err("sp804: %s clock not found: %d\n", name, (int)PTR_ERR(clk)); - return; + return PTR_ERR(clk); } rate = sp804_get_clock_rate(clk); if (rate < 0) - return; + return -EINVAL; clkevt_base = base; clkevt_reload = DIV_ROUND_CLOSEST(rate, HZ); @@ -213,27 +214,31 @@ void __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struc setup_irq(irq, &sp804_timer_irq); clockevents_config_and_register(evt, rate, 0xf, 0xffffffff); + + return 0; } -static void __init sp804_of_init(struct device_node *np) +static int __init sp804_of_init(struct device_node *np) { static bool initialized = false; void __iomem *base; - int irq; + int irq, ret = -EINVAL; u32 irq_num = 0; struct clk *clk1, *clk2; const char *name = of_get_property(np, "compatible", NULL); base = of_iomap(np, 0); - if (WARN_ON(!base)) - return; + if (!base) + return -ENXIO; /* Ensure timers are disabled */ writel(0, base + TIMER_CTRL); writel(0, base + TIMER_2_BASE + TIMER_CTRL); - if (initialized || !of_device_is_available(np)) + if (initialized || !of_device_is_available(np)) { + ret = -EINVAL; goto err; + } clk1 = of_clk_get(np, 0); if (IS_ERR(clk1)) @@ -256,35 +261,53 @@ static void __init sp804_of_init(struct device_node *np) of_property_read_u32(np, "arm,sp804-has-irq", &irq_num); if (irq_num == 2) { - __sp804_clockevents_init(base + TIMER_2_BASE, irq, clk2, name); - __sp804_clocksource_and_sched_clock_init(base, name, clk1, 1); + + ret = __sp804_clockevents_init(base + TIMER_2_BASE, irq, clk2, name); + if (ret) + goto err; + + ret = __sp804_clocksource_and_sched_clock_init(base, name, clk1, 1); + if (ret) + goto err; } else { - __sp804_clockevents_init(base, irq, clk1 , name); - __sp804_clocksource_and_sched_clock_init(base + TIMER_2_BASE, - name, clk2, 1); + + ret = __sp804_clockevents_init(base, irq, clk1 , name); + if (ret) + goto err; + + ret =__sp804_clocksource_and_sched_clock_init(base + TIMER_2_BASE, + name, clk2, 1); + if (ret) + goto err; } initialized = true; - return; + return 0; err: iounmap(base); + return ret; } CLOCKSOURCE_OF_DECLARE(sp804, "arm,sp804", sp804_of_init); -static void __init integrator_cp_of_init(struct device_node *np) +static int __init integrator_cp_of_init(struct device_node *np) { static int init_count = 0; void __iomem *base; - int irq; + int irq, ret = -EINVAL; const char *name = of_get_property(np, "compatible", NULL); struct clk *clk; base = of_iomap(np, 0); - if (WARN_ON(!base)) - return; + if (!base) { + pr_err("Failed to iomap"); + return -ENXIO; + } + clk = of_clk_get(np, 0); - if (WARN_ON(IS_ERR(clk))) - return; + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } /* Ensure timer is disabled */ writel(0, base + TIMER_CTRL); @@ -292,19 +315,24 @@ static void __init integrator_cp_of_init(struct device_node *np) if (init_count == 2 || !of_device_is_available(np)) goto err; - if (!init_count) - __sp804_clocksource_and_sched_clock_init(base, name, clk, 0); - else { + if (!init_count) { + ret = __sp804_clocksource_and_sched_clock_init(base, name, clk, 0); + if (ret) + goto err; + } else { irq = irq_of_parse_and_map(np, 0); if (irq <= 0) goto err; - __sp804_clockevents_init(base, irq, clk, name); + ret = __sp804_clockevents_init(base, irq, clk, name); + if (ret) + goto err; } init_count++; - return; + return 0; err: iounmap(base); + return ret; } CLOCKSOURCE_OF_DECLARE(intcp, "arm,integrator-cp-timer", integrator_cp_of_init); diff --git a/drivers/clocksource/timer-stm32.c b/drivers/clocksource/timer-stm32.c index f3dcb76799b4..1b2574c4fb97 100644 --- a/drivers/clocksource/timer-stm32.c +++ b/drivers/clocksource/timer-stm32.c @@ -98,7 +98,7 @@ static struct stm32_clock_event_ddata clock_event_ddata = { }, }; -static void __init stm32_clockevent_init(struct device_node *np) +static int __init stm32_clockevent_init(struct device_node *np) { struct stm32_clock_event_ddata *data = &clock_event_ddata; struct clk *clk; @@ -130,12 +130,14 @@ static void __init stm32_clockevent_init(struct device_node *np) data->base = of_iomap(np, 0); if (!data->base) { + ret = -ENXIO; pr_err("failed to map registers for clockevent\n"); goto err_iomap; } irq = irq_of_parse_and_map(np, 0); if (!irq) { + ret = -EINVAL; pr_err("%s: failed to get irq.\n", np->full_name); goto err_get_irq; } @@ -173,7 +175,7 @@ static void __init stm32_clockevent_init(struct device_node *np) pr_info("%s: STM32 clockevent driver initialized (%d bits)\n", np->full_name, bits); - return; + return ret; err_get_irq: iounmap(data->base); @@ -182,7 +184,7 @@ err_iomap: err_clk_enable: clk_put(clk); err_clk_get: - return; + return ret; } CLOCKSOURCE_OF_DECLARE(stm32, "st,stm32-timer", stm32_clockevent_init); diff --git a/drivers/clocksource/timer-sun5i.c b/drivers/clocksource/timer-sun5i.c index 24c83f9efd87..c184eb84101e 100644 --- a/drivers/clocksource/timer-sun5i.c +++ b/drivers/clocksource/timer-sun5i.c @@ -311,33 +311,42 @@ err_free: return ret; } -static void __init sun5i_timer_init(struct device_node *node) +static int __init sun5i_timer_init(struct device_node *node) { struct reset_control *rstc; void __iomem *timer_base; struct clk *clk; - int irq; + int irq, ret; timer_base = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(timer_base)) - panic("Can't map registers"); + if (IS_ERR(timer_base)) { + pr_err("Can't map registers"); + return PTR_ERR(timer_base);; + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_err("Can't parse IRQ"); + return -EINVAL; + } clk = of_clk_get(node, 0); - if (IS_ERR(clk)) - panic("Can't get timer clock"); + if (IS_ERR(clk)) { + pr_err("Can't get timer clock"); + return PTR_ERR(clk); + } rstc = of_reset_control_get(node, NULL); if (!IS_ERR(rstc)) reset_control_deassert(rstc); - sun5i_setup_clocksource(node, timer_base, clk, irq); - sun5i_setup_clockevent(node, timer_base, clk, irq); + ret = sun5i_setup_clocksource(node, timer_base, clk, irq); + if (ret) + return ret; + + return sun5i_setup_clockevent(node, timer_base, clk, irq); } CLOCKSOURCE_OF_DECLARE(sun5i_a13, "allwinner,sun5i-a13-hstimer", - sun5i_timer_init); + sun5i_timer_init); CLOCKSOURCE_OF_DECLARE(sun7i_a20, "allwinner,sun7i-a20-hstimer", - sun5i_timer_init); + sun5i_timer_init); diff --git a/drivers/clocksource/timer-ti-32k.c b/drivers/clocksource/timer-ti-32k.c index 8518d9dfba5c..92b7e390f6c8 100644 --- a/drivers/clocksource/timer-ti-32k.c +++ b/drivers/clocksource/timer-ti-32k.c @@ -88,14 +88,14 @@ static u64 notrace omap_32k_read_sched_clock(void) return ti_32k_read_cycles(&ti_32k_timer.cs); } -static void __init ti_32k_timer_init(struct device_node *np) +static int __init ti_32k_timer_init(struct device_node *np) { int ret; ti_32k_timer.base = of_iomap(np, 0); if (!ti_32k_timer.base) { pr_err("Can't ioremap 32k timer base\n"); - return; + return -ENXIO; } ti_32k_timer.counter = ti_32k_timer.base; @@ -116,11 +116,13 @@ static void __init ti_32k_timer_init(struct device_node *np) ret = clocksource_register_hz(&ti_32k_timer.cs, 32768); if (ret) { pr_err("32k_counter: can't register clocksource\n"); - return; + return ret; } sched_clock_register(omap_32k_read_sched_clock, 32, 32768); pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n"); + + return 0; } CLOCKSOURCE_OF_DECLARE(ti_32k_timer, "ti,omap-counter32k", ti_32k_timer_init); diff --git a/drivers/clocksource/timer-u300.c b/drivers/clocksource/timer-u300.c index 1744b243898a..704e40c6f151 100644 --- a/drivers/clocksource/timer-u300.c +++ b/drivers/clocksource/timer-u300.c @@ -359,27 +359,37 @@ static struct delay_timer u300_delay_timer; /* * This sets up the system timers, clock source and clock event. */ -static void __init u300_timer_init_of(struct device_node *np) +static int __init u300_timer_init_of(struct device_node *np) { unsigned int irq; struct clk *clk; unsigned long rate; + int ret; u300_timer_base = of_iomap(np, 0); - if (!u300_timer_base) - panic("could not ioremap system timer\n"); + if (!u300_timer_base) { + pr_err("could not ioremap system timer\n"); + return -ENXIO; + } /* Get the IRQ for the GP1 timer */ irq = irq_of_parse_and_map(np, 2); - if (!irq) - panic("no IRQ for system timer\n"); + if (!irq) { + pr_err("no IRQ for system timer\n"); + return -EINVAL; + } pr_info("U300 GP1 timer @ base: %p, IRQ: %u\n", u300_timer_base, irq); /* Clock the interrupt controller */ clk = of_clk_get(np, 0); - BUG_ON(IS_ERR(clk)); - clk_prepare_enable(clk); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + ret = clk_prepare_enable(clk); + if (ret) + return ret; + rate = clk_get_rate(clk); u300_clockevent_data.ticks_per_jiffy = DIV_ROUND_CLOSEST(rate, HZ); @@ -410,7 +420,9 @@ static void __init u300_timer_init_of(struct device_node *np) u300_timer_base + U300_TIMER_APP_RGPT1); /* Set up the IRQ handler */ - setup_irq(irq, &u300_timer_irq); + ret = setup_irq(irq, &u300_timer_irq); + if (ret) + return ret; /* Reset the General Purpose timer 2 */ writel(U300_TIMER_APP_RGPT2_TIMER_RESET, @@ -428,9 +440,12 @@ static void __init u300_timer_init_of(struct device_node *np) u300_timer_base + U300_TIMER_APP_EGPT2); /* Use general purpose timer 2 as clock source */ - if (clocksource_mmio_init(u300_timer_base + U300_TIMER_APP_GPT2CC, - "GPT2", rate, 300, 32, clocksource_mmio_readl_up)) + ret = clocksource_mmio_init(u300_timer_base + U300_TIMER_APP_GPT2CC, + "GPT2", rate, 300, 32, clocksource_mmio_readl_up); + if (ret) { pr_err("timer: failed to initialize U300 clock source\n"); + return ret; + } /* Configure and register the clockevent */ clockevents_config_and_register(&u300_clockevent_data.cevd, rate, @@ -440,6 +455,7 @@ static void __init u300_timer_init_of(struct device_node *np) * TODO: init and register the rest of the timers too, they can be * used by hrtimers! */ + return 0; } CLOCKSOURCE_OF_DECLARE(u300_timer, "stericsson,u300-apptimer", diff --git a/drivers/clocksource/versatile.c b/drivers/clocksource/versatile.c index 0a26d3dde6c0..220b490a8142 100644 --- a/drivers/clocksource/versatile.c +++ b/drivers/clocksource/versatile.c @@ -25,16 +25,18 @@ static u64 notrace versatile_sys_24mhz_read(void) return readl(versatile_sys_24mhz); } -static void __init versatile_sched_clock_init(struct device_node *node) +static int __init versatile_sched_clock_init(struct device_node *node) { void __iomem *base = of_iomap(node, 0); if (!base) - return; + return -ENXIO; versatile_sys_24mhz = base + SYS_24MHZ; sched_clock_register(versatile_sys_24mhz_read, 32, 24000000); + + return 0; } CLOCKSOURCE_OF_DECLARE(vexpress, "arm,vexpress-sysreg", versatile_sched_clock_init); diff --git a/drivers/clocksource/vf_pit_timer.c b/drivers/clocksource/vf_pit_timer.c index a0e6c68536a1..55d8d8402d90 100644 --- a/drivers/clocksource/vf_pit_timer.c +++ b/drivers/clocksource/vf_pit_timer.c @@ -156,15 +156,18 @@ static int __init pit_clockevent_init(unsigned long rate, int irq) return 0; } -static void __init pit_timer_init(struct device_node *np) +static int __init pit_timer_init(struct device_node *np) { struct clk *pit_clk; void __iomem *timer_base; unsigned long clk_rate; - int irq; + int irq, ret; timer_base = of_iomap(np, 0); - BUG_ON(!timer_base); + if (!timer_base) { + pr_err("Failed to iomap"); + return -ENXIO; + } /* * PIT0 and PIT1 can be chained to build a 64-bit timer, @@ -175,12 +178,16 @@ static void __init pit_timer_init(struct device_node *np) clkevt_base = timer_base + PITn_OFFSET(3); irq = irq_of_parse_and_map(np, 0); - BUG_ON(irq <= 0); + if (irq <= 0) + return -EINVAL; pit_clk = of_clk_get(np, 0); - BUG_ON(IS_ERR(pit_clk)); + if (IS_ERR(pit_clk)) + return PTR_ERR(pit_clk); - BUG_ON(clk_prepare_enable(pit_clk)); + ret = clk_prepare_enable(pit_clk); + if (ret) + return ret; clk_rate = clk_get_rate(pit_clk); cycle_per_jiffy = clk_rate / (HZ); @@ -188,8 +195,10 @@ static void __init pit_timer_init(struct device_node *np) /* enable the pit module */ __raw_writel(~PITMCR_MDIS, timer_base + PITMCR); - BUG_ON(pit_clocksource_init(clk_rate)); + ret = pit_clocksource_init(clk_rate); + if (ret) + return ret; - pit_clockevent_init(clk_rate, irq); + return pit_clockevent_init(clk_rate, irq); } CLOCKSOURCE_OF_DECLARE(vf610, "fsl,vf610-pit", pit_timer_init); diff --git a/drivers/clocksource/vt8500_timer.c b/drivers/clocksource/vt8500_timer.c index ddb409274f45..b15069483fbd 100644 --- a/drivers/clocksource/vt8500_timer.c +++ b/drivers/clocksource/vt8500_timer.c @@ -121,38 +121,48 @@ static struct irqaction irq = { .dev_id = &clockevent, }; -static void __init vt8500_timer_init(struct device_node *np) +static int __init vt8500_timer_init(struct device_node *np) { - int timer_irq; + int timer_irq, ret; regbase = of_iomap(np, 0); if (!regbase) { pr_err("%s: Missing iobase description in Device Tree\n", __func__); - return; + return -ENXIO; } + timer_irq = irq_of_parse_and_map(np, 0); if (!timer_irq) { pr_err("%s: Missing irq description in Device Tree\n", __func__); - return; + return -EINVAL; } writel(1, regbase + TIMER_CTRL_VAL); writel(0xf, regbase + TIMER_STATUS_VAL); writel(~0, regbase + TIMER_MATCH_VAL); - if (clocksource_register_hz(&clocksource, VT8500_TIMER_HZ)) + ret = clocksource_register_hz(&clocksource, VT8500_TIMER_HZ); + if (ret) { pr_err("%s: vt8500_timer_init: clocksource_register failed for %s\n", - __func__, clocksource.name); + __func__, clocksource.name); + return ret; + } clockevent.cpumask = cpumask_of(0); - if (setup_irq(timer_irq, &irq)) + ret = setup_irq(timer_irq, &irq); + if (ret) { pr_err("%s: setup_irq failed for %s\n", __func__, clockevent.name); + return ret; + } + clockevents_config_and_register(&clockevent, VT8500_TIMER_HZ, MIN_OSCR_DELTA * 2, 0xf0000000); + + return 0; } CLOCKSOURCE_OF_DECLARE(vt8500, "via,vt8500-timer", vt8500_timer_init); diff --git a/drivers/clocksource/zevio-timer.c b/drivers/clocksource/zevio-timer.c index ceaa6133f9c2..9a53f5ef6157 100644 --- a/drivers/clocksource/zevio-timer.c +++ b/drivers/clocksource/zevio-timer.c @@ -210,9 +210,9 @@ error_free: return ret; } -static void __init zevio_timer_init(struct device_node *node) +static int __init zevio_timer_init(struct device_node *node) { - BUG_ON(zevio_timer_add(node)); + return zevio_timer_add(node); } CLOCKSOURCE_OF_DECLARE(zevio_timer, "lsi,zevio-timer", zevio_timer_init); diff --git a/drivers/connector/cn_proc.c b/drivers/connector/cn_proc.c index 15d06fcf0b50..a782ce87715c 100644 --- a/drivers/connector/cn_proc.c +++ b/drivers/connector/cn_proc.c @@ -22,7 +22,6 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#include #include #include #include @@ -56,11 +55,21 @@ static struct cb_id cn_proc_event_id = { CN_IDX_PROC, CN_VAL_PROC }; /* proc_event_counts is used as the sequence number of the netlink message */ static DEFINE_PER_CPU(__u32, proc_event_counts) = { 0 }; -static inline void get_seq(__u32 *ts, int *cpu) +static inline void send_msg(struct cn_msg *msg) { preempt_disable(); - *ts = __this_cpu_inc_return(proc_event_counts) - 1; - *cpu = smp_processor_id(); + + msg->seq = __this_cpu_inc_return(proc_event_counts) - 1; + ((struct proc_event *)msg->data)->cpu = smp_processor_id(); + + /* + * Preemption remains disabled during send to ensure the messages are + * ordered according to their sequence numbers. + * + * If cn_netlink_send() fails, the data is not sent. + */ + cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_NOWAIT); + preempt_enable(); } @@ -77,7 +86,6 @@ void proc_fork_connector(struct task_struct *task) msg = buffer_to_cn_msg(buffer); ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); - get_seq(&msg->seq, &ev->cpu); ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_FORK; rcu_read_lock(); @@ -92,8 +100,7 @@ void proc_fork_connector(struct task_struct *task) msg->ack = 0; /* not used */ msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - /* If cn_netlink_send() failed, the data is not sent */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } void proc_exec_connector(struct task_struct *task) @@ -108,7 +115,6 @@ void proc_exec_connector(struct task_struct *task) msg = buffer_to_cn_msg(buffer); ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); - get_seq(&msg->seq, &ev->cpu); ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_EXEC; ev->event_data.exec.process_pid = task->pid; @@ -118,7 +124,7 @@ void proc_exec_connector(struct task_struct *task) msg->ack = 0; /* not used */ msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } void proc_id_connector(struct task_struct *task, int which_id) @@ -150,14 +156,13 @@ void proc_id_connector(struct task_struct *task, int which_id) return; } rcu_read_unlock(); - get_seq(&msg->seq, &ev->cpu); ev->timestamp_ns = ktime_get_ns(); memcpy(&msg->id, &cn_proc_event_id, sizeof(msg->id)); msg->ack = 0; /* not used */ msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } void proc_sid_connector(struct task_struct *task) @@ -172,7 +177,6 @@ void proc_sid_connector(struct task_struct *task) msg = buffer_to_cn_msg(buffer); ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); - get_seq(&msg->seq, &ev->cpu); ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_SID; ev->event_data.sid.process_pid = task->pid; @@ -182,7 +186,7 @@ void proc_sid_connector(struct task_struct *task) msg->ack = 0; /* not used */ msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } void proc_ptrace_connector(struct task_struct *task, int ptrace_id) @@ -197,7 +201,6 @@ void proc_ptrace_connector(struct task_struct *task, int ptrace_id) msg = buffer_to_cn_msg(buffer); ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); - get_seq(&msg->seq, &ev->cpu); ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_PTRACE; ev->event_data.ptrace.process_pid = task->pid; @@ -215,7 +218,7 @@ void proc_ptrace_connector(struct task_struct *task, int ptrace_id) msg->ack = 0; /* not used */ msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } void proc_comm_connector(struct task_struct *task) @@ -230,7 +233,6 @@ void proc_comm_connector(struct task_struct *task) msg = buffer_to_cn_msg(buffer); ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); - get_seq(&msg->seq, &ev->cpu); ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_COMM; ev->event_data.comm.process_pid = task->pid; @@ -241,7 +243,7 @@ void proc_comm_connector(struct task_struct *task) msg->ack = 0; /* not used */ msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } void proc_coredump_connector(struct task_struct *task) @@ -256,7 +258,6 @@ void proc_coredump_connector(struct task_struct *task) msg = buffer_to_cn_msg(buffer); ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); - get_seq(&msg->seq, &ev->cpu); ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_COREDUMP; ev->event_data.coredump.process_pid = task->pid; @@ -266,7 +267,7 @@ void proc_coredump_connector(struct task_struct *task) msg->ack = 0; /* not used */ msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } void proc_exit_connector(struct task_struct *task) @@ -281,7 +282,6 @@ void proc_exit_connector(struct task_struct *task) msg = buffer_to_cn_msg(buffer); ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); - get_seq(&msg->seq, &ev->cpu); ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_EXIT; ev->event_data.exit.process_pid = task->pid; @@ -293,7 +293,7 @@ void proc_exit_connector(struct task_struct *task) msg->ack = 0; /* not used */ msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } /* @@ -325,7 +325,7 @@ static void cn_proc_ack(int err, int rcvd_seq, int rcvd_ack) msg->ack = rcvd_ack + 1; msg->len = sizeof(*ev); msg->flags = 0; /* not used */ - cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_KERNEL); + send_msg(msg); } /** @@ -389,5 +389,4 @@ static int __init cn_proc_init(void) } return 0; } - -module_init(cn_proc_init); +device_initcall(cn_proc_init); diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index b7445b6ae5a4..c822d72629d5 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -31,23 +31,18 @@ config CPU_FREQ_BOOST_SW depends on THERMAL config CPU_FREQ_STAT - tristate "CPU frequency translation statistics" + bool "CPU frequency transition statistics" default y help - This driver exports CPU frequency statistics information through sysfs - file system. - - To compile this driver as a module, choose M here: the - module will be called cpufreq_stats. + Export CPU frequency statistics information through sysfs. If in doubt, say N. config CPU_FREQ_STAT_DETAILS - bool "CPU frequency translation statistics details" + bool "CPU frequency transition statistics details" depends on CPU_FREQ_STAT help - This will show detail CPU frequency translation table in sysfs file - system. + Show detailed CPU frequency transition table in sysfs. If in doubt, say N. diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 32a15052f363..297e9128fe9f 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -468,20 +468,17 @@ unsigned int acpi_cpufreq_fast_switch(struct cpufreq_policy *policy, struct acpi_cpufreq_data *data = policy->driver_data; struct acpi_processor_performance *perf; struct cpufreq_frequency_table *entry; - unsigned int next_perf_state, next_freq, freq; + unsigned int next_perf_state, next_freq, index; /* * Find the closest frequency above target_freq. - * - * The table is sorted in the reverse order with respect to the - * frequency and all of the entries are valid (see the initialization). */ - entry = policy->freq_table; - do { - entry++; - freq = entry->frequency; - } while (freq >= target_freq && freq != CPUFREQ_TABLE_END); - entry--; + if (policy->cached_target_freq == target_freq) + index = policy->cached_resolved_idx; + else + index = cpufreq_table_find_index_dl(policy, target_freq); + + entry = &policy->freq_table[index]; next_freq = entry->frequency; next_perf_state = entry->driver_data; diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c index 404360cad25c..042023bbbf62 100644 --- a/drivers/cpufreq/amd_freq_sensitivity.c +++ b/drivers/cpufreq/amd_freq_sensitivity.c @@ -48,9 +48,8 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy, struct policy_dbs_info *policy_dbs = policy->governor_data; struct dbs_data *od_data = policy_dbs->dbs_data; struct od_dbs_tuners *od_tuners = od_data->tuners; - struct od_policy_dbs_info *od_info = to_dbs_info(policy_dbs); - if (!od_info->freq_table) + if (!policy->freq_table) return freq_next; rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, @@ -92,10 +91,9 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy, else { unsigned int index; - cpufreq_frequency_table_target(policy, - od_info->freq_table, policy->cur - 1, - CPUFREQ_RELATION_H, &index); - freq_next = od_info->freq_table[index].frequency; + index = cpufreq_table_find_index_h(policy, + policy->cur - 1); + freq_next = policy->freq_table[index].frequency; } data->freq_prev = freq_next; diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index 3646b143bbf5..0bb44d5b5df4 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -79,15 +79,16 @@ static const struct of_device_id machines[] __initconst = { static int __init cpufreq_dt_platdev_init(void) { struct device_node *np = of_find_node_by_path("/"); + const struct of_device_id *match; if (!np) return -ENODEV; - if (!of_match_node(machines, np)) + match = of_match_node(machines, np); + of_node_put(np); + if (!match) return -ENODEV; - of_node_put(of_root); - return PTR_ERR_OR_ZERO(platform_device_register_simple("cpufreq-dt", -1, NULL, 0)); } diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 9009295f5134..3dd4884c6f9e 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -74,19 +74,12 @@ static inline bool has_target(void) } /* internal prototypes */ -static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event); static unsigned int __cpufreq_get(struct cpufreq_policy *policy); +static int cpufreq_init_governor(struct cpufreq_policy *policy); +static void cpufreq_exit_governor(struct cpufreq_policy *policy); static int cpufreq_start_governor(struct cpufreq_policy *policy); - -static inline void cpufreq_exit_governor(struct cpufreq_policy *policy) -{ - (void)cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); -} - -static inline void cpufreq_stop_governor(struct cpufreq_policy *policy) -{ - (void)cpufreq_governor(policy, CPUFREQ_GOV_STOP); -} +static void cpufreq_stop_governor(struct cpufreq_policy *policy); +static void cpufreq_governor_limits(struct cpufreq_policy *policy); /** * Two notifier lists: the "policy" list is involved in the @@ -133,15 +126,6 @@ struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) } EXPORT_SYMBOL_GPL(get_governor_parent_kobj); -struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu) -{ - struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); - - return policy && !policy_is_inactive(policy) ? - policy->freq_table : NULL; -} -EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table); - static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) { u64 idle_time; @@ -354,6 +338,7 @@ static void __cpufreq_notify_transition(struct cpufreq_policy *policy, pr_debug("FREQ: %lu - CPU: %lu\n", (unsigned long)freqs->new, (unsigned long)freqs->cpu); trace_cpu_frequency(freqs->new, freqs->cpu); + cpufreq_stats_record_transition(policy, freqs->new); srcu_notifier_call_chain(&cpufreq_transition_notifier_list, CPUFREQ_POSTCHANGE, freqs); if (likely(policy) && likely(policy->cpu == freqs->cpu)) @@ -507,6 +492,38 @@ void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) } EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); +/** + * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported + * one. + * @target_freq: target frequency to resolve. + * + * The target to driver frequency mapping is cached in the policy. + * + * Return: Lowest driver-supported frequency greater than or equal to the + * given target_freq, subject to policy (min/max) and driver limitations. + */ +unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + target_freq = clamp_val(target_freq, policy->min, policy->max); + policy->cached_target_freq = target_freq; + + if (cpufreq_driver->target_index) { + int idx; + + idx = cpufreq_frequency_table_target(policy, target_freq, + CPUFREQ_RELATION_L); + policy->cached_resolved_idx = idx; + return policy->freq_table[idx].frequency; + } + + if (cpufreq_driver->resolve_freq) + return cpufreq_driver->resolve_freq(policy, target_freq); + + return target_freq; +} +EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); + /********************************************************************* * SYSFS INTERFACE * *********************************************************************/ @@ -1115,6 +1132,7 @@ static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify) CPUFREQ_REMOVE_POLICY, policy); down_write(&policy->rwsem); + cpufreq_stats_free_table(policy); cpufreq_remove_dev_symlink(policy); kobj = &policy->kobj; cmp = &policy->kobj_unregister; @@ -1265,13 +1283,12 @@ static int cpufreq_online(unsigned int cpu) } } - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, - CPUFREQ_START, policy); - if (new_policy) { ret = cpufreq_add_dev_interface(policy); if (ret) goto out_exit_policy; + + cpufreq_stats_create_table(policy); blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_CREATE_POLICY, policy); @@ -1280,6 +1297,9 @@ static int cpufreq_online(unsigned int cpu) write_unlock_irqrestore(&cpufreq_driver_lock, flags); } + blocking_notifier_call_chain(&cpufreq_policy_notifier_list, + CPUFREQ_START, policy); + ret = cpufreq_init_policy(policy); if (ret) { pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", @@ -1556,9 +1576,6 @@ static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy) { unsigned int new_freq; - if (cpufreq_suspended) - return 0; - new_freq = cpufreq_driver->get(policy->cpu); if (!new_freq) return 0; @@ -1864,14 +1881,17 @@ static int __target_intermediate(struct cpufreq_policy *policy, return ret; } -static int __target_index(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *freq_table, int index) +static int __target_index(struct cpufreq_policy *policy, int index) { struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; unsigned int intermediate_freq = 0; + unsigned int newfreq = policy->freq_table[index].frequency; int retval = -EINVAL; bool notify; + if (newfreq == policy->cur) + return 0; + notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); if (notify) { /* Handle switching to intermediate frequency */ @@ -1886,7 +1906,7 @@ static int __target_index(struct cpufreq_policy *policy, freqs.old = freqs.new; } - freqs.new = freq_table[index].frequency; + freqs.new = newfreq; pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", __func__, policy->cpu, freqs.old, freqs.new); @@ -1923,17 +1943,13 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy, unsigned int relation) { unsigned int old_target_freq = target_freq; - struct cpufreq_frequency_table *freq_table; - int index, retval; + int index; if (cpufreq_disabled()) return -ENODEV; /* Make sure that target_freq is within supported range */ - if (target_freq > policy->max) - target_freq = policy->max; - if (target_freq < policy->min) - target_freq = policy->min; + target_freq = clamp_val(target_freq, policy->min, policy->max); pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", policy->cpu, target_freq, relation, old_target_freq); @@ -1956,23 +1972,9 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy, if (!cpufreq_driver->target_index) return -EINVAL; - freq_table = cpufreq_frequency_get_table(policy->cpu); - if (unlikely(!freq_table)) { - pr_err("%s: Unable to find freq_table\n", __func__); - return -EINVAL; - } - - retval = cpufreq_frequency_table_target(policy, freq_table, target_freq, - relation, &index); - if (unlikely(retval)) { - pr_err("%s: Unable to find matching freq\n", __func__); - return retval; - } - - if (freq_table[index].frequency == policy->cur) - return 0; + index = cpufreq_frequency_table_target(policy, target_freq, relation); - return __target_index(policy, freq_table, index); + return __target_index(policy, index); } EXPORT_SYMBOL_GPL(__cpufreq_driver_target); @@ -1997,7 +1999,7 @@ __weak struct cpufreq_governor *cpufreq_fallback_governor(void) return NULL; } -static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event) +static int cpufreq_init_governor(struct cpufreq_policy *policy) { int ret; @@ -2025,36 +2027,82 @@ static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event) } } - if (event == CPUFREQ_GOV_POLICY_INIT) - if (!try_module_get(policy->governor->owner)) - return -EINVAL; - - pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event); + if (!try_module_get(policy->governor->owner)) + return -EINVAL; - ret = policy->governor->governor(policy, event); + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); - if (event == CPUFREQ_GOV_POLICY_INIT) { - if (ret) + if (policy->governor->init) { + ret = policy->governor->init(policy); + if (ret) { module_put(policy->governor->owner); - else - policy->governor->initialized++; - } else if (event == CPUFREQ_GOV_POLICY_EXIT) { - policy->governor->initialized--; - module_put(policy->governor->owner); + return ret; + } } - return ret; + return 0; +} + +static void cpufreq_exit_governor(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + if (policy->governor->exit) + policy->governor->exit(policy); + + module_put(policy->governor->owner); } static int cpufreq_start_governor(struct cpufreq_policy *policy) { int ret; + if (cpufreq_suspended) + return 0; + + if (!policy->governor) + return -EINVAL; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + if (cpufreq_driver->get && !cpufreq_driver->setpolicy) cpufreq_update_current_freq(policy); - ret = cpufreq_governor(policy, CPUFREQ_GOV_START); - return ret ? ret : cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); + if (policy->governor->start) { + ret = policy->governor->start(policy); + if (ret) + return ret; + } + + if (policy->governor->limits) + policy->governor->limits(policy); + + return 0; +} + +static void cpufreq_stop_governor(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + if (policy->governor->stop) + policy->governor->stop(policy); +} + +static void cpufreq_governor_limits(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + if (policy->governor->limits) + policy->governor->limits(policy); } int cpufreq_register_governor(struct cpufreq_governor *governor) @@ -2069,7 +2117,6 @@ int cpufreq_register_governor(struct cpufreq_governor *governor) mutex_lock(&cpufreq_governor_mutex); - governor->initialized = 0; err = -EBUSY; if (!find_governor(governor->name)) { err = 0; @@ -2184,6 +2231,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, policy->min = new_policy->min; policy->max = new_policy->max; + policy->cached_target_freq = UINT_MAX; + pr_debug("new min and max freqs are %u - %u kHz\n", policy->min, policy->max); @@ -2195,7 +2244,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, if (new_policy->governor == policy->governor) { pr_debug("cpufreq: governor limits update\n"); - return cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); + cpufreq_governor_limits(policy); + return 0; } pr_debug("governor switch\n"); @@ -2210,7 +2260,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, /* start new governor */ policy->governor = new_policy->governor; - ret = cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT); + ret = cpufreq_init_governor(policy); if (!ret) { ret = cpufreq_start_governor(policy); if (!ret) { @@ -2224,7 +2274,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, pr_debug("starting governor %s failed\n", policy->governor->name); if (old_gov) { policy->governor = old_gov; - if (cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) + if (cpufreq_init_governor(policy)) policy->governor = NULL; else cpufreq_start_governor(policy); @@ -2261,6 +2311,10 @@ int cpufreq_update_policy(unsigned int cpu) * -> ask driver for current freq and notify governors about a change */ if (cpufreq_driver->get && !cpufreq_driver->setpolicy) { + if (cpufreq_suspended) { + ret = -EAGAIN; + goto unlock; + } new_policy.cur = cpufreq_update_current_freq(policy); if (WARN_ON(!new_policy.cur)) { ret = -EIO; @@ -2305,26 +2359,25 @@ static struct notifier_block __refdata cpufreq_cpu_notifier = { *********************************************************************/ static int cpufreq_boost_set_sw(int state) { - struct cpufreq_frequency_table *freq_table; struct cpufreq_policy *policy; int ret = -EINVAL; for_each_active_policy(policy) { - freq_table = cpufreq_frequency_get_table(policy->cpu); - if (freq_table) { - ret = cpufreq_frequency_table_cpuinfo(policy, - freq_table); - if (ret) { - pr_err("%s: Policy frequency update failed\n", - __func__); - break; - } - - down_write(&policy->rwsem); - policy->user_policy.max = policy->max; - cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); - up_write(&policy->rwsem); + if (!policy->freq_table) + continue; + + ret = cpufreq_frequency_table_cpuinfo(policy, + policy->freq_table); + if (ret) { + pr_err("%s: Policy frequency update failed\n", + __func__); + break; } + + down_write(&policy->rwsem); + policy->user_policy.max = policy->max; + cpufreq_governor_limits(policy); + up_write(&policy->rwsem); } return ret; diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index 316df247e00d..18da4f8051d3 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -17,7 +17,6 @@ struct cs_policy_dbs_info { struct policy_dbs_info policy_dbs; unsigned int down_skip; - unsigned int requested_freq; }; static inline struct cs_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs) @@ -75,19 +74,17 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) /* Check for frequency increase */ if (load > dbs_data->up_threshold) { + unsigned int requested_freq = policy->cur; + dbs_info->down_skip = 0; /* if we are already at full speed then break out early */ - if (dbs_info->requested_freq == policy->max) + if (requested_freq == policy->max) goto out; - dbs_info->requested_freq += get_freq_target(cs_tuners, policy); - - if (dbs_info->requested_freq > policy->max) - dbs_info->requested_freq = policy->max; + requested_freq += get_freq_target(cs_tuners, policy); - __cpufreq_driver_target(policy, dbs_info->requested_freq, - CPUFREQ_RELATION_H); + __cpufreq_driver_target(policy, requested_freq, CPUFREQ_RELATION_H); goto out; } @@ -98,36 +95,27 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) /* Check for frequency decrease */ if (load < cs_tuners->down_threshold) { - unsigned int freq_target; + unsigned int freq_target, requested_freq = policy->cur; /* * if we cannot reduce the frequency anymore, break out early */ - if (policy->cur == policy->min) + if (requested_freq == policy->min) goto out; freq_target = get_freq_target(cs_tuners, policy); - if (dbs_info->requested_freq > freq_target) - dbs_info->requested_freq -= freq_target; + if (requested_freq > freq_target) + requested_freq -= freq_target; else - dbs_info->requested_freq = policy->min; + requested_freq = policy->min; - __cpufreq_driver_target(policy, dbs_info->requested_freq, - CPUFREQ_RELATION_L); + __cpufreq_driver_target(policy, requested_freq, CPUFREQ_RELATION_L); } out: return dbs_data->sampling_rate; } -static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, - void *data); - -static struct notifier_block cs_cpufreq_notifier_block = { - .notifier_call = dbs_cpufreq_notifier, -}; - /************************** sysfs interface ************************/ -static struct dbs_governor cs_dbs_gov; static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set, const char *buf, size_t count) @@ -268,15 +256,13 @@ static void cs_free(struct policy_dbs_info *policy_dbs) kfree(to_dbs_info(policy_dbs)); } -static int cs_init(struct dbs_data *dbs_data, bool notify) +static int cs_init(struct dbs_data *dbs_data) { struct cs_dbs_tuners *tuners; tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); - if (!tuners) { - pr_err("%s: kzalloc failed\n", __func__); + if (!tuners) return -ENOMEM; - } tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD; tuners->freq_step = DEF_FREQUENCY_STEP; @@ -288,19 +274,11 @@ static int cs_init(struct dbs_data *dbs_data, bool notify) dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10); - if (notify) - cpufreq_register_notifier(&cs_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - return 0; } -static void cs_exit(struct dbs_data *dbs_data, bool notify) +static void cs_exit(struct dbs_data *dbs_data) { - if (notify) - cpufreq_unregister_notifier(&cs_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - kfree(dbs_data->tuners); } @@ -309,16 +287,10 @@ static void cs_start(struct cpufreq_policy *policy) struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); dbs_info->down_skip = 0; - dbs_info->requested_freq = policy->cur; } -static struct dbs_governor cs_dbs_gov = { - .gov = { - .name = "conservative", - .governor = cpufreq_governor_dbs, - .max_transition_latency = TRANSITION_LATENCY_LIMIT, - .owner = THIS_MODULE, - }, +static struct dbs_governor cs_governor = { + .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"), .kobj_type = { .default_attrs = cs_attributes }, .gov_dbs_timer = cs_dbs_timer, .alloc = cs_alloc, @@ -328,33 +300,7 @@ static struct dbs_governor cs_dbs_gov = { .start = cs_start, }; -#define CPU_FREQ_GOV_CONSERVATIVE (&cs_dbs_gov.gov) - -static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, - void *data) -{ - struct cpufreq_freqs *freq = data; - struct cpufreq_policy *policy = cpufreq_cpu_get_raw(freq->cpu); - struct cs_policy_dbs_info *dbs_info; - - if (!policy) - return 0; - - /* policy isn't governed by conservative governor */ - if (policy->governor != CPU_FREQ_GOV_CONSERVATIVE) - return 0; - - dbs_info = to_dbs_info(policy->governor_data); - /* - * we only care if our internally tracked freq moves outside the 'valid' - * ranges of frequency available to us otherwise we do not change it - */ - if (dbs_info->requested_freq > policy->max - || dbs_info->requested_freq < policy->min) - dbs_info->requested_freq = freq->new; - - return 0; -} +#define CPU_FREQ_GOV_CONSERVATIVE (&cs_governor.gov) static int __init cpufreq_gov_dbs_init(void) { diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index be498d56dd69..e415349ab31b 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -336,17 +336,6 @@ static inline void gov_clear_update_util(struct cpufreq_policy *policy) synchronize_sched(); } -static void gov_cancel_work(struct cpufreq_policy *policy) -{ - struct policy_dbs_info *policy_dbs = policy->governor_data; - - gov_clear_update_util(policy_dbs->policy); - irq_work_sync(&policy_dbs->irq_work); - cancel_work_sync(&policy_dbs->work); - atomic_set(&policy_dbs->work_count, 0); - policy_dbs->work_in_progress = false; -} - static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy, struct dbs_governor *gov) { @@ -389,7 +378,7 @@ static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs, gov->free(policy_dbs); } -static int cpufreq_governor_init(struct cpufreq_policy *policy) +int cpufreq_dbs_governor_init(struct cpufreq_policy *policy) { struct dbs_governor *gov = dbs_governor_of(policy); struct dbs_data *dbs_data; @@ -429,7 +418,7 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy) gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list); - ret = gov->init(dbs_data, !policy->governor->initialized); + ret = gov->init(dbs_data); if (ret) goto free_policy_dbs_info; @@ -458,13 +447,13 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy) goto out; /* Failure, so roll back. */ - pr_err("cpufreq: Governor initialization failed (dbs_data kobject init error %d)\n", ret); + pr_err("initialization failed (dbs_data kobject init error %d)\n", ret); policy->governor_data = NULL; if (!have_governor_per_policy()) gov->gdbs_data = NULL; - gov->exit(dbs_data, !policy->governor->initialized); + gov->exit(dbs_data); kfree(dbs_data); free_policy_dbs_info: @@ -474,8 +463,9 @@ out: mutex_unlock(&gov_dbs_data_mutex); return ret; } +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_init); -static int cpufreq_governor_exit(struct cpufreq_policy *policy) +void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy) { struct dbs_governor *gov = dbs_governor_of(policy); struct policy_dbs_info *policy_dbs = policy->governor_data; @@ -493,17 +483,17 @@ static int cpufreq_governor_exit(struct cpufreq_policy *policy) if (!have_governor_per_policy()) gov->gdbs_data = NULL; - gov->exit(dbs_data, policy->governor->initialized == 1); + gov->exit(dbs_data); kfree(dbs_data); } free_policy_dbs_info(policy_dbs, gov); mutex_unlock(&gov_dbs_data_mutex); - return 0; } +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_exit); -static int cpufreq_governor_start(struct cpufreq_policy *policy) +int cpufreq_dbs_governor_start(struct cpufreq_policy *policy) { struct dbs_governor *gov = dbs_governor_of(policy); struct policy_dbs_info *policy_dbs = policy->governor_data; @@ -539,47 +529,28 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy) gov_set_update_util(policy_dbs, sampling_rate); return 0; } +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_start); -static int cpufreq_governor_stop(struct cpufreq_policy *policy) +void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy) { - gov_cancel_work(policy); - return 0; + struct policy_dbs_info *policy_dbs = policy->governor_data; + + gov_clear_update_util(policy_dbs->policy); + irq_work_sync(&policy_dbs->irq_work); + cancel_work_sync(&policy_dbs->work); + atomic_set(&policy_dbs->work_count, 0); + policy_dbs->work_in_progress = false; } +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop); -static int cpufreq_governor_limits(struct cpufreq_policy *policy) +void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) { struct policy_dbs_info *policy_dbs = policy->governor_data; mutex_lock(&policy_dbs->timer_mutex); - - if (policy->max < policy->cur) - __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); - else if (policy->min > policy->cur) - __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); - + cpufreq_policy_apply_limits(policy); gov_update_sample_delay(policy_dbs, 0); mutex_unlock(&policy_dbs->timer_mutex); - - return 0; -} - -int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event) -{ - if (event == CPUFREQ_GOV_POLICY_INIT) { - return cpufreq_governor_init(policy); - } else if (policy->governor_data) { - switch (event) { - case CPUFREQ_GOV_POLICY_EXIT: - return cpufreq_governor_exit(policy); - case CPUFREQ_GOV_START: - return cpufreq_governor_start(policy); - case CPUFREQ_GOV_STOP: - return cpufreq_governor_stop(policy); - case CPUFREQ_GOV_LIMITS: - return cpufreq_governor_limits(policy); - } - } - return -EINVAL; } -EXPORT_SYMBOL_GPL(cpufreq_governor_dbs); +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_limits); diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index 34eb214b6d57..ef1037e9c92b 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -138,8 +138,8 @@ struct dbs_governor { unsigned int (*gov_dbs_timer)(struct cpufreq_policy *policy); struct policy_dbs_info *(*alloc)(void); void (*free)(struct policy_dbs_info *policy_dbs); - int (*init)(struct dbs_data *dbs_data, bool notify); - void (*exit)(struct dbs_data *dbs_data, bool notify); + int (*init)(struct dbs_data *dbs_data); + void (*exit)(struct dbs_data *dbs_data); void (*start)(struct cpufreq_policy *policy); }; @@ -148,6 +148,25 @@ static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy return container_of(policy->governor, struct dbs_governor, gov); } +/* Governor callback routines */ +int cpufreq_dbs_governor_init(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy); +int cpufreq_dbs_governor_start(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy); + +#define CPUFREQ_DBS_GOVERNOR_INITIALIZER(_name_) \ + { \ + .name = _name_, \ + .max_transition_latency = TRANSITION_LATENCY_LIMIT, \ + .owner = THIS_MODULE, \ + .init = cpufreq_dbs_governor_init, \ + .exit = cpufreq_dbs_governor_exit, \ + .start = cpufreq_dbs_governor_start, \ + .stop = cpufreq_dbs_governor_stop, \ + .limits = cpufreq_dbs_governor_limits, \ + } + /* Governor specific operations */ struct od_ops { unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy, @@ -155,7 +174,6 @@ struct od_ops { }; unsigned int dbs_update(struct cpufreq_policy *policy); -int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event); void od_register_powersave_bias_handler(unsigned int (*f) (struct cpufreq_policy *, unsigned int, unsigned int), unsigned int powersave_bias); diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index 300163430516..3a1f49f5f4c6 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -65,34 +65,30 @@ static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy, { unsigned int freq_req, freq_reduc, freq_avg; unsigned int freq_hi, freq_lo; - unsigned int index = 0; + unsigned int index; unsigned int delay_hi_us; struct policy_dbs_info *policy_dbs = policy->governor_data; struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); struct dbs_data *dbs_data = policy_dbs->dbs_data; struct od_dbs_tuners *od_tuners = dbs_data->tuners; + struct cpufreq_frequency_table *freq_table = policy->freq_table; - if (!dbs_info->freq_table) { + if (!freq_table) { dbs_info->freq_lo = 0; dbs_info->freq_lo_delay_us = 0; return freq_next; } - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next, - relation, &index); - freq_req = dbs_info->freq_table[index].frequency; + index = cpufreq_frequency_table_target(policy, freq_next, relation); + freq_req = freq_table[index].frequency; freq_reduc = freq_req * od_tuners->powersave_bias / 1000; freq_avg = freq_req - freq_reduc; /* Find freq bounds for freq_avg in freq_table */ - index = 0; - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, - CPUFREQ_RELATION_H, &index); - freq_lo = dbs_info->freq_table[index].frequency; - index = 0; - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, - CPUFREQ_RELATION_L, &index); - freq_hi = dbs_info->freq_table[index].frequency; + index = cpufreq_table_find_index_h(policy, freq_avg); + freq_lo = freq_table[index].frequency; + index = cpufreq_table_find_index_l(policy, freq_avg); + freq_hi = freq_table[index].frequency; /* Find out how long we have to be in hi and lo freqs */ if (freq_hi == freq_lo) { @@ -113,7 +109,6 @@ static void ondemand_powersave_bias_init(struct cpufreq_policy *policy) { struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); - dbs_info->freq_table = cpufreq_frequency_get_table(policy->cpu); dbs_info->freq_lo = 0; } @@ -361,17 +356,15 @@ static void od_free(struct policy_dbs_info *policy_dbs) kfree(to_dbs_info(policy_dbs)); } -static int od_init(struct dbs_data *dbs_data, bool notify) +static int od_init(struct dbs_data *dbs_data) { struct od_dbs_tuners *tuners; u64 idle_time; int cpu; tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); - if (!tuners) { - pr_err("%s: kzalloc failed\n", __func__); + if (!tuners) return -ENOMEM; - } cpu = get_cpu(); idle_time = get_cpu_idle_time_us(cpu, NULL); @@ -402,7 +395,7 @@ static int od_init(struct dbs_data *dbs_data, bool notify) return 0; } -static void od_exit(struct dbs_data *dbs_data, bool notify) +static void od_exit(struct dbs_data *dbs_data) { kfree(dbs_data->tuners); } @@ -420,12 +413,7 @@ static struct od_ops od_ops = { }; static struct dbs_governor od_dbs_gov = { - .gov = { - .name = "ondemand", - .governor = cpufreq_governor_dbs, - .max_transition_latency = TRANSITION_LATENCY_LIMIT, - .owner = THIS_MODULE, - }, + .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"), .kobj_type = { .default_attrs = od_attributes }, .gov_dbs_timer = od_dbs_timer, .alloc = od_alloc, diff --git a/drivers/cpufreq/cpufreq_ondemand.h b/drivers/cpufreq/cpufreq_ondemand.h index f0121db3cd9e..640ea4e97106 100644 --- a/drivers/cpufreq/cpufreq_ondemand.h +++ b/drivers/cpufreq/cpufreq_ondemand.h @@ -13,7 +13,6 @@ struct od_policy_dbs_info { struct policy_dbs_info policy_dbs; - struct cpufreq_frequency_table *freq_table; unsigned int freq_lo; unsigned int freq_lo_delay_us; unsigned int freq_hi_delay_us; diff --git a/drivers/cpufreq/cpufreq_performance.c b/drivers/cpufreq/cpufreq_performance.c index af9f4b96f5a8..dafb679adc58 100644 --- a/drivers/cpufreq/cpufreq_performance.c +++ b/drivers/cpufreq/cpufreq_performance.c @@ -16,27 +16,16 @@ #include #include -static int cpufreq_governor_performance(struct cpufreq_policy *policy, - unsigned int event) +static void cpufreq_gov_performance_limits(struct cpufreq_policy *policy) { - switch (event) { - case CPUFREQ_GOV_START: - case CPUFREQ_GOV_LIMITS: - pr_debug("setting to %u kHz because of event %u\n", - policy->max, event); - __cpufreq_driver_target(policy, policy->max, - CPUFREQ_RELATION_H); - break; - default: - break; - } - return 0; + pr_debug("setting to %u kHz\n", policy->max); + __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); } static struct cpufreq_governor cpufreq_gov_performance = { .name = "performance", - .governor = cpufreq_governor_performance, .owner = THIS_MODULE, + .limits = cpufreq_gov_performance_limits, }; static int __init cpufreq_gov_performance_init(void) diff --git a/drivers/cpufreq/cpufreq_powersave.c b/drivers/cpufreq/cpufreq_powersave.c index b8b400232a74..78a651038faf 100644 --- a/drivers/cpufreq/cpufreq_powersave.c +++ b/drivers/cpufreq/cpufreq_powersave.c @@ -16,26 +16,15 @@ #include #include -static int cpufreq_governor_powersave(struct cpufreq_policy *policy, - unsigned int event) +static void cpufreq_gov_powersave_limits(struct cpufreq_policy *policy) { - switch (event) { - case CPUFREQ_GOV_START: - case CPUFREQ_GOV_LIMITS: - pr_debug("setting to %u kHz because of event %u\n", - policy->min, event); - __cpufreq_driver_target(policy, policy->min, - CPUFREQ_RELATION_L); - break; - default: - break; - } - return 0; + pr_debug("setting to %u kHz\n", policy->min); + __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); } static struct cpufreq_governor cpufreq_gov_powersave = { .name = "powersave", - .governor = cpufreq_governor_powersave, + .limits = cpufreq_gov_powersave_limits, .owner = THIS_MODULE, }; diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 5e370a30a964..06d3abdffd3a 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -15,7 +15,7 @@ #include #include -static spinlock_t cpufreq_stats_lock; +static DEFINE_SPINLOCK(cpufreq_stats_lock); struct cpufreq_stats { unsigned int total_trans; @@ -52,6 +52,9 @@ static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf) ssize_t len = 0; int i; + if (policy->fast_switch_enabled) + return 0; + cpufreq_stats_update(stats); for (i = 0; i < stats->state_num; i++) { len += sprintf(buf + len, "%u %llu\n", stats->freq_table[i], @@ -68,6 +71,9 @@ static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf) ssize_t len = 0; int i, j; + if (policy->fast_switch_enabled) + return 0; + len += snprintf(buf + len, PAGE_SIZE - len, " From : To\n"); len += snprintf(buf + len, PAGE_SIZE - len, " : "); for (i = 0; i < stats->state_num; i++) { @@ -130,7 +136,7 @@ static int freq_table_get_index(struct cpufreq_stats *stats, unsigned int freq) return -1; } -static void __cpufreq_stats_free_table(struct cpufreq_policy *policy) +void cpufreq_stats_free_table(struct cpufreq_policy *policy) { struct cpufreq_stats *stats = policy->stats; @@ -146,39 +152,25 @@ static void __cpufreq_stats_free_table(struct cpufreq_policy *policy) policy->stats = NULL; } -static void cpufreq_stats_free_table(unsigned int cpu) -{ - struct cpufreq_policy *policy; - - policy = cpufreq_cpu_get(cpu); - if (!policy) - return; - - __cpufreq_stats_free_table(policy); - - cpufreq_cpu_put(policy); -} - -static int __cpufreq_stats_create_table(struct cpufreq_policy *policy) +void cpufreq_stats_create_table(struct cpufreq_policy *policy) { unsigned int i = 0, count = 0, ret = -ENOMEM; struct cpufreq_stats *stats; unsigned int alloc_size; - unsigned int cpu = policy->cpu; struct cpufreq_frequency_table *pos, *table; /* We need cpufreq table for creating stats table */ - table = cpufreq_frequency_get_table(cpu); + table = policy->freq_table; if (unlikely(!table)) - return 0; + return; /* stats already initialized */ if (policy->stats) - return -EEXIST; + return; stats = kzalloc(sizeof(*stats), GFP_KERNEL); if (!stats) - return -ENOMEM; + return; /* Find total allocation size */ cpufreq_for_each_valid_entry(pos, table) @@ -215,80 +207,32 @@ static int __cpufreq_stats_create_table(struct cpufreq_policy *policy) policy->stats = stats; ret = sysfs_create_group(&policy->kobj, &stats_attr_group); if (!ret) - return 0; + return; /* We failed, release resources */ policy->stats = NULL; kfree(stats->time_in_state); free_stat: kfree(stats); - - return ret; -} - -static void cpufreq_stats_create_table(unsigned int cpu) -{ - struct cpufreq_policy *policy; - - /* - * "likely(!policy)" because normally cpufreq_stats will be registered - * before cpufreq driver - */ - policy = cpufreq_cpu_get(cpu); - if (likely(!policy)) - return; - - __cpufreq_stats_create_table(policy); - - cpufreq_cpu_put(policy); } -static int cpufreq_stat_notifier_policy(struct notifier_block *nb, - unsigned long val, void *data) +void cpufreq_stats_record_transition(struct cpufreq_policy *policy, + unsigned int new_freq) { - int ret = 0; - struct cpufreq_policy *policy = data; - - if (val == CPUFREQ_CREATE_POLICY) - ret = __cpufreq_stats_create_table(policy); - else if (val == CPUFREQ_REMOVE_POLICY) - __cpufreq_stats_free_table(policy); - - return ret; -} - -static int cpufreq_stat_notifier_trans(struct notifier_block *nb, - unsigned long val, void *data) -{ - struct cpufreq_freqs *freq = data; - struct cpufreq_policy *policy = cpufreq_cpu_get(freq->cpu); - struct cpufreq_stats *stats; + struct cpufreq_stats *stats = policy->stats; int old_index, new_index; - if (!policy) { - pr_err("%s: No policy found\n", __func__); - return 0; - } - - if (val != CPUFREQ_POSTCHANGE) - goto put_policy; - - if (!policy->stats) { + if (!stats) { pr_debug("%s: No stats found\n", __func__); - goto put_policy; + return; } - stats = policy->stats; - old_index = stats->last_index; - new_index = freq_table_get_index(stats, freq->new); + new_index = freq_table_get_index(stats, new_freq); /* We can't do stats->time_in_state[-1]= .. */ - if (old_index == -1 || new_index == -1) - goto put_policy; - - if (old_index == new_index) - goto put_policy; + if (old_index == -1 || new_index == -1 || old_index == new_index) + return; cpufreq_stats_update(stats); @@ -297,61 +241,4 @@ static int cpufreq_stat_notifier_trans(struct notifier_block *nb, stats->trans_table[old_index * stats->max_state + new_index]++; #endif stats->total_trans++; - -put_policy: - cpufreq_cpu_put(policy); - return 0; } - -static struct notifier_block notifier_policy_block = { - .notifier_call = cpufreq_stat_notifier_policy -}; - -static struct notifier_block notifier_trans_block = { - .notifier_call = cpufreq_stat_notifier_trans -}; - -static int __init cpufreq_stats_init(void) -{ - int ret; - unsigned int cpu; - - spin_lock_init(&cpufreq_stats_lock); - ret = cpufreq_register_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - if (ret) - return ret; - - for_each_online_cpu(cpu) - cpufreq_stats_create_table(cpu); - - ret = cpufreq_register_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); - if (ret) { - cpufreq_unregister_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - for_each_online_cpu(cpu) - cpufreq_stats_free_table(cpu); - return ret; - } - - return 0; -} -static void __exit cpufreq_stats_exit(void) -{ - unsigned int cpu; - - cpufreq_unregister_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - cpufreq_unregister_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); - for_each_online_cpu(cpu) - cpufreq_stats_free_table(cpu); -} - -MODULE_AUTHOR("Zou Nan hai "); -MODULE_DESCRIPTION("Export cpufreq stats via sysfs"); -MODULE_LICENSE("GPL"); - -module_init(cpufreq_stats_init); -module_exit(cpufreq_stats_exit); diff --git a/drivers/cpufreq/cpufreq_userspace.c b/drivers/cpufreq/cpufreq_userspace.c index 9f3dec9a3f36..bd897e3e134d 100644 --- a/drivers/cpufreq/cpufreq_userspace.c +++ b/drivers/cpufreq/cpufreq_userspace.c @@ -65,66 +65,66 @@ static int cpufreq_userspace_policy_init(struct cpufreq_policy *policy) return 0; } -static int cpufreq_governor_userspace(struct cpufreq_policy *policy, - unsigned int event) +static void cpufreq_userspace_policy_exit(struct cpufreq_policy *policy) +{ + mutex_lock(&userspace_mutex); + kfree(policy->governor_data); + policy->governor_data = NULL; + mutex_unlock(&userspace_mutex); +} + +static int cpufreq_userspace_policy_start(struct cpufreq_policy *policy) { unsigned int *setspeed = policy->governor_data; - unsigned int cpu = policy->cpu; - int rc = 0; - if (event == CPUFREQ_GOV_POLICY_INIT) - return cpufreq_userspace_policy_init(policy); + BUG_ON(!policy->cur); + pr_debug("started managing cpu %u\n", policy->cpu); - if (!setspeed) - return -EINVAL; - - switch (event) { - case CPUFREQ_GOV_POLICY_EXIT: - mutex_lock(&userspace_mutex); - policy->governor_data = NULL; - kfree(setspeed); - mutex_unlock(&userspace_mutex); - break; - case CPUFREQ_GOV_START: - BUG_ON(!policy->cur); - pr_debug("started managing cpu %u\n", cpu); - - mutex_lock(&userspace_mutex); - per_cpu(cpu_is_managed, cpu) = 1; - *setspeed = policy->cur; - mutex_unlock(&userspace_mutex); - break; - case CPUFREQ_GOV_STOP: - pr_debug("managing cpu %u stopped\n", cpu); - - mutex_lock(&userspace_mutex); - per_cpu(cpu_is_managed, cpu) = 0; - *setspeed = 0; - mutex_unlock(&userspace_mutex); - break; - case CPUFREQ_GOV_LIMITS: - mutex_lock(&userspace_mutex); - pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n", - cpu, policy->min, policy->max, policy->cur, *setspeed); - - if (policy->max < *setspeed) - __cpufreq_driver_target(policy, policy->max, - CPUFREQ_RELATION_H); - else if (policy->min > *setspeed) - __cpufreq_driver_target(policy, policy->min, - CPUFREQ_RELATION_L); - else - __cpufreq_driver_target(policy, *setspeed, - CPUFREQ_RELATION_L); - mutex_unlock(&userspace_mutex); - break; - } - return rc; + mutex_lock(&userspace_mutex); + per_cpu(cpu_is_managed, policy->cpu) = 1; + *setspeed = policy->cur; + mutex_unlock(&userspace_mutex); + return 0; +} + +static void cpufreq_userspace_policy_stop(struct cpufreq_policy *policy) +{ + unsigned int *setspeed = policy->governor_data; + + pr_debug("managing cpu %u stopped\n", policy->cpu); + + mutex_lock(&userspace_mutex); + per_cpu(cpu_is_managed, policy->cpu) = 0; + *setspeed = 0; + mutex_unlock(&userspace_mutex); +} + +static void cpufreq_userspace_policy_limits(struct cpufreq_policy *policy) +{ + unsigned int *setspeed = policy->governor_data; + + mutex_lock(&userspace_mutex); + + pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n", + policy->cpu, policy->min, policy->max, policy->cur, *setspeed); + + if (policy->max < *setspeed) + __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); + else if (policy->min > *setspeed) + __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); + else + __cpufreq_driver_target(policy, *setspeed, CPUFREQ_RELATION_L); + + mutex_unlock(&userspace_mutex); } static struct cpufreq_governor cpufreq_gov_userspace = { .name = "userspace", - .governor = cpufreq_governor_userspace, + .init = cpufreq_userspace_policy_init, + .exit = cpufreq_userspace_policy_exit, + .start = cpufreq_userspace_policy_start, + .stop = cpufreq_userspace_policy_stop, + .limits = cpufreq_userspace_policy_limits, .store_setspeed = cpufreq_set, .show_setspeed = show_speed, .owner = THIS_MODULE, diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 7e336d20c184..b95a872800ec 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -38,26 +38,6 @@ struct davinci_cpufreq { }; static struct davinci_cpufreq cpufreq; -static int davinci_verify_speed(struct cpufreq_policy *policy) -{ - struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data; - struct cpufreq_frequency_table *freq_table = pdata->freq_table; - struct clk *armclk = cpufreq.armclk; - - if (freq_table) - return cpufreq_frequency_table_verify(policy, freq_table); - - if (policy->cpu) - return -EINVAL; - - cpufreq_verify_within_cpu_limits(policy); - policy->min = clk_round_rate(armclk, policy->min * 1000) / 1000; - policy->max = clk_round_rate(armclk, policy->max * 1000) / 1000; - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - return 0; -} - static int davinci_target(struct cpufreq_policy *policy, unsigned int idx) { struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data; @@ -121,7 +101,7 @@ static int davinci_cpu_init(struct cpufreq_policy *policy) static struct cpufreq_driver davinci_driver = { .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, - .verify = davinci_verify_speed, + .verify = cpufreq_generic_frequency_table_verify, .target_index = davinci_target, .get = cpufreq_generic_get, .init = davinci_cpu_init, diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index a8f1daffc9bc..3bbbf9e6960c 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -63,8 +63,6 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, else return 0; } -EXPORT_SYMBOL_GPL(cpufreq_frequency_table_cpuinfo); - int cpufreq_frequency_table_verify(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table) @@ -108,20 +106,16 @@ EXPORT_SYMBOL_GPL(cpufreq_frequency_table_verify); */ int cpufreq_generic_frequency_table_verify(struct cpufreq_policy *policy) { - struct cpufreq_frequency_table *table = - cpufreq_frequency_get_table(policy->cpu); - if (!table) + if (!policy->freq_table) return -ENODEV; - return cpufreq_frequency_table_verify(policy, table); + return cpufreq_frequency_table_verify(policy, policy->freq_table); } EXPORT_SYMBOL_GPL(cpufreq_generic_frequency_table_verify); -int cpufreq_frequency_table_target(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table, - unsigned int target_freq, - unsigned int relation, - unsigned int *index) +int cpufreq_table_index_unsorted(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) { struct cpufreq_frequency_table optimal = { .driver_data = ~0, @@ -132,7 +126,9 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, .frequency = 0, }; struct cpufreq_frequency_table *pos; + struct cpufreq_frequency_table *table = policy->freq_table; unsigned int freq, diff, i = 0; + int index; pr_debug("request for target %u kHz (relation: %u) for cpu %u\n", target_freq, relation, policy->cpu); @@ -196,25 +192,26 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, } } if (optimal.driver_data > i) { - if (suboptimal.driver_data > i) - return -EINVAL; - *index = suboptimal.driver_data; - } else - *index = optimal.driver_data; + if (suboptimal.driver_data > i) { + WARN(1, "Invalid frequency table: %d\n", policy->cpu); + return 0; + } - pr_debug("target index is %u, freq is:%u kHz\n", *index, - table[*index].frequency); + index = suboptimal.driver_data; + } else + index = optimal.driver_data; - return 0; + pr_debug("target index is %u, freq is:%u kHz\n", index, + table[index].frequency); + return index; } -EXPORT_SYMBOL_GPL(cpufreq_frequency_table_target); +EXPORT_SYMBOL_GPL(cpufreq_table_index_unsorted); int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy, unsigned int freq) { - struct cpufreq_frequency_table *pos, *table; + struct cpufreq_frequency_table *pos, *table = policy->freq_table; - table = cpufreq_frequency_get_table(policy->cpu); if (unlikely(!table)) { pr_debug("%s: Unable to find frequency table\n", __func__); return -ENOENT; @@ -300,15 +297,72 @@ struct freq_attr *cpufreq_generic_attr[] = { }; EXPORT_SYMBOL_GPL(cpufreq_generic_attr); +static int set_freq_table_sorted(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *pos, *table = policy->freq_table; + struct cpufreq_frequency_table *prev = NULL; + int ascending = 0; + + policy->freq_table_sorted = CPUFREQ_TABLE_UNSORTED; + + cpufreq_for_each_valid_entry(pos, table) { + if (!prev) { + prev = pos; + continue; + } + + if (pos->frequency == prev->frequency) { + pr_warn("Duplicate freq-table entries: %u\n", + pos->frequency); + return -EINVAL; + } + + /* Frequency increased from prev to pos */ + if (pos->frequency > prev->frequency) { + /* But frequency was decreasing earlier */ + if (ascending < 0) { + pr_debug("Freq table is unsorted\n"); + return 0; + } + + ascending++; + } else { + /* Frequency decreased from prev to pos */ + + /* But frequency was increasing earlier */ + if (ascending > 0) { + pr_debug("Freq table is unsorted\n"); + return 0; + } + + ascending--; + } + + prev = pos; + } + + if (ascending > 0) + policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_ASCENDING; + else + policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_DESCENDING; + + pr_debug("Freq table is sorted in %s order\n", + ascending > 0 ? "ascending" : "descending"); + + return 0; +} + int cpufreq_table_validate_and_show(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table) { - int ret = cpufreq_frequency_table_cpuinfo(policy, table); + int ret; - if (!ret) - policy->freq_table = table; + ret = cpufreq_frequency_table_cpuinfo(policy, table); + if (ret) + return ret; - return ret; + policy->freq_table = table; + return set_freq_table_sorted(policy); } EXPORT_SYMBOL_GPL(cpufreq_table_validate_and_show); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index fe9dc17ea873..9ec033b4f2d9 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -35,6 +35,7 @@ #include #include #include +#include #define ATOM_RATIOS 0x66a #define ATOM_VIDS 0x66b @@ -96,7 +97,6 @@ static inline u64 div_ext_fp(u64 x, u64 y) * read from MPERF MSR between last and current sample * @tsc: Difference of time stamp counter between last and * current sample - * @freq: Effective frequency calculated from APERF/MPERF * @time: Current time from scheduler * * This structure is used in the cpudata structure to store performance sample @@ -108,7 +108,6 @@ struct sample { u64 aperf; u64 mperf; u64 tsc; - int freq; u64 time; }; @@ -281,9 +280,9 @@ struct cpu_defaults { static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu); static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu); -static struct pstate_adjust_policy pid_params; -static struct pstate_funcs pstate_funcs; -static int hwp_active; +static struct pstate_adjust_policy pid_params __read_mostly; +static struct pstate_funcs pstate_funcs __read_mostly; +static int hwp_active __read_mostly; #ifdef CONFIG_ACPI static bool acpi_ppc; @@ -807,7 +806,8 @@ static void __init intel_pstate_sysfs_expose_params(void) static void intel_pstate_hwp_enable(struct cpudata *cpudata) { /* First disable HWP notification interrupt as we don't process them */ - wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); + if (static_cpu_has(X86_FEATURE_HWP_NOTIFY)) + wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); } @@ -944,7 +944,7 @@ static int core_get_max_pstate(void) if (err) goto skip_tar; - tdp_msr = MSR_CONFIG_TDP_NOMINAL + tdp_ctrl; + tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x3); err = rdmsrl_safe(tdp_msr, &tdp_ratio); if (err) goto skip_tar; @@ -972,7 +972,7 @@ static int core_get_turbo_pstate(void) u64 value; int nont, ret; - rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value); + rdmsrl(MSR_TURBO_RATIO_LIMIT, value); nont = core_get_max_pstate(); ret = (value) & 255; if (ret <= nont) @@ -1001,7 +1001,7 @@ static int knl_get_turbo_pstate(void) u64 value; int nont, ret; - rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value); + rdmsrl(MSR_TURBO_RATIO_LIMIT, value); nont = core_get_max_pstate(); ret = (((value) >> 8) & 0xFF); if (ret <= nont) @@ -1091,6 +1091,26 @@ static struct cpu_defaults knl_params = { }, }; +static struct cpu_defaults bxt_params = { + .pid_policy = { + .sample_rate_ms = 10, + .deadband = 0, + .setpoint = 60, + .p_gain_pct = 14, + .d_gain_pct = 0, + .i_gain_pct = 4, + }, + .funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = core_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, + .get_target_pstate = get_target_pstate_use_cpu_load, + }, +}; + static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) { int max_perf = cpu->pstate.turbo_pstate; @@ -1113,17 +1133,12 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf); } -static inline void intel_pstate_record_pstate(struct cpudata *cpu, int pstate) -{ - trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); - cpu->pstate.current_pstate = pstate; -} - static void intel_pstate_set_min_pstate(struct cpudata *cpu) { int pstate = cpu->pstate.min_pstate; - intel_pstate_record_pstate(cpu, pstate); + trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); + cpu->pstate.current_pstate = pstate; /* * Generally, there is no guarantee that this code will always run on * the CPU being updated, so force the register update to run on the @@ -1283,10 +1298,11 @@ static inline void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) intel_pstate_get_min_max(cpu, &min_perf, &max_perf); pstate = clamp_t(int, pstate, min_perf, max_perf); + trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); if (pstate == cpu->pstate.current_pstate) return; - intel_pstate_record_pstate(cpu, pstate); + cpu->pstate.current_pstate = pstate; wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); } @@ -1334,29 +1350,30 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time, (unsigned long)&policy } static const struct x86_cpu_id intel_pstate_cpu_ids[] = { - ICPU(0x2a, core_params), - ICPU(0x2d, core_params), - ICPU(0x37, silvermont_params), - ICPU(0x3a, core_params), - ICPU(0x3c, core_params), - ICPU(0x3d, core_params), - ICPU(0x3e, core_params), - ICPU(0x3f, core_params), - ICPU(0x45, core_params), - ICPU(0x46, core_params), - ICPU(0x47, core_params), - ICPU(0x4c, airmont_params), - ICPU(0x4e, core_params), - ICPU(0x4f, core_params), - ICPU(0x5e, core_params), - ICPU(0x56, core_params), - ICPU(0x57, knl_params), + ICPU(INTEL_FAM6_SANDYBRIDGE, core_params), + ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_params), + ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_params), + ICPU(INTEL_FAM6_IVYBRIDGE, core_params), + ICPU(INTEL_FAM6_HASWELL_CORE, core_params), + ICPU(INTEL_FAM6_BROADWELL_CORE, core_params), + ICPU(INTEL_FAM6_IVYBRIDGE_X, core_params), + ICPU(INTEL_FAM6_HASWELL_X, core_params), + ICPU(INTEL_FAM6_HASWELL_ULT, core_params), + ICPU(INTEL_FAM6_HASWELL_GT3E, core_params), + ICPU(INTEL_FAM6_BROADWELL_GT3E, core_params), + ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_params), + ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_params), + ICPU(INTEL_FAM6_BROADWELL_X, core_params), + ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params), + ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), + ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params), + ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params), {} }; MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); -static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] = { - ICPU(0x56, core_params), +static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { + ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), {} }; @@ -1400,6 +1417,9 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num) { struct cpudata *cpu = all_cpu_data[cpu_num]; + if (cpu->update_util_set) + return; + /* Prevent intel_pstate_update_util() from using stale data. */ cpu->sample.time = 0; cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, @@ -1440,8 +1460,6 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) if (!policy->cpuinfo.max_freq) return -ENODEV; - intel_pstate_clear_update_util_hook(policy->cpu); - pr_debug("set_policy cpuinfo.max %u policy->max %u\n", policy->cpuinfo.max_freq, policy->max); @@ -1574,12 +1592,12 @@ static struct cpufreq_driver intel_pstate_driver = { .name = "intel_pstate", }; -static int __initdata no_load; -static int __initdata no_hwp; -static int __initdata hwp_only; -static unsigned int force_load; +static int no_load __initdata; +static int no_hwp __initdata; +static int hwp_only __initdata; +static unsigned int force_load __initdata; -static int intel_pstate_msrs_not_valid(void) +static int __init intel_pstate_msrs_not_valid(void) { if (!pstate_funcs.get_max() || !pstate_funcs.get_min() || @@ -1589,7 +1607,7 @@ static int intel_pstate_msrs_not_valid(void) return 0; } -static void copy_pid_params(struct pstate_adjust_policy *policy) +static void __init copy_pid_params(struct pstate_adjust_policy *policy) { pid_params.sample_rate_ms = policy->sample_rate_ms; pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; @@ -1600,7 +1618,7 @@ static void copy_pid_params(struct pstate_adjust_policy *policy) pid_params.setpoint = policy->setpoint; } -static void copy_cpu_funcs(struct pstate_funcs *funcs) +static void __init copy_cpu_funcs(struct pstate_funcs *funcs) { pstate_funcs.get_max = funcs->get_max; pstate_funcs.get_max_physical = funcs->get_max_physical; @@ -1615,7 +1633,7 @@ static void copy_cpu_funcs(struct pstate_funcs *funcs) #ifdef CONFIG_ACPI -static bool intel_pstate_no_acpi_pss(void) +static bool __init intel_pstate_no_acpi_pss(void) { int i; @@ -1644,7 +1662,7 @@ static bool intel_pstate_no_acpi_pss(void) return true; } -static bool intel_pstate_has_acpi_ppc(void) +static bool __init intel_pstate_has_acpi_ppc(void) { int i; @@ -1672,7 +1690,7 @@ struct hw_vendor_info { }; /* Hardware vendor-specific info that has its own power management modes */ -static struct hw_vendor_info vendor_info[] = { +static struct hw_vendor_info vendor_info[] __initdata = { {1, "HP ", "ProLiant", PSS}, {1, "ORACLE", "X4-2 ", PPC}, {1, "ORACLE", "X4-2L ", PPC}, @@ -1691,7 +1709,7 @@ static struct hw_vendor_info vendor_info[] = { {0, "", ""}, }; -static bool intel_pstate_platform_pwr_mgmt_exists(void) +static bool __init intel_pstate_platform_pwr_mgmt_exists(void) { struct acpi_table_header hdr; struct hw_vendor_info *v_info; diff --git a/drivers/cpufreq/mvebu-cpufreq.c b/drivers/cpufreq/mvebu-cpufreq.c index e920889b9ac2..ed915ee85dd9 100644 --- a/drivers/cpufreq/mvebu-cpufreq.c +++ b/drivers/cpufreq/mvebu-cpufreq.c @@ -70,7 +70,7 @@ static int __init armada_xp_pmsu_cpufreq_init(void) continue; } - clk = clk_get(cpu_dev, 0); + clk = clk_get(cpu_dev, NULL); if (IS_ERR(clk)) { pr_err("Cannot get clock for CPU %d\n", cpu); return PTR_ERR(clk); diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c index 808a320e9d5d..3f0ce2ae35ee 100644 --- a/drivers/cpufreq/pcc-cpufreq.c +++ b/drivers/cpufreq/pcc-cpufreq.c @@ -487,7 +487,7 @@ static int __init pcc_cpufreq_probe(void) doorbell.space_id = reg_resource->space_id; doorbell.bit_width = reg_resource->bit_width; doorbell.bit_offset = reg_resource->bit_offset; - doorbell.access_width = 64; + doorbell.access_width = 4; doorbell.address = reg_resource->address; pr_debug("probe: doorbell: space_id is %d, bit_width is %d, " @@ -555,8 +555,6 @@ static int pcc_cpufreq_cpu_init(struct cpufreq_policy *policy) policy->min = policy->cpuinfo.min_freq = ioread32(&pcch_hdr->minimum_frequency) * 1000; - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - pr_debug("init: policy->max is %d, policy->min is %d\n", policy->max, policy->min); out: diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c index 54c45368e3f1..87796e0864e9 100644 --- a/drivers/cpufreq/powernv-cpufreq.c +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -64,12 +64,14 @@ /** * struct global_pstate_info - Per policy data structure to maintain history of * global pstates - * @highest_lpstate: The local pstate from which we are ramping down + * @highest_lpstate_idx: The local pstate index from which we are + * ramping down * @elapsed_time: Time in ms spent in ramping down from - * highest_lpstate + * highest_lpstate_idx * @last_sampled_time: Time from boot in ms when global pstates were * last set - * @last_lpstate,last_gpstate: Last set values for local and global pstates + * @last_lpstate_idx, Last set value of local pstate and global + * last_gpstate_idx pstate in terms of cpufreq table index * @timer: Is used for ramping down if cpu goes idle for * a long time with global pstate held high * @gpstate_lock: A spinlock to maintain synchronization between @@ -77,11 +79,11 @@ * governer's target_index calls */ struct global_pstate_info { - int highest_lpstate; + int highest_lpstate_idx; unsigned int elapsed_time; unsigned int last_sampled_time; - int last_lpstate; - int last_gpstate; + int last_lpstate_idx; + int last_gpstate_idx; spinlock_t gpstate_lock; struct timer_list timer; }; @@ -124,29 +126,47 @@ static int nr_chips; static DEFINE_PER_CPU(struct chip *, chip_info); /* - * Note: The set of pstates consists of contiguous integers, the - * smallest of which is indicated by powernv_pstate_info.min, the - * largest of which is indicated by powernv_pstate_info.max. + * Note: + * The set of pstates consists of contiguous integers. + * powernv_pstate_info stores the index of the frequency table for + * max, min and nominal frequencies. It also stores number of + * available frequencies. * - * The nominal pstate is the highest non-turbo pstate in this - * platform. This is indicated by powernv_pstate_info.nominal. + * powernv_pstate_info.nominal indicates the index to the highest + * non-turbo frequency. */ static struct powernv_pstate_info { - int min; - int max; - int nominal; - int nr_pstates; + unsigned int min; + unsigned int max; + unsigned int nominal; + unsigned int nr_pstates; } powernv_pstate_info; +/* Use following macros for conversions between pstate_id and index */ +static inline int idx_to_pstate(unsigned int i) +{ + return powernv_freqs[i].driver_data; +} + +static inline unsigned int pstate_to_idx(int pstate) +{ + /* + * abs() is deliberately used so that is works with + * both monotonically increasing and decreasing + * pstate values + */ + return abs(pstate - idx_to_pstate(powernv_pstate_info.max)); +} + static inline void reset_gpstates(struct cpufreq_policy *policy) { struct global_pstate_info *gpstates = policy->driver_data; - gpstates->highest_lpstate = 0; + gpstates->highest_lpstate_idx = 0; gpstates->elapsed_time = 0; gpstates->last_sampled_time = 0; - gpstates->last_lpstate = 0; - gpstates->last_gpstate = 0; + gpstates->last_lpstate_idx = 0; + gpstates->last_gpstate_idx = 0; } /* @@ -156,9 +176,10 @@ static inline void reset_gpstates(struct cpufreq_policy *policy) static int init_powernv_pstates(void) { struct device_node *power_mgt; - int i, pstate_min, pstate_max, pstate_nominal, nr_pstates = 0; + int i, nr_pstates = 0; const __be32 *pstate_ids, *pstate_freqs; u32 len_ids, len_freqs; + u32 pstate_min, pstate_max, pstate_nominal; power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); if (!power_mgt) { @@ -208,6 +229,7 @@ static int init_powernv_pstates(void) return -ENODEV; } + powernv_pstate_info.nr_pstates = nr_pstates; pr_debug("NR PStates %d\n", nr_pstates); for (i = 0; i < nr_pstates; i++) { u32 id = be32_to_cpu(pstate_ids[i]); @@ -216,15 +238,17 @@ static int init_powernv_pstates(void) pr_debug("PState id %d freq %d MHz\n", id, freq); powernv_freqs[i].frequency = freq * 1000; /* kHz */ powernv_freqs[i].driver_data = id; + + if (id == pstate_max) + powernv_pstate_info.max = i; + else if (id == pstate_nominal) + powernv_pstate_info.nominal = i; + else if (id == pstate_min) + powernv_pstate_info.min = i; } + /* End of list marker entry */ powernv_freqs[i].frequency = CPUFREQ_TABLE_END; - - powernv_pstate_info.min = pstate_min; - powernv_pstate_info.max = pstate_max; - powernv_pstate_info.nominal = pstate_nominal; - powernv_pstate_info.nr_pstates = nr_pstates; - return 0; } @@ -233,12 +257,12 @@ static unsigned int pstate_id_to_freq(int pstate_id) { int i; - i = powernv_pstate_info.max - pstate_id; + i = pstate_to_idx(pstate_id); if (i >= powernv_pstate_info.nr_pstates || i < 0) { pr_warn("PState id %d outside of PState table, " "reporting nominal id %d instead\n", - pstate_id, powernv_pstate_info.nominal); - i = powernv_pstate_info.max - powernv_pstate_info.nominal; + pstate_id, idx_to_pstate(powernv_pstate_info.nominal)); + i = powernv_pstate_info.nominal; } return powernv_freqs[i].frequency; @@ -252,7 +276,7 @@ static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy, char *buf) { return sprintf(buf, "%u\n", - pstate_id_to_freq(powernv_pstate_info.nominal)); + powernv_freqs[powernv_pstate_info.nominal].frequency); } struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq = @@ -426,7 +450,7 @@ static void set_pstate(void *data) */ static inline unsigned int get_nominal_index(void) { - return powernv_pstate_info.max - powernv_pstate_info.nominal; + return powernv_pstate_info.nominal; } static void powernv_cpufreq_throttle_check(void *data) @@ -435,20 +459,22 @@ static void powernv_cpufreq_throttle_check(void *data) unsigned int cpu = smp_processor_id(); unsigned long pmsr; int pmsr_pmax; + unsigned int pmsr_pmax_idx; pmsr = get_pmspr(SPRN_PMSR); chip = this_cpu_read(chip_info); /* Check for Pmax Capping */ pmsr_pmax = (s8)PMSR_MAX(pmsr); - if (pmsr_pmax != powernv_pstate_info.max) { + pmsr_pmax_idx = pstate_to_idx(pmsr_pmax); + if (pmsr_pmax_idx != powernv_pstate_info.max) { if (chip->throttled) goto next; chip->throttled = true; - if (pmsr_pmax < powernv_pstate_info.nominal) { - pr_warn_once("CPU %d on Chip %u has Pmax reduced below nominal frequency (%d < %d)\n", + if (pmsr_pmax_idx > powernv_pstate_info.nominal) { + pr_warn_once("CPU %d on Chip %u has Pmax(%d) reduced below nominal frequency(%d)\n", cpu, chip->id, pmsr_pmax, - powernv_pstate_info.nominal); + idx_to_pstate(powernv_pstate_info.nominal)); chip->throttle_sub_turbo++; } else { chip->throttle_turbo++; @@ -484,34 +510,35 @@ next: /** * calc_global_pstate - Calculate global pstate - * @elapsed_time: Elapsed time in milliseconds - * @local_pstate: New local pstate - * @highest_lpstate: pstate from which its ramping down + * @elapsed_time: Elapsed time in milliseconds + * @local_pstate_idx: New local pstate + * @highest_lpstate_idx: pstate from which its ramping down * * Finds the appropriate global pstate based on the pstate from which its * ramping down and the time elapsed in ramping down. It follows a quadratic * equation which ensures that it reaches ramping down to pmin in 5sec. */ static inline int calc_global_pstate(unsigned int elapsed_time, - int highest_lpstate, int local_pstate) + int highest_lpstate_idx, + int local_pstate_idx) { - int pstate_diff; + int index_diff; /* * Using ramp_down_percent we get the percentage of rampdown * that we are expecting to be dropping. Difference between - * highest_lpstate and powernv_pstate_info.min will give a absolute + * highest_lpstate_idx and powernv_pstate_info.min will give a absolute * number of how many pstates we will drop eventually by the end of * 5 seconds, then just scale it get the number pstates to be dropped. */ - pstate_diff = ((int)ramp_down_percent(elapsed_time) * - (highest_lpstate - powernv_pstate_info.min)) / 100; + index_diff = ((int)ramp_down_percent(elapsed_time) * + (powernv_pstate_info.min - highest_lpstate_idx)) / 100; /* Ensure that global pstate is >= to local pstate */ - if (highest_lpstate - pstate_diff < local_pstate) - return local_pstate; + if (highest_lpstate_idx + index_diff >= local_pstate_idx) + return local_pstate_idx; else - return highest_lpstate - pstate_diff; + return highest_lpstate_idx + index_diff; } static inline void queue_gpstate_timer(struct global_pstate_info *gpstates) @@ -530,8 +557,7 @@ static inline void queue_gpstate_timer(struct global_pstate_info *gpstates) else timer_interval = GPSTATE_TIMER_INTERVAL; - mod_timer_pinned(&gpstates->timer, jiffies + - msecs_to_jiffies(timer_interval)); + mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval)); } /** @@ -547,7 +573,7 @@ void gpstate_timer_handler(unsigned long data) { struct cpufreq_policy *policy = (struct cpufreq_policy *)data; struct global_pstate_info *gpstates = policy->driver_data; - int gpstate_id; + int gpstate_idx; unsigned int time_diff = jiffies_to_msecs(jiffies) - gpstates->last_sampled_time; struct powernv_smp_call_data freq_data; @@ -557,29 +583,29 @@ void gpstate_timer_handler(unsigned long data) gpstates->last_sampled_time += time_diff; gpstates->elapsed_time += time_diff; - freq_data.pstate_id = gpstates->last_lpstate; + freq_data.pstate_id = idx_to_pstate(gpstates->last_lpstate_idx); - if ((gpstates->last_gpstate == freq_data.pstate_id) || + if ((gpstates->last_gpstate_idx == gpstates->last_lpstate_idx) || (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME)) { - gpstate_id = freq_data.pstate_id; + gpstate_idx = pstate_to_idx(freq_data.pstate_id); reset_gpstates(policy); - gpstates->highest_lpstate = freq_data.pstate_id; + gpstates->highest_lpstate_idx = gpstate_idx; } else { - gpstate_id = calc_global_pstate(gpstates->elapsed_time, - gpstates->highest_lpstate, - freq_data.pstate_id); + gpstate_idx = calc_global_pstate(gpstates->elapsed_time, + gpstates->highest_lpstate_idx, + freq_data.pstate_id); } /* * If local pstate is equal to global pstate, rampdown is over * So timer is not required to be queued. */ - if (gpstate_id != freq_data.pstate_id) + if (gpstate_idx != gpstates->last_lpstate_idx) queue_gpstate_timer(gpstates); - freq_data.gpstate_id = gpstate_id; - gpstates->last_gpstate = freq_data.gpstate_id; - gpstates->last_lpstate = freq_data.pstate_id; + freq_data.gpstate_id = idx_to_pstate(gpstate_idx); + gpstates->last_gpstate_idx = pstate_to_idx(freq_data.gpstate_id); + gpstates->last_lpstate_idx = pstate_to_idx(freq_data.pstate_id); spin_unlock(&gpstates->gpstate_lock); @@ -596,7 +622,7 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, unsigned int new_index) { struct powernv_smp_call_data freq_data; - unsigned int cur_msec, gpstate_id; + unsigned int cur_msec, gpstate_idx; struct global_pstate_info *gpstates = policy->driver_data; if (unlikely(rebooting) && new_index != get_nominal_index()) @@ -608,15 +634,15 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, cur_msec = jiffies_to_msecs(get_jiffies_64()); spin_lock(&gpstates->gpstate_lock); - freq_data.pstate_id = powernv_freqs[new_index].driver_data; + freq_data.pstate_id = idx_to_pstate(new_index); if (!gpstates->last_sampled_time) { - gpstate_id = freq_data.pstate_id; - gpstates->highest_lpstate = freq_data.pstate_id; + gpstate_idx = new_index; + gpstates->highest_lpstate_idx = new_index; goto gpstates_done; } - if (gpstates->last_gpstate > freq_data.pstate_id) { + if (gpstates->last_gpstate_idx < new_index) { gpstates->elapsed_time += cur_msec - gpstates->last_sampled_time; @@ -627,34 +653,34 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, */ if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) { reset_gpstates(policy); - gpstates->highest_lpstate = freq_data.pstate_id; - gpstate_id = freq_data.pstate_id; + gpstates->highest_lpstate_idx = new_index; + gpstate_idx = new_index; } else { /* Elaspsed_time is less than 5 seconds, continue to rampdown */ - gpstate_id = calc_global_pstate(gpstates->elapsed_time, - gpstates->highest_lpstate, - freq_data.pstate_id); + gpstate_idx = calc_global_pstate(gpstates->elapsed_time, + gpstates->highest_lpstate_idx, + new_index); } } else { reset_gpstates(policy); - gpstates->highest_lpstate = freq_data.pstate_id; - gpstate_id = freq_data.pstate_id; + gpstates->highest_lpstate_idx = new_index; + gpstate_idx = new_index; } /* * If local pstate is equal to global pstate, rampdown is over * So timer is not required to be queued. */ - if (gpstate_id != freq_data.pstate_id) + if (gpstate_idx != new_index) queue_gpstate_timer(gpstates); else del_timer_sync(&gpstates->timer); gpstates_done: - freq_data.gpstate_id = gpstate_id; + freq_data.gpstate_id = idx_to_pstate(gpstate_idx); gpstates->last_sampled_time = cur_msec; - gpstates->last_gpstate = freq_data.gpstate_id; - gpstates->last_lpstate = freq_data.pstate_id; + gpstates->last_gpstate_idx = gpstate_idx; + gpstates->last_lpstate_idx = new_index; spin_unlock(&gpstates->gpstate_lock); @@ -699,7 +725,7 @@ static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy) policy->driver_data = gpstates; /* initialize timer */ - init_timer_deferrable(&gpstates->timer); + init_timer_pinned_deferrable(&gpstates->timer); gpstates->timer.data = (unsigned long)policy; gpstates->timer.function = gpstate_timer_handler; gpstates->timer.expires = jiffies + @@ -760,9 +786,7 @@ void powernv_cpufreq_work_fn(struct work_struct *work) struct cpufreq_policy policy; cpufreq_get_policy(&policy, cpu); - cpufreq_frequency_table_target(&policy, policy.freq_table, - policy.cur, - CPUFREQ_RELATION_C, &index); + index = cpufreq_table_find_index_c(&policy, policy.cur); powernv_cpufreq_target_index(&policy, index); cpumask_andnot(&mask, &mask, policy.cpus); } @@ -848,8 +872,8 @@ static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy) struct powernv_smp_call_data freq_data; struct global_pstate_info *gpstates = policy->driver_data; - freq_data.pstate_id = powernv_pstate_info.min; - freq_data.gpstate_id = powernv_pstate_info.min; + freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min); + freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min); smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1); del_timer_sync(&gpstates->timer); } diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c index 7c4cd5c634f2..dc112481a408 100644 --- a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c +++ b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c @@ -94,7 +94,7 @@ static int pmi_notifier(struct notifier_block *nb, unsigned long event, void *data) { struct cpufreq_policy *policy = data; - struct cpufreq_frequency_table *cbe_freqs; + struct cpufreq_frequency_table *cbe_freqs = policy->freq_table; u8 node; /* Should this really be called for CPUFREQ_ADJUST and CPUFREQ_NOTIFY @@ -103,7 +103,6 @@ static int pmi_notifier(struct notifier_block *nb, if (event == CPUFREQ_START) return 0; - cbe_freqs = cpufreq_frequency_get_table(policy->cpu); node = cbe_cpu_to_node(policy->cpu); pr_debug("got notified, event=%lu, node=%u\n", event, node); diff --git a/drivers/cpufreq/s3c24xx-cpufreq.c b/drivers/cpufreq/s3c24xx-cpufreq.c index ae8eaed77b70..7b596fa38ad2 100644 --- a/drivers/cpufreq/s3c24xx-cpufreq.c +++ b/drivers/cpufreq/s3c24xx-cpufreq.c @@ -293,12 +293,8 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy, __func__, policy, target_freq, relation); if (ftab) { - if (cpufreq_frequency_table_target(policy, ftab, - target_freq, relation, - &index)) { - s3c_freq_dbg("%s: table failed\n", __func__); - return -EINVAL; - } + index = cpufreq_frequency_table_target(policy, target_freq, + relation); s3c_freq_dbg("%s: adjust %d to entry %d (%u)\n", __func__, target_freq, index, ftab[index].frequency); @@ -315,7 +311,6 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy, pll = NULL; } else { struct cpufreq_policy tmp_policy; - int ret; /* we keep the cpu pll table in Hz, to ensure we get an * accurate value for the PLL output. */ @@ -323,20 +318,14 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy, tmp_policy.min = policy->min * 1000; tmp_policy.max = policy->max * 1000; tmp_policy.cpu = policy->cpu; + tmp_policy.freq_table = pll_reg; - /* cpufreq_frequency_table_target uses a pointer to 'index' - * which is the number of the table entry, not the value of + /* cpufreq_frequency_table_target returns the index + * of the table entry, not the value of * the table entry's index field. */ - ret = cpufreq_frequency_table_target(&tmp_policy, pll_reg, - target_freq, relation, - &index); - - if (ret < 0) { - pr_err("%s: no PLL available\n", __func__); - goto err_notpossible; - } - + index = cpufreq_frequency_table_target(&tmp_policy, target_freq, + relation); pll = pll_reg + index; s3c_freq_dbg("%s: target %u => %u\n", @@ -346,10 +335,6 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy, } return s3c_cpufreq_settarget(policy, target_freq, pll); - - err_notpossible: - pr_err("no compatible settings for %d\n", target_freq); - return -EINVAL; } struct clk *s3c_cpufreq_clk_get(struct device *dev, const char *name) @@ -571,11 +556,7 @@ static int s3c_cpufreq_build_freq(void) { int size, ret; - if (!cpu_cur.info->calc_freqtable) - return -EINVAL; - kfree(ftab); - ftab = NULL; size = cpu_cur.info->calc_freqtable(&cpu_cur, NULL, 0); size++; diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index 06d85917b6d5..9e07588ea9f5 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -246,12 +246,7 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) new_freq = s5pv210_freq_table[index].frequency; /* Finding current running level index */ - if (cpufreq_frequency_table_target(policy, s5pv210_freq_table, - old_freq, CPUFREQ_RELATION_H, - &priv_index)) { - ret = -EINVAL; - goto exit; - } + priv_index = cpufreq_table_find_index_h(policy, old_freq); arm_volt = dvs_conf[index].arm_volt; int_volt = dvs_conf[index].int_volt; diff --git a/drivers/cpuidle/cpuidle-arm.c b/drivers/cpuidle/cpuidle-arm.c index e342565e8715..4ba3d3fe142f 100644 --- a/drivers/cpuidle/cpuidle-arm.c +++ b/drivers/cpuidle/cpuidle-arm.c @@ -36,26 +36,12 @@ static int arm_enter_idle_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, int idx) { - int ret; - - if (!idx) { - cpu_do_idle(); - return idx; - } - - ret = cpu_pm_enter(); - if (!ret) { - /* - * Pass idle state index to cpu_suspend which in turn will - * call the CPU ops suspend protocol with idle index as a - * parameter. - */ - ret = arm_cpuidle_suspend(idx); - - cpu_pm_exit(); - } - - return ret ? -1 : idx; + /* + * Pass idle state index to arm_cpuidle_suspend which in turn + * will call the CPU ops suspend protocol with idle index as a + * parameter. + */ + return CPU_PM_CPU_IDLE_ENTER(arm_cpuidle_suspend, idx); } static struct cpuidle_driver arm_idle_driver = { diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index a4d0059e232c..c73207abb5a4 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -173,7 +173,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, struct cpuidle_state *target_state = &drv->states[index]; bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP); - u64 time_start, time_end; + ktime_t time_start, time_end; s64 diff; /* @@ -195,13 +195,13 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, sched_idle_set_state(target_state); trace_cpu_idle_rcuidle(index, dev->cpu); - time_start = local_clock(); + time_start = ns_to_ktime(local_clock()); stop_critical_timings(); entered_state = target_state->enter(dev, drv, index); start_critical_timings(); - time_end = local_clock(); + time_end = ns_to_ktime(local_clock()); trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu); /* The cpu is no longer idle or about to enter idle. */ @@ -217,11 +217,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, if (!cpuidle_state_is_coupled(drv, index)) local_irq_enable(); - /* - * local_clock() returns the time in nanosecond, let's shift - * by 10 (divide by 1024) to have microsecond based time. - */ - diff = (time_end - time_start) >> 10; + diff = ktime_us_delta(time_end, time_start); if (diff > INT_MAX) diff = INT_MAX; diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index d77ba2f12242..1af94e2d1a25 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -159,6 +159,19 @@ config CRYPTO_GHASH_S390 It is available as of z196. +config CRYPTO_CRC32_S390 + tristate "CRC-32 algorithms" + depends on S390 + select CRYPTO_HASH + select CRC32 + help + Select this option if you want to use hardware accelerated + implementations of CRC algorithms. With this option, you + can optimize the computation of CRC-32 (IEEE 802.3 Ethernet) + and CRC-32C (Castagnoli). + + It is available with IBM z13 or later. + config CRYPTO_DEV_MV_CESA tristate "Marvell's Cryptographic Engine" depends on PLAT_ORION diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index 95b73968cf72..10db7df366c8 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -588,11 +588,6 @@ static int bfin_crypto_crc_probe(struct platform_device *pdev) crypto_init_queue(&crc->queue, CRC_CCRYPTO_QUEUE_LENGTH); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res == NULL) { - dev_err(&pdev->dev, "Cannot get IORESOURCE_MEM\n"); - return -ENOENT; - } - crc->regs = devm_ioremap_resource(dev, res); if (IS_ERR((void *)crc->regs)) { dev_err(&pdev->dev, "Cannot map CRC IO\n"); diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index 5652a53415dc..64bf3024b680 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -1,6 +1,6 @@ config CRYPTO_DEV_FSL_CAAM tristate "Freescale CAAM-Multicore driver backend" - depends on FSL_SOC || ARCH_MXC + depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE help Enables the driver module for Freescale's Cryptographic Accelerator and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). @@ -99,6 +99,18 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API To compile this as a module, choose M here: the module will be called caamhash. +config CRYPTO_DEV_FSL_CAAM_PKC_API + tristate "Register public key cryptography implementations with Crypto API" + depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR + default y + select CRYPTO_RSA + help + Selecting this will allow SEC Public key support for RSA. + Supported cryptographic primitives: encryption, decryption, + signature and verification. + To compile this as a module, choose M here: the module + will be called caam_pkc. + config CRYPTO_DEV_FSL_CAAM_RNG_API tristate "Register caam device for hwrng API" depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR @@ -116,10 +128,6 @@ config CRYPTO_DEV_FSL_CAAM_IMX def_bool SOC_IMX6 || SOC_IMX7D depends on CRYPTO_DEV_FSL_CAAM -config CRYPTO_DEV_FSL_CAAM_LE - def_bool CRYPTO_DEV_FSL_CAAM_IMX || SOC_LS1021A - depends on CRYPTO_DEV_FSL_CAAM - config CRYPTO_DEV_FSL_CAAM_DEBUG bool "Enable debug output in CAAM driver" depends on CRYPTO_DEV_FSL_CAAM diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 550758a333e7..08bf5515ae8a 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -2,7 +2,7 @@ # Makefile for the CAAM backend and dependent components # ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y) - EXTRA_CFLAGS := -DDEBUG + ccflags-y := -DDEBUG endif obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o @@ -10,6 +10,8 @@ obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o caam-objs := ctrl.o caam_jr-objs := jr.o key_gen.o error.o +caam_pkc-y := caampkc.o pkc_desc.o diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 5845d4a08797..f1ecc8df8d41 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -847,7 +847,7 @@ static int ahash_update_ctx(struct ahash_request *req) *next_buflen, 0); } else { (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= - SEC4_SG_LEN_FIN; + cpu_to_caam32(SEC4_SG_LEN_FIN); } state->current_buf = !state->current_buf; @@ -949,7 +949,8 @@ static int ahash_final_ctx(struct ahash_request *req) state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, buf, state->buf_dma, buflen, last_buflen); - (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= SEC4_SG_LEN_FIN; + (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= + cpu_to_caam32(SEC4_SG_LEN_FIN); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c new file mode 100644 index 000000000000..851015e652b8 --- /dev/null +++ b/drivers/crypto/caam/caampkc.c @@ -0,0 +1,607 @@ +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ +#include "compat.h" +#include "regs.h" +#include "intern.h" +#include "jr.h" +#include "error.h" +#include "desc_constr.h" +#include "sg_sw_sec4.h" +#include "caampkc.h" + +#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb)) +#define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \ + sizeof(struct rsa_priv_f1_pdb)) + +static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); + + if (edesc->sec4_sg_bytes) + dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes, + DMA_TO_DEVICE); +} + +static void rsa_pub_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_pub_pdb *pdb = &edesc->pdb.pub; + + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->e_dma, key->e_sz, DMA_TO_DEVICE); +} + +static void rsa_priv_f1_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f1_pdb *pdb = &edesc->pdb.priv_f1; + + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); +} + +/* RSA Job Completion handler */ +static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context) +{ + struct akcipher_request *req = context; + struct rsa_edesc *edesc; + + if (err) + caam_jr_strstatus(dev, err); + + edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); + + rsa_pub_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + akcipher_request_complete(req, err); +} + +static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err, + void *context) +{ + struct akcipher_request *req = context; + struct rsa_edesc *edesc; + + if (err) + caam_jr_strstatus(dev, err); + + edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); + + rsa_priv_f1_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + akcipher_request_complete(req, err); +} + +static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, + size_t desclen) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = ctx->dev; + struct rsa_edesc *edesc; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + int sgc; + int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; + int src_nents, dst_nents; + + src_nents = sg_nents_for_len(req->src, req->src_len); + dst_nents = sg_nents_for_len(req->dst, req->dst_len); + + if (src_nents > 1) + sec4_sg_len = src_nents; + if (dst_nents > 1) + sec4_sg_len += dst_nents; + + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); + + /* allocate space for base edesc, hw desc commands and link tables */ + edesc = kzalloc(sizeof(*edesc) + desclen + sec4_sg_bytes, + GFP_DMA | flags); + if (!edesc) + return ERR_PTR(-ENOMEM); + + sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + if (unlikely(!sgc)) { + dev_err(dev, "unable to map source\n"); + goto src_fail; + } + + sgc = dma_map_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE); + if (unlikely(!sgc)) { + dev_err(dev, "unable to map destination\n"); + goto dst_fail; + } + + edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen; + + sec4_sg_index = 0; + if (src_nents > 1) { + sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); + sec4_sg_index += src_nents; + } + if (dst_nents > 1) + sg_to_sec4_sg_last(req->dst, dst_nents, + edesc->sec4_sg + sec4_sg_index, 0); + + /* Save nents for later use in Job Descriptor */ + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + + if (!sec4_sg_bytes) + return edesc; + + edesc->sec4_sg_dma = dma_map_single(dev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(dev, edesc->sec4_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + goto sec4_sg_fail; + } + + edesc->sec4_sg_bytes = sec4_sg_bytes; + + return edesc; + +sec4_sg_fail: + dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE); +dst_fail: + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); +src_fail: + kfree(edesc); + return ERR_PTR(-ENOMEM); +} + +static int set_rsa_pub_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_pub_pdb *pdb = &edesc->pdb.pub; + int sec4_sg_index = 0; + + pdb->n_dma = dma_map_single(dev, key->n, key->n_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->n_dma)) { + dev_err(dev, "Unable to map RSA modulus memory\n"); + return -ENOMEM; + } + + pdb->e_dma = dma_map_single(dev, key->e, key->e_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->e_dma)) { + dev_err(dev, "Unable to map RSA public exponent memory\n"); + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + return -ENOMEM; + } + + if (edesc->src_nents > 1) { + pdb->sgf |= RSA_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + } else { + pdb->f_dma = sg_dma_address(req->src); + } + + if (edesc->dst_nents > 1) { + pdb->sgf |= RSA_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->g_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz; + pdb->f_len = req->src_len; + + return 0; +} + +static int set_rsa_priv_f1_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f1_pdb *pdb = &edesc->pdb.priv_f1; + int sec4_sg_index = 0; + + pdb->n_dma = dma_map_single(dev, key->n, key->n_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->n_dma)) { + dev_err(dev, "Unable to map modulus memory\n"); + return -ENOMEM; + } + + pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->d_dma)) { + dev_err(dev, "Unable to map RSA private exponent memory\n"); + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + return -ENOMEM; + } + + if (edesc->src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + } else { + pdb->g_dma = sg_dma_address(req->src); + } + + if (edesc->dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->d_sz << RSA_PDB_D_SHIFT) | key->n_sz; + + return 0; +} + +static int caam_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + if (unlikely(!key->n || !key->e)) + return -EINVAL; + + if (req->dst_len < key->n_sz) { + req->dst_len = key->n_sz; + dev_err(jrdev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PUB_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Encrypt Protocol Data Block */ + ret = set_rsa_pub_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_pub_desc(edesc->hw_desc, &edesc->pdb.pub); + + ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_pub_done, req); + if (!ret) + return -EINPROGRESS; + + rsa_pub_unmap(jrdev, edesc, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + if (unlikely(!key->n || !key->d)) + return -EINVAL; + + if (req->dst_len < key->n_sz) { + req->dst_len = key->n_sz; + dev_err(jrdev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F1_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #1 */ + ret = set_rsa_priv_f1_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f1_desc(edesc->hw_desc, &edesc->pdb.priv_f1); + + ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f1_done, req); + if (!ret) + return -EINPROGRESS; + + rsa_priv_f1_unmap(jrdev, edesc, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static void caam_rsa_free_key(struct caam_rsa_key *key) +{ + kzfree(key->d); + kfree(key->e); + kfree(key->n); + key->d = NULL; + key->e = NULL; + key->n = NULL; + key->d_sz = 0; + key->e_sz = 0; + key->n_sz = 0; +} + +/** + * caam_read_raw_data - Read a raw byte stream as a positive integer. + * The function skips buffer's leading zeros, copies the remained data + * to a buffer allocated in the GFP_DMA | GFP_KERNEL zone and returns + * the address of the new buffer. + * + * @buf : The data to read + * @nbytes: The amount of data to read + */ +static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes) +{ + u8 *val; + + while (!*buf && *nbytes) { + buf++; + (*nbytes)--; + } + + val = kzalloc(*nbytes, GFP_DMA | GFP_KERNEL); + if (!val) + return NULL; + + memcpy(val, buf, *nbytes); + + return val; +} + +static int caam_rsa_check_key_length(unsigned int len) +{ + if (len > 4096) + return -EINVAL; + return 0; +} + +static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {0}; + struct caam_rsa_key *rsa_key = &ctx->key; + int ret; + + /* Free the old RSA key if any */ + caam_rsa_free_key(rsa_key); + + ret = rsa_parse_pub_key(&raw_key, key, keylen); + if (ret) + return ret; + + /* Copy key in DMA zone */ + rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->e) + goto err; + + /* + * Skip leading zeros and copy the positive integer to a buffer + * allocated in the GFP_DMA | GFP_KERNEL zone. The decryption descriptor + * expects a positive integer for the RSA modulus and uses its length as + * decryption output length. + */ + rsa_key->n = caam_read_raw_data(raw_key.n, &raw_key.n_sz); + if (!rsa_key->n) + goto err; + + if (caam_rsa_check_key_length(raw_key.n_sz << 3)) { + caam_rsa_free_key(rsa_key); + return -EINVAL; + } + + rsa_key->e_sz = raw_key.e_sz; + rsa_key->n_sz = raw_key.n_sz; + + memcpy(rsa_key->e, raw_key.e, raw_key.e_sz); + + return 0; +err: + caam_rsa_free_key(rsa_key); + return -ENOMEM; +} + +static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {0}; + struct caam_rsa_key *rsa_key = &ctx->key; + int ret; + + /* Free the old RSA key if any */ + caam_rsa_free_key(rsa_key); + + ret = rsa_parse_priv_key(&raw_key, key, keylen); + if (ret) + return ret; + + /* Copy key in DMA zone */ + rsa_key->d = kzalloc(raw_key.d_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->d) + goto err; + + rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->e) + goto err; + + /* + * Skip leading zeros and copy the positive integer to a buffer + * allocated in the GFP_DMA | GFP_KERNEL zone. The decryption descriptor + * expects a positive integer for the RSA modulus and uses its length as + * decryption output length. + */ + rsa_key->n = caam_read_raw_data(raw_key.n, &raw_key.n_sz); + if (!rsa_key->n) + goto err; + + if (caam_rsa_check_key_length(raw_key.n_sz << 3)) { + caam_rsa_free_key(rsa_key); + return -EINVAL; + } + + rsa_key->d_sz = raw_key.d_sz; + rsa_key->e_sz = raw_key.e_sz; + rsa_key->n_sz = raw_key.n_sz; + + memcpy(rsa_key->d, raw_key.d, raw_key.d_sz); + memcpy(rsa_key->e, raw_key.e, raw_key.e_sz); + + return 0; + +err: + caam_rsa_free_key(rsa_key); + return -ENOMEM; +} + +static int caam_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + + return (key->n) ? key->n_sz : -EINVAL; +} + +/* Per session pkc's driver context creation function */ +static int caam_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + + ctx->dev = caam_jr_alloc(); + + if (IS_ERR(ctx->dev)) { + dev_err(ctx->dev, "Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->dev); + } + + return 0; +} + +/* Per session pkc's driver context cleanup function */ +static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + + caam_rsa_free_key(key); + caam_jr_free(ctx->dev); +} + +static struct akcipher_alg caam_rsa = { + .encrypt = caam_rsa_enc, + .decrypt = caam_rsa_dec, + .sign = caam_rsa_dec, + .verify = caam_rsa_enc, + .set_pub_key = caam_rsa_set_pub_key, + .set_priv_key = caam_rsa_set_priv_key, + .max_size = caam_rsa_max_size, + .init = caam_rsa_init_tfm, + .exit = caam_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "rsa-caam", + .cra_priority = 3000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct caam_rsa_ctx), + }, +}; + +/* Public Key Cryptography module initialization handler */ +static int __init caam_pkc_init(void) +{ + struct device_node *dev_node; + struct platform_device *pdev; + struct device *ctrldev; + struct caam_drv_private *priv; + u32 cha_inst, pk_inst; + int err; + + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + if (!dev_node) { + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); + if (!dev_node) + return -ENODEV; + } + + pdev = of_find_device_by_node(dev_node); + if (!pdev) { + of_node_put(dev_node); + return -ENODEV; + } + + ctrldev = &pdev->dev; + priv = dev_get_drvdata(ctrldev); + of_node_put(dev_node); + + /* + * If priv is NULL, it's probably because the caam driver wasn't + * properly initialized (e.g. RNG4 init failed). Thus, bail out here. + */ + if (!priv) + return -ENODEV; + + /* Determine public key hardware accelerator presence. */ + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + pk_inst = (cha_inst & CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT; + + /* Do not register algorithms if PKHA is not present. */ + if (!pk_inst) + return -ENODEV; + + err = crypto_register_akcipher(&caam_rsa); + if (err) + dev_warn(ctrldev, "%s alg registration failed\n", + caam_rsa.base.cra_driver_name); + else + dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n"); + + return err; +} + +static void __exit caam_pkc_exit(void) +{ + crypto_unregister_akcipher(&caam_rsa); +} + +module_init(caam_pkc_init); +module_exit(caam_pkc_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API"); +MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h new file mode 100644 index 000000000000..f595d159b112 --- /dev/null +++ b/drivers/crypto/caam/caampkc.h @@ -0,0 +1,70 @@ +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography descriptors + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ + +#ifndef _PKC_DESC_H_ +#define _PKC_DESC_H_ +#include "compat.h" +#include "pdb.h" + +/** + * caam_rsa_key - CAAM RSA key structure. Keys are allocated in DMA zone. + * @n : RSA modulus raw byte stream + * @e : RSA public exponent raw byte stream + * @d : RSA private exponent raw byte stream + * @n_sz : length in bytes of RSA modulus n + * @e_sz : length in bytes of RSA public exponent + * @d_sz : length in bytes of RSA private exponent + */ +struct caam_rsa_key { + u8 *n; + u8 *e; + u8 *d; + size_t n_sz; + size_t e_sz; + size_t d_sz; +}; + +/** + * caam_rsa_ctx - per session context. + * @key : RSA key in DMA zone + * @dev : device structure + */ +struct caam_rsa_ctx { + struct caam_rsa_key key; + struct device *dev; +}; + +/** + * rsa_edesc - s/w-extended rsa descriptor + * @src_nents : number of segments in input scatterlist + * @dst_nents : number of segments in output scatterlist + * @sec4_sg_bytes : length of h/w link table + * @sec4_sg_dma : dma address of h/w link table + * @sec4_sg : pointer to h/w link table + * @pdb : specific RSA Protocol Data Block (PDB) + * @hw_desc : descriptor followed by link tables if any + */ +struct rsa_edesc { + int src_nents; + int dst_nents; + int sec4_sg_bytes; + dma_addr_t sec4_sg_dma; + struct sec4_sg_entry *sec4_sg; + union { + struct rsa_pub_pdb pub; + struct rsa_priv_f1_pdb priv_f1; + } pdb; + u32 hw_desc[]; +}; + +/* Descriptor construction primitives. */ +void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb); +void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb); + +#endif diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index b6955ecdfb3f..7149cd2492e0 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -35,8 +35,11 @@ #include #include #include +#include #include #include #include +#include +#include #endif /* !defined(CAAM_COMPAT_H) */ diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 5ad5f3009ae0..0ec112ee5204 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -15,6 +15,9 @@ #include "desc_constr.h" #include "error.h" +bool caam_little_end; +EXPORT_SYMBOL(caam_little_end); + /* * i.MX targets tend to have clock control subsystems that can * enable/disable clocking to our device. @@ -106,7 +109,7 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, if (ctrlpriv->virt_en == 1) { - setbits32(&ctrl->deco_rsr, DECORSR_JR0); + clrsetbits_32(&ctrl->deco_rsr, 0, DECORSR_JR0); while (!(rd_reg32(&ctrl->deco_rsr) & DECORSR_VALID) && --timeout) @@ -115,7 +118,7 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, timeout = 100000; } - setbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); + clrsetbits_32(&ctrl->deco_rq, 0, DECORR_RQD0ENABLE); while (!(rd_reg32(&ctrl->deco_rq) & DECORR_DEN0) && --timeout) @@ -123,12 +126,12 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, if (!timeout) { dev_err(ctrldev, "failed to acquire DECO 0\n"); - clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); + clrsetbits_32(&ctrl->deco_rq, DECORR_RQD0ENABLE, 0); return -ENODEV; } for (i = 0; i < desc_len(desc); i++) - wr_reg32(&deco->descbuf[i], *(desc + i)); + wr_reg32(&deco->descbuf[i], caam32_to_cpu(*(desc + i))); flags = DECO_JQCR_WHL; /* @@ -139,7 +142,7 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, flags |= DECO_JQCR_FOUR; /* Instruct the DECO to execute it */ - setbits32(&deco->jr_ctl_hi, flags); + clrsetbits_32(&deco->jr_ctl_hi, 0, flags); timeout = 10000000; do { @@ -158,10 +161,10 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, DECO_OP_STATUS_HI_ERR_MASK; if (ctrlpriv->virt_en == 1) - clrbits32(&ctrl->deco_rsr, DECORSR_JR0); + clrsetbits_32(&ctrl->deco_rsr, DECORSR_JR0, 0); /* Mark the DECO as free */ - clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); + clrsetbits_32(&ctrl->deco_rq, DECORR_RQD0ENABLE, 0); if (!timeout) return -EAGAIN; @@ -349,7 +352,7 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) r4tst = &ctrl->r4tst[0]; /* put RNG4 into program mode */ - setbits32(&r4tst->rtmctl, RTMCTL_PRGM); + clrsetbits_32(&r4tst->rtmctl, 0, RTMCTL_PRGM); /* * Performance-wise, it does not make sense to @@ -363,7 +366,7 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) >> RTSDCTL_ENT_DLY_SHIFT; if (ent_delay <= val) { /* put RNG4 into run mode */ - clrbits32(&r4tst->rtmctl, RTMCTL_PRGM); + clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, 0); return; } @@ -381,9 +384,9 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) * select raw sampling in both entropy shifter * and statistical checker */ - setbits32(&val, RTMCTL_SAMP_MODE_RAW_ES_SC); + clrsetbits_32(&val, 0, RTMCTL_SAMP_MODE_RAW_ES_SC); /* put RNG4 into run mode */ - clrbits32(&val, RTMCTL_PRGM); + clrsetbits_32(&val, RTMCTL_PRGM, 0); /* write back the control register */ wr_reg32(&r4tst->rtmctl, val); } @@ -406,6 +409,23 @@ int caam_get_era(void) } EXPORT_SYMBOL(caam_get_era); +#ifdef CONFIG_DEBUG_FS +static int caam_debugfs_u64_get(void *data, u64 *val) +{ + *val = caam64_to_cpu(*(u64 *)data); + return 0; +} + +static int caam_debugfs_u32_get(void *data, u64 *val) +{ + *val = caam32_to_cpu(*(u32 *)data); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); +#endif + /* Probe routine for CAAM top (controller) level */ static int caam_probe(struct platform_device *pdev) { @@ -504,6 +524,10 @@ static int caam_probe(struct platform_device *pdev) ret = -ENOMEM; goto disable_caam_emi_slow; } + + caam_little_end = !(bool)(rd_reg32(&ctrl->perfmon.status) & + (CSTA_PLEND | CSTA_ALT_PLEND)); + /* Finding the page size for using the CTPR_MS register */ comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms); pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT; @@ -559,9 +583,9 @@ static int caam_probe(struct platform_device *pdev) } if (ctrlpriv->virt_en == 1) - setbits32(&ctrl->jrstart, JRSTART_JR0_START | - JRSTART_JR1_START | JRSTART_JR2_START | - JRSTART_JR3_START); + clrsetbits_32(&ctrl->jrstart, 0, JRSTART_JR0_START | + JRSTART_JR1_START | JRSTART_JR2_START | + JRSTART_JR3_START); if (sizeof(dma_addr_t) == sizeof(u64)) if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) @@ -693,7 +717,7 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->rng4_sh_init = ~ctrlpriv->rng4_sh_init & RDSTA_IFMASK; /* Enable RDB bit so that RNG works faster */ - setbits32(&ctrl->scfgr, SCFGR_RDBENABLE); + clrsetbits_32(&ctrl->scfgr, 0, SCFGR_RDBENABLE); } /* NOTE: RTIC detection ought to go here, around Si time */ @@ -719,48 +743,59 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); /* Controller-level - performance monitor counters */ + ctrlpriv->ctl_rq_dequeued = - debugfs_create_u64("rq_dequeued", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->req_dequeued); + debugfs_create_file("rq_dequeued", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->req_dequeued, + &caam_fops_u64_ro); ctrlpriv->ctl_ob_enc_req = - debugfs_create_u64("ob_rq_encrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_req); + debugfs_create_file("ob_rq_encrypted", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_enc_req, + &caam_fops_u64_ro); ctrlpriv->ctl_ib_dec_req = - debugfs_create_u64("ib_rq_decrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_req); + debugfs_create_file("ib_rq_decrypted", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_dec_req, + &caam_fops_u64_ro); ctrlpriv->ctl_ob_enc_bytes = - debugfs_create_u64("ob_bytes_encrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_bytes); + debugfs_create_file("ob_bytes_encrypted", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_enc_bytes, + &caam_fops_u64_ro); ctrlpriv->ctl_ob_prot_bytes = - debugfs_create_u64("ob_bytes_protected", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_prot_bytes); + debugfs_create_file("ob_bytes_protected", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_prot_bytes, + &caam_fops_u64_ro); ctrlpriv->ctl_ib_dec_bytes = - debugfs_create_u64("ib_bytes_decrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_bytes); + debugfs_create_file("ib_bytes_decrypted", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_dec_bytes, + &caam_fops_u64_ro); ctrlpriv->ctl_ib_valid_bytes = - debugfs_create_u64("ib_bytes_validated", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_valid_bytes); + debugfs_create_file("ib_bytes_validated", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_valid_bytes, + &caam_fops_u64_ro); /* Controller level - global status values */ ctrlpriv->ctl_faultaddr = - debugfs_create_u64("fault_addr", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultaddr); + debugfs_create_file("fault_addr", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->faultaddr, + &caam_fops_u32_ro); ctrlpriv->ctl_faultdetail = - debugfs_create_u32("fault_detail", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultdetail); + debugfs_create_file("fault_detail", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->faultdetail, + &caam_fops_u32_ro); ctrlpriv->ctl_faultstatus = - debugfs_create_u32("fault_status", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->status); + debugfs_create_file("fault_status", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->status, + &caam_fops_u32_ro); /* Internal covering keys (useful in non-secure mode only) */ ctrlpriv->ctl_kek_wrap.data = &ctrlpriv->ctrl->kek[0]; diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index 1e93c6af2275..26427c11ad87 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -20,19 +20,18 @@ #define SEC4_SG_BPID_MASK 0x000000ff #define SEC4_SG_BPID_SHIFT 16 #define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ -#define SEC4_SG_OFFS_MASK 0x00001fff +#define SEC4_SG_OFFSET_MASK 0x00001fff struct sec4_sg_entry { -#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX +#if !defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) && \ + defined(CONFIG_CRYPTO_DEV_FSL_CAAM_IMX) u32 rsvd1; dma_addr_t ptr; #else u64 ptr; #endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_IMX */ u32 len; - u8 rsvd2; - u8 buf_pool_id; - u16 offset; + u32 bpid_offset; }; /* Max size of any CAAM descriptor in 32-bit words, inclusive of header */ @@ -454,6 +453,8 @@ struct sec4_sg_entry { #define OP_PCLID_PUBLICKEYPAIR (0x14 << OP_PCLID_SHIFT) #define OP_PCLID_DSASIGN (0x15 << OP_PCLID_SHIFT) #define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT) +#define OP_PCLID_RSAENC_PUBKEY (0x18 << OP_PCLID_SHIFT) +#define OP_PCLID_RSADEC_PRVKEY (0x19 << OP_PCLID_SHIFT) /* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */ #define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT) diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 98d07de24fc4..d3869b95e7b1 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -5,6 +5,7 @@ */ #include "desc.h" +#include "regs.h" #define IMMEDIATE (1 << 23) #define CAAM_CMD_SZ sizeof(u32) @@ -30,9 +31,11 @@ LDST_SRCDST_WORD_DECOCTRL | \ (LDOFF_ENABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT)) +extern bool caam_little_end; + static inline int desc_len(u32 *desc) { - return *desc & HDR_DESCLEN_MASK; + return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK; } static inline int desc_bytes(void *desc) @@ -52,7 +55,7 @@ static inline void *sh_desc_pdb(u32 *desc) static inline void init_desc(u32 *desc, u32 options) { - *desc = (options | HDR_ONE) + 1; + *desc = cpu_to_caam32((options | HDR_ONE) + 1); } static inline void init_sh_desc(u32 *desc, u32 options) @@ -74,13 +77,21 @@ static inline void init_job_desc(u32 *desc, u32 options) init_desc(desc, CMD_DESC_HDR | options); } +static inline void init_job_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes) +{ + u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; + + init_job_desc(desc, (((pdb_len + 1) << HDR_START_IDX_SHIFT)) | options); +} + static inline void append_ptr(u32 *desc, dma_addr_t ptr) { dma_addr_t *offset = (dma_addr_t *)desc_end(desc); - *offset = ptr; + *offset = cpu_to_caam_dma(ptr); - (*desc) += CAAM_PTR_SZ / CAAM_CMD_SZ; + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + + CAAM_PTR_SZ / CAAM_CMD_SZ); } static inline void init_job_desc_shared(u32 *desc, dma_addr_t ptr, int len, @@ -99,16 +110,17 @@ static inline void append_data(u32 *desc, void *data, int len) if (len) /* avoid sparse warning: memcpy with byte count of 0 */ memcpy(offset, data, len); - (*desc) += (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + + (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ); } static inline void append_cmd(u32 *desc, u32 command) { u32 *cmd = desc_end(desc); - *cmd = command; + *cmd = cpu_to_caam32(command); - (*desc)++; + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + 1); } #define append_u32 append_cmd @@ -117,16 +129,22 @@ static inline void append_u64(u32 *desc, u64 data) { u32 *offset = desc_end(desc); - *offset = upper_32_bits(data); - *(++offset) = lower_32_bits(data); + /* Only 32-bit alignment is guaranteed in descriptor buffer */ + if (caam_little_end) { + *offset = cpu_to_caam32(lower_32_bits(data)); + *(++offset) = cpu_to_caam32(upper_32_bits(data)); + } else { + *offset = cpu_to_caam32(upper_32_bits(data)); + *(++offset) = cpu_to_caam32(lower_32_bits(data)); + } - (*desc) += 2; + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + 2); } /* Write command without affecting header, and return pointer to next word */ static inline u32 *write_cmd(u32 *desc, u32 command) { - *desc = command; + *desc = cpu_to_caam32(command); return desc + 1; } @@ -168,14 +186,17 @@ APPEND_CMD_RET(move, MOVE) static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd) { - *jump_cmd = *jump_cmd | (desc_len(desc) - (jump_cmd - desc)); + *jump_cmd = cpu_to_caam32(caam32_to_cpu(*jump_cmd) | + (desc_len(desc) - (jump_cmd - desc))); } static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd) { - *move_cmd &= ~MOVE_OFFSET_MASK; - *move_cmd = *move_cmd | ((desc_len(desc) << (MOVE_OFFSET_SHIFT + 2)) & - MOVE_OFFSET_MASK); + u32 val = caam32_to_cpu(*move_cmd); + + val &= ~MOVE_OFFSET_MASK; + val |= (desc_len(desc) << (MOVE_OFFSET_SHIFT + 2)) & MOVE_OFFSET_MASK; + *move_cmd = cpu_to_caam32(val); } #define APPEND_CMD(cmd, op) \ diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 5ef4be22eb80..a81f551ac222 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -31,7 +31,7 @@ static int caam_reset_hw_jr(struct device *dev) * mask interrupts since we are going to poll * for reset completion status */ - setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK); /* initiate flush (required prior to reset) */ wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); @@ -57,7 +57,7 @@ static int caam_reset_hw_jr(struct device *dev) } /* unmask interrupts */ - clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0); return 0; } @@ -147,7 +147,7 @@ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) } /* mask valid interrupts */ - setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK); /* Have valid interrupt at this point, just ACK and trigger */ wr_reg32(&jrp->rregs->jrintstatus, irqstate); @@ -182,7 +182,7 @@ static void caam_jr_dequeue(unsigned long devarg) sw_idx = (tail + i) & (JOBR_DEPTH - 1); if (jrp->outring[hw_idx].desc == - jrp->entinfo[sw_idx].desc_addr_dma) + caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma)) break; /* found */ } /* we should never fail to find a matching descriptor */ @@ -200,7 +200,7 @@ static void caam_jr_dequeue(unsigned long devarg) usercall = jrp->entinfo[sw_idx].callbk; userarg = jrp->entinfo[sw_idx].cbkarg; userdesc = jrp->entinfo[sw_idx].desc_addr_virt; - userstatus = jrp->outring[hw_idx].jrstatus; + userstatus = caam32_to_cpu(jrp->outring[hw_idx].jrstatus); /* * Make sure all information from the job has been obtained @@ -236,7 +236,7 @@ static void caam_jr_dequeue(unsigned long devarg) } /* reenable / unmask IRQs */ - clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0); } /** @@ -330,7 +330,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, int head, tail, desc_size; dma_addr_t desc_dma; - desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32); + desc_size = (caam32_to_cpu(*desc) & HDR_JD_LENGTH_MASK) * sizeof(u32); desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE); if (dma_mapping_error(dev, desc_dma)) { dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n"); @@ -356,7 +356,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, head_entry->cbkarg = areq; head_entry->desc_addr_dma = desc_dma; - jrp->inpring[jrp->inp_ring_write_index] = desc_dma; + jrp->inpring[jrp->inp_ring_write_index] = cpu_to_caam_dma(desc_dma); /* * Guarantee that the descriptor's DMA address has been written to @@ -444,9 +444,9 @@ static int caam_jr_init(struct device *dev) spin_lock_init(&jrp->outlock); /* Select interrupt coalescing parameters */ - setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC | - (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | - (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC | + (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | + (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); return 0; diff --git a/drivers/crypto/caam/pdb.h b/drivers/crypto/caam/pdb.h index 3a87c0cf879a..aaa00dd1c601 100644 --- a/drivers/crypto/caam/pdb.h +++ b/drivers/crypto/caam/pdb.h @@ -1,18 +1,19 @@ /* * CAAM Protocol Data Block (PDB) definition header file * - * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2008-2016 Freescale Semiconductor, Inc. * */ #ifndef CAAM_PDB_H #define CAAM_PDB_H +#include "compat.h" /* * PDB- IPSec ESP Header Modification Options */ -#define PDBHMO_ESP_DECAP_SHIFT 12 -#define PDBHMO_ESP_ENCAP_SHIFT 4 +#define PDBHMO_ESP_DECAP_SHIFT 28 +#define PDBHMO_ESP_ENCAP_SHIFT 28 /* * Encap and Decap - Decrement TTL (Hop Limit) - Based on the value of the * Options Byte IP version (IPvsn) field: @@ -32,12 +33,23 @@ */ #define PDBHMO_ESP_DFBIT (0x04 << PDBHMO_ESP_ENCAP_SHIFT) +#define PDBNH_ESP_ENCAP_SHIFT 16 +#define PDBNH_ESP_ENCAP_MASK (0xff << PDBNH_ESP_ENCAP_SHIFT) + +#define PDBHDRLEN_ESP_DECAP_SHIFT 16 +#define PDBHDRLEN_MASK (0x0fff << PDBHDRLEN_ESP_DECAP_SHIFT) + +#define PDB_NH_OFFSET_SHIFT 8 +#define PDB_NH_OFFSET_MASK (0xff << PDB_NH_OFFSET_SHIFT) + /* * PDB - IPSec ESP Encap/Decap Options */ #define PDBOPTS_ESP_ARSNONE 0x00 /* no antireplay window */ #define PDBOPTS_ESP_ARS32 0x40 /* 32-entry antireplay window */ +#define PDBOPTS_ESP_ARS128 0x80 /* 128-entry antireplay window */ #define PDBOPTS_ESP_ARS64 0xc0 /* 64-entry antireplay window */ +#define PDBOPTS_ESP_ARS_MASK 0xc0 /* antireplay window mask */ #define PDBOPTS_ESP_IVSRC 0x20 /* IV comes from internal random gen */ #define PDBOPTS_ESP_ESN 0x10 /* extended sequence included */ #define PDBOPTS_ESP_OUTFMT 0x08 /* output only decapsulation (decap) */ @@ -54,35 +66,73 @@ /* * General IPSec encap/decap PDB definitions */ + +/** + * ipsec_encap_cbc - PDB part for IPsec CBC encapsulation + * @iv: 16-byte array initialization vector + */ struct ipsec_encap_cbc { - u32 iv[4]; + u8 iv[16]; }; +/** + * ipsec_encap_ctr - PDB part for IPsec CTR encapsulation + * @ctr_nonce: 4-byte array nonce + * @ctr_initial: initial count constant + * @iv: initialization vector + */ struct ipsec_encap_ctr { - u32 ctr_nonce; + u8 ctr_nonce[4]; u32 ctr_initial; - u32 iv[2]; + u64 iv; }; +/** + * ipsec_encap_ccm - PDB part for IPsec CCM encapsulation + * @salt: 3-byte array salt (lower 24 bits) + * @ccm_opt: CCM algorithm options - MSB-LSB description: + * b0_flags (8b) - CCM B0; use 0x5B for 8-byte ICV, 0x6B for 12-byte ICV, + * 0x7B for 16-byte ICV (cf. RFC4309, RFC3610) + * ctr_flags (8b) - counter flags; constant equal to 0x3 + * ctr_initial (16b) - initial count constant + * @iv: initialization vector + */ struct ipsec_encap_ccm { - u32 salt; /* lower 24 bits */ - u8 b0_flags; - u8 ctr_flags; - u16 ctr_initial; - u32 iv[2]; + u8 salt[4]; + u32 ccm_opt; + u64 iv; }; +/** + * ipsec_encap_gcm - PDB part for IPsec GCM encapsulation + * @salt: 3-byte array salt (lower 24 bits) + * @rsvd: reserved, do not use + * @iv: initialization vector + */ struct ipsec_encap_gcm { - u32 salt; /* lower 24 bits */ + u8 salt[4]; u32 rsvd1; - u32 iv[2]; + u64 iv; }; +/** + * ipsec_encap_pdb - PDB for IPsec encapsulation + * @options: MSB-LSB description + * hmo (header manipulation options) - 4b + * reserved - 4b + * next header - 8b + * next header offset - 8b + * option flags (depend on selected algorithm) - 8b + * @seq_num_ext_hi: (optional) IPsec Extended Sequence Number (ESN) + * @seq_num: IPsec sequence number + * @spi: IPsec SPI (Security Parameters Index) + * @ip_hdr_len: optional IP Header length (in bytes) + * reserved - 16b + * Opt. IP Hdr Len - 16b + * @ip_hdr: optional IP Header content + */ struct ipsec_encap_pdb { - u8 hmo_rsvd; - u8 ip_nh; - u8 ip_nh_offset; - u8 options; + u32 options; u32 seq_num_ext_hi; u32 seq_num; union { @@ -92,36 +142,65 @@ struct ipsec_encap_pdb { struct ipsec_encap_gcm gcm; }; u32 spi; - u16 rsvd1; - u16 ip_hdr_len; - u32 ip_hdr[0]; /* optional IP Header content */ + u32 ip_hdr_len; + u32 ip_hdr[0]; }; +/** + * ipsec_decap_cbc - PDB part for IPsec CBC decapsulation + * @rsvd: reserved, do not use + */ struct ipsec_decap_cbc { u32 rsvd[2]; }; +/** + * ipsec_decap_ctr - PDB part for IPsec CTR decapsulation + * @ctr_nonce: 4-byte array nonce + * @ctr_initial: initial count constant + */ struct ipsec_decap_ctr { - u32 salt; + u8 ctr_nonce[4]; u32 ctr_initial; }; +/** + * ipsec_decap_ccm - PDB part for IPsec CCM decapsulation + * @salt: 3-byte salt (lower 24 bits) + * @ccm_opt: CCM algorithm options - MSB-LSB description: + * b0_flags (8b) - CCM B0; use 0x5B for 8-byte ICV, 0x6B for 12-byte ICV, + * 0x7B for 16-byte ICV (cf. RFC4309, RFC3610) + * ctr_flags (8b) - counter flags; constant equal to 0x3 + * ctr_initial (16b) - initial count constant + */ struct ipsec_decap_ccm { - u32 salt; - u8 iv_flags; - u8 ctr_flags; - u16 ctr_initial; + u8 salt[4]; + u32 ccm_opt; }; +/** + * ipsec_decap_gcm - PDB part for IPsec GCN decapsulation + * @salt: 4-byte salt + * @rsvd: reserved, do not use + */ struct ipsec_decap_gcm { - u32 salt; + u8 salt[4]; u32 resvd; }; +/** + * ipsec_decap_pdb - PDB for IPsec decapsulation + * @options: MSB-LSB description + * hmo (header manipulation options) - 4b + * IP header length - 12b + * next header offset - 8b + * option flags (depend on selected algorithm) - 8b + * @seq_num_ext_hi: (optional) IPsec Extended Sequence Number (ESN) + * @seq_num: IPsec sequence number + * @anti_replay: Anti-replay window; size depends on ARS (option flags) + */ struct ipsec_decap_pdb { - u16 hmo_ip_hdr_len; - u8 ip_nh_offset; - u8 options; + u32 options; union { struct ipsec_decap_cbc cbc; struct ipsec_decap_ctr ctr; @@ -130,8 +209,7 @@ struct ipsec_decap_pdb { }; u32 seq_num_ext_hi; u32 seq_num; - u32 anti_replay[2]; - u32 end_index[0]; + __be32 anti_replay[4]; }; /* @@ -399,4 +477,52 @@ struct dsa_verify_pdb { u8 *ab; /* only used if ECC processing */ }; +/* RSA Protocol Data Block */ +#define RSA_PDB_SGF_SHIFT 28 +#define RSA_PDB_E_SHIFT 12 +#define RSA_PDB_E_MASK (0xFFF << RSA_PDB_E_SHIFT) +#define RSA_PDB_D_SHIFT 12 +#define RSA_PDB_D_MASK (0xFFF << RSA_PDB_D_SHIFT) + +#define RSA_PDB_SGF_F (0x8 << RSA_PDB_SGF_SHIFT) +#define RSA_PDB_SGF_G (0x4 << RSA_PDB_SGF_SHIFT) +#define RSA_PRIV_PDB_SGF_F (0x4 << RSA_PDB_SGF_SHIFT) +#define RSA_PRIV_PDB_SGF_G (0x8 << RSA_PDB_SGF_SHIFT) + +#define RSA_PRIV_KEY_FRM_1 0 + +/** + * RSA Encrypt Protocol Data Block + * @sgf: scatter-gather field + * @f_dma: dma address of input data + * @g_dma: dma address of encrypted output data + * @n_dma: dma address of RSA modulus + * @e_dma: dma address of RSA public exponent + * @f_len: length in octets of the input data + */ +struct rsa_pub_pdb { + u32 sgf; + dma_addr_t f_dma; + dma_addr_t g_dma; + dma_addr_t n_dma; + dma_addr_t e_dma; + u32 f_len; +} __packed; + +/** + * RSA Decrypt PDB - Private Key Form #1 + * @sgf: scatter-gather field + * @g_dma: dma address of encrypted input data + * @f_dma: dma address of output data + * @n_dma: dma address of RSA modulus + * @d_dma: dma address of RSA private exponent + */ +struct rsa_priv_f1_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t n_dma; + dma_addr_t d_dma; +} __packed; + #endif diff --git a/drivers/crypto/caam/pkc_desc.c b/drivers/crypto/caam/pkc_desc.c new file mode 100644 index 000000000000..4e4183e615ea --- /dev/null +++ b/drivers/crypto/caam/pkc_desc.c @@ -0,0 +1,36 @@ +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography descriptors + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ +#include "caampkc.h" +#include "desc_constr.h" + +/* Descriptor for RSA Public operation */ +void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, sizeof(*pdb)); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->n_dma); + append_ptr(desc, pdb->e_dma); + append_cmd(desc, pdb->f_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSAENC_PUBKEY); +} + +/* Descriptor for RSA Private operation - Private Key Form #1 */ +void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, sizeof(*pdb)); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->n_dma); + append_ptr(desc, pdb->d_dma); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_1); +} diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index 0ba9c40597dc..b3c5016f6458 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -8,6 +8,7 @@ #define REGS_H #include +#include #include /* @@ -65,46 +66,56 @@ * */ -#ifdef CONFIG_ARM -/* These are common macros for Power, put here for ARM */ -#define setbits32(_addr, _v) writel((readl(_addr) | (_v)), (_addr)) -#define clrbits32(_addr, _v) writel((readl(_addr) & ~(_v)), (_addr)) +extern bool caam_little_end; -#define out_arch(type, endian, a, v) __raw_write##type(cpu_to_##endian(v), a) -#define in_arch(type, endian, a) endian##_to_cpu(__raw_read##type(a)) +#define caam_to_cpu(len) \ +static inline u##len caam##len ## _to_cpu(u##len val) \ +{ \ + if (caam_little_end) \ + return le##len ## _to_cpu(val); \ + else \ + return be##len ## _to_cpu(val); \ +} -#define out_le32(a, v) out_arch(l, le32, a, v) -#define in_le32(a) in_arch(l, le32, a) +#define cpu_to_caam(len) \ +static inline u##len cpu_to_caam##len(u##len val) \ +{ \ + if (caam_little_end) \ + return cpu_to_le##len(val); \ + else \ + return cpu_to_be##len(val); \ +} -#define out_be32(a, v) out_arch(l, be32, a, v) -#define in_be32(a) in_arch(l, be32, a) +caam_to_cpu(16) +caam_to_cpu(32) +caam_to_cpu(64) +cpu_to_caam(16) +cpu_to_caam(32) +cpu_to_caam(64) -#define clrsetbits(type, addr, clear, set) \ - out_##type((addr), (in_##type(addr) & ~(clear)) | (set)) +static inline void wr_reg32(void __iomem *reg, u32 data) +{ + if (caam_little_end) + iowrite32(data, reg); + else + iowrite32be(data, reg); +} -#define clrsetbits_be32(addr, clear, set) clrsetbits(be32, addr, clear, set) -#define clrsetbits_le32(addr, clear, set) clrsetbits(le32, addr, clear, set) -#endif +static inline u32 rd_reg32(void __iomem *reg) +{ + if (caam_little_end) + return ioread32(reg); -#ifdef __BIG_ENDIAN -#define wr_reg32(reg, data) out_be32(reg, data) -#define rd_reg32(reg) in_be32(reg) -#define clrsetbits_32(addr, clear, set) clrsetbits_be32(addr, clear, set) -#ifdef CONFIG_64BIT -#define wr_reg64(reg, data) out_be64(reg, data) -#define rd_reg64(reg) in_be64(reg) -#endif -#else -#ifdef __LITTLE_ENDIAN -#define wr_reg32(reg, data) __raw_writel(data, reg) -#define rd_reg32(reg) __raw_readl(reg) -#define clrsetbits_32(addr, clear, set) clrsetbits_le32(addr, clear, set) -#ifdef CONFIG_64BIT -#define wr_reg64(reg, data) __raw_writeq(data, reg) -#define rd_reg64(reg) __raw_readq(reg) -#endif -#endif -#endif + return ioread32be(reg); +} + +static inline void clrsetbits_32(void __iomem *reg, u32 clear, u32 set) +{ + if (caam_little_end) + iowrite32((ioread32(reg) & ~clear) | set, reg); + else + iowrite32be((ioread32be(reg) & ~clear) | set, reg); +} /* * The only users of these wr/rd_reg64 functions is the Job Ring (JR). @@ -123,29 +134,67 @@ * base + 0x0000 : least-significant 32 bits * base + 0x0004 : most-significant 32 bits */ +#ifdef CONFIG_64BIT +static inline void wr_reg64(void __iomem *reg, u64 data) +{ + if (caam_little_end) + iowrite64(data, reg); + else + iowrite64be(data, reg); +} -#ifndef CONFIG_64BIT -#if !defined(CONFIG_CRYPTO_DEV_FSL_CAAM_LE) || \ - defined(CONFIG_CRYPTO_DEV_FSL_CAAM_IMX) -#define REG64_MS32(reg) ((u32 __iomem *)(reg)) -#define REG64_LS32(reg) ((u32 __iomem *)(reg) + 1) -#else -#define REG64_MS32(reg) ((u32 __iomem *)(reg) + 1) -#define REG64_LS32(reg) ((u32 __iomem *)(reg)) -#endif - -static inline void wr_reg64(u64 __iomem *reg, u64 data) +static inline u64 rd_reg64(void __iomem *reg) { - wr_reg32(REG64_MS32(reg), data >> 32); - wr_reg32(REG64_LS32(reg), data); + if (caam_little_end) + return ioread64(reg); + else + return ioread64be(reg); } -static inline u64 rd_reg64(u64 __iomem *reg) +#else /* CONFIG_64BIT */ +static inline void wr_reg64(void __iomem *reg, u64 data) { - return ((u64)rd_reg32(REG64_MS32(reg)) << 32 | - (u64)rd_reg32(REG64_LS32(reg))); +#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX + if (caam_little_end) { + wr_reg32((u32 __iomem *)(reg) + 1, data >> 32); + wr_reg32((u32 __iomem *)(reg), data); + } else +#endif + { + wr_reg32((u32 __iomem *)(reg), data >> 32); + wr_reg32((u32 __iomem *)(reg) + 1, data); + } } + +static inline u64 rd_reg64(void __iomem *reg) +{ +#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX + if (caam_little_end) + return ((u64)rd_reg32((u32 __iomem *)(reg) + 1) << 32 | + (u64)rd_reg32((u32 __iomem *)(reg))); + else #endif + return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 | + (u64)rd_reg32((u32 __iomem *)(reg) + 1)); +} +#endif /* CONFIG_64BIT */ + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT +#ifdef CONFIG_SOC_IMX7D +#define cpu_to_caam_dma(value) \ + (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \ + (u64)cpu_to_caam32(upper_32_bits(value))) +#define caam_dma_to_cpu(value) \ + (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | \ + (u64)caam32_to_cpu(upper_32_bits(value))) +#else +#define cpu_to_caam_dma(value) cpu_to_caam64(value) +#define caam_dma_to_cpu(value) caam64_to_cpu(value) +#endif /* CONFIG_SOC_IMX7D */ +#else +#define cpu_to_caam_dma(value) cpu_to_caam32(value) +#define caam_dma_to_cpu(value) caam32_to_cpu(value) +#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ /* * jr_outentry @@ -249,6 +298,8 @@ struct caam_perfmon { u32 faultliodn; /* FALR - Fault Address LIODN */ u32 faultdetail; /* FADR - Fault Addr Detail */ u32 rsvd2; +#define CSTA_PLEND BIT(10) +#define CSTA_ALT_PLEND BIT(18) u32 status; /* CSTA - CAAM Status */ u64 rsvd3; diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index 12ec6616e89d..19dc64fede0d 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -5,18 +5,19 @@ * */ +#include "regs.h" + struct sec4_sg_entry; /* * convert single dma address to h/w link table format */ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, - dma_addr_t dma, u32 len, u32 offset) + dma_addr_t dma, u32 len, u16 offset) { - sec4_sg_ptr->ptr = dma; - sec4_sg_ptr->len = len; - sec4_sg_ptr->buf_pool_id = 0; - sec4_sg_ptr->offset = offset; + sec4_sg_ptr->ptr = cpu_to_caam_dma(dma); + sec4_sg_ptr->len = cpu_to_caam32(len); + sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); #ifdef DEBUG print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, @@ -30,7 +31,7 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, */ static inline struct sec4_sg_entry * sg_to_sec4_sg(struct scatterlist *sg, int sg_count, - struct sec4_sg_entry *sec4_sg_ptr, u32 offset) + struct sec4_sg_entry *sec4_sg_ptr, u16 offset) { while (sg_count) { dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), @@ -48,10 +49,10 @@ sg_to_sec4_sg(struct scatterlist *sg, int sg_count, */ static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count, struct sec4_sg_entry *sec4_sg_ptr, - u32 offset) + u16 offset) { sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); - sec4_sg_ptr->len |= SEC4_SG_LEN_FIN; + sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); } static inline struct sec4_sg_entry *sg_to_sec4_sg_len( diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c index 0d0d4529ee36..58a4244b4752 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c @@ -14,9 +14,8 @@ #include #include #include -#include -#include #include +#include #include #include "ccp-crypto.h" @@ -110,15 +109,12 @@ static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, ctx->u.aes.key_len = key_len / 2; sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); - return crypto_ablkcipher_setkey(ctx->u.aes.tfm_ablkcipher, key, - key_len); + return crypto_skcipher_setkey(ctx->u.aes.tfm_skcipher, key, key_len); } static int ccp_aes_xts_crypt(struct ablkcipher_request *req, unsigned int encrypt) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); unsigned int unit; @@ -146,14 +142,19 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req, if ((unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) || (ctx->u.aes.key_len != AES_KEYSIZE_128)) { + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->u.aes.tfm_skcipher); + /* Use the fallback to process the request for any * unsupported unit sizes or key sizes */ - ablkcipher_request_set_tfm(req, ctx->u.aes.tfm_ablkcipher); - ret = (encrypt) ? crypto_ablkcipher_encrypt(req) : - crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); - + skcipher_request_set_tfm(subreq, ctx->u.aes.tfm_skcipher); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + ret = encrypt ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return ret; } @@ -192,23 +193,21 @@ static int ccp_aes_xts_decrypt(struct ablkcipher_request *req) static int ccp_aes_xts_cra_init(struct crypto_tfm *tfm) { struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_ablkcipher *fallback_tfm; + struct crypto_skcipher *fallback_tfm; ctx->complete = ccp_aes_xts_complete; ctx->u.aes.key_len = 0; - fallback_tfm = crypto_alloc_ablkcipher(crypto_tfm_alg_name(tfm), 0, - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK); + fallback_tfm = crypto_alloc_skcipher("xts(aes)", 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback_tfm)) { - pr_warn("could not load fallback driver %s\n", - crypto_tfm_alg_name(tfm)); + pr_warn("could not load fallback driver xts(aes)\n"); return PTR_ERR(fallback_tfm); } - ctx->u.aes.tfm_ablkcipher = fallback_tfm; + ctx->u.aes.tfm_skcipher = fallback_tfm; - tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx) + - fallback_tfm->base.crt_ablkcipher.reqsize; + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx); return 0; } @@ -217,9 +216,7 @@ static void ccp_aes_xts_cra_exit(struct crypto_tfm *tfm) { struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->u.aes.tfm_ablkcipher) - crypto_free_ablkcipher(ctx->u.aes.tfm_ablkcipher); - ctx->u.aes.tfm_ablkcipher = NULL; + crypto_free_skcipher(ctx->u.aes.tfm_skcipher); } static int ccp_register_aes_xts_alg(struct list_head *head, diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index a326ec20bfa8..8335b32e815e 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -69,7 +68,7 @@ static inline struct ccp_crypto_ahash_alg * /***** AES related defines *****/ struct ccp_aes_ctx { /* Fallback cipher for XTS with unsupported unit sizes */ - struct crypto_ablkcipher *tfm_ablkcipher; + struct crypto_skcipher *tfm_skcipher; /* Cipher used to generate CMAC K1/K2 keys */ struct crypto_cipher *tfm_cipher; diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index e8ef9fd24a16..e373cc6557c6 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -31,22 +31,42 @@ #include "cesa.h" +/* Limit of the crypto queue before reaching the backlog */ +#define CESA_CRYPTO_DEFAULT_MAX_QLEN 128 + static int allhwsupport = !IS_ENABLED(CONFIG_CRYPTO_DEV_MV_CESA); module_param_named(allhwsupport, allhwsupport, int, 0444); MODULE_PARM_DESC(allhwsupport, "Enable support for all hardware (even it if overlaps with the mv_cesa driver)"); struct mv_cesa_dev *cesa_dev; -static void mv_cesa_dequeue_req_unlocked(struct mv_cesa_engine *engine) +struct crypto_async_request * +mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine, + struct crypto_async_request **backlog) { - struct crypto_async_request *req, *backlog; + struct crypto_async_request *req; + + *backlog = crypto_get_backlog(&engine->queue); + req = crypto_dequeue_request(&engine->queue); + + if (!req) + return NULL; + + return req; +} + +static void mv_cesa_rearm_engine(struct mv_cesa_engine *engine) +{ + struct crypto_async_request *req = NULL, *backlog = NULL; struct mv_cesa_ctx *ctx; - spin_lock_bh(&cesa_dev->lock); - backlog = crypto_get_backlog(&cesa_dev->queue); - req = crypto_dequeue_request(&cesa_dev->queue); - engine->req = req; - spin_unlock_bh(&cesa_dev->lock); + + spin_lock_bh(&engine->lock); + if (!engine->req) { + req = mv_cesa_dequeue_req_locked(engine, &backlog); + engine->req = req; + } + spin_unlock_bh(&engine->lock); if (!req) return; @@ -55,8 +75,47 @@ static void mv_cesa_dequeue_req_unlocked(struct mv_cesa_engine *engine) backlog->complete(backlog, -EINPROGRESS); ctx = crypto_tfm_ctx(req->tfm); - ctx->ops->prepare(req, engine); ctx->ops->step(req); + + return; +} + +static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) +{ + struct crypto_async_request *req; + struct mv_cesa_ctx *ctx; + int res; + + req = engine->req; + ctx = crypto_tfm_ctx(req->tfm); + res = ctx->ops->process(req, status); + + if (res == 0) { + ctx->ops->complete(req); + mv_cesa_engine_enqueue_complete_request(engine, req); + } else if (res == -EINPROGRESS) { + ctx->ops->step(req); + } + + return res; +} + +static int mv_cesa_int_process(struct mv_cesa_engine *engine, u32 status) +{ + if (engine->chain.first && engine->chain.last) + return mv_cesa_tdma_process(engine, status); + + return mv_cesa_std_process(engine, status); +} + +static inline void +mv_cesa_complete_req(struct mv_cesa_ctx *ctx, struct crypto_async_request *req, + int res) +{ + ctx->ops->cleanup(req); + local_bh_disable(); + req->complete(req, res); + local_bh_enable(); } static irqreturn_t mv_cesa_int(int irq, void *priv) @@ -83,49 +142,54 @@ static irqreturn_t mv_cesa_int(int irq, void *priv) writel(~status, engine->regs + CESA_SA_FPGA_INT_STATUS); writel(~status, engine->regs + CESA_SA_INT_STATUS); + /* Process fetched requests */ + res = mv_cesa_int_process(engine, status & mask); ret = IRQ_HANDLED; + spin_lock_bh(&engine->lock); req = engine->req; + if (res != -EINPROGRESS) + engine->req = NULL; spin_unlock_bh(&engine->lock); - if (req) { - ctx = crypto_tfm_ctx(req->tfm); - res = ctx->ops->process(req, status & mask); - if (res != -EINPROGRESS) { - spin_lock_bh(&engine->lock); - engine->req = NULL; - mv_cesa_dequeue_req_unlocked(engine); - spin_unlock_bh(&engine->lock); - ctx->ops->cleanup(req); - local_bh_disable(); - req->complete(req, res); - local_bh_enable(); - } else { - ctx->ops->step(req); - } + + ctx = crypto_tfm_ctx(req->tfm); + + if (res && res != -EINPROGRESS) + mv_cesa_complete_req(ctx, req, res); + + /* Launch the next pending request */ + mv_cesa_rearm_engine(engine); + + /* Iterate over the complete queue */ + while (true) { + req = mv_cesa_engine_dequeue_complete_request(engine); + if (!req) + break; + + mv_cesa_complete_req(ctx, req, 0); } } return ret; } -int mv_cesa_queue_req(struct crypto_async_request *req) +int mv_cesa_queue_req(struct crypto_async_request *req, + struct mv_cesa_req *creq) { int ret; - int i; + struct mv_cesa_engine *engine = creq->engine; + + spin_lock_bh(&engine->lock); + if (mv_cesa_req_get_type(creq) == CESA_DMA_REQ) + mv_cesa_tdma_chain(engine, creq); - spin_lock_bh(&cesa_dev->lock); - ret = crypto_enqueue_request(&cesa_dev->queue, req); - spin_unlock_bh(&cesa_dev->lock); + ret = crypto_enqueue_request(&engine->queue, req); + spin_unlock_bh(&engine->lock); if (ret != -EINPROGRESS) return ret; - for (i = 0; i < cesa_dev->caps->nengines; i++) { - spin_lock_bh(&cesa_dev->engines[i].lock); - if (!cesa_dev->engines[i].req) - mv_cesa_dequeue_req_unlocked(&cesa_dev->engines[i]); - spin_unlock_bh(&cesa_dev->engines[i].lock); - } + mv_cesa_rearm_engine(engine); return -EINPROGRESS; } @@ -309,6 +373,10 @@ static int mv_cesa_dev_dma_init(struct mv_cesa_dev *cesa) if (!dma->padding_pool) return -ENOMEM; + dma->iv_pool = dmam_pool_create("cesa_iv", dev, 16, 1, 0); + if (!dma->iv_pool) + return -ENOMEM; + cesa->dma = dma; return 0; @@ -416,7 +484,7 @@ static int mv_cesa_probe(struct platform_device *pdev) return -ENOMEM; spin_lock_init(&cesa->lock); - crypto_init_queue(&cesa->queue, 50); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); cesa->regs = devm_ioremap_resource(dev, res); if (IS_ERR(cesa->regs)) @@ -489,6 +557,10 @@ static int mv_cesa_probe(struct platform_device *pdev) engine); if (ret) goto err_cleanup; + + crypto_init_queue(&engine->queue, CESA_CRYPTO_DEFAULT_MAX_QLEN); + atomic_set(&engine->load, 0); + INIT_LIST_HEAD(&engine->complete_queue); } cesa_dev = cesa; diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index 74071e45ada0..e423d33decd4 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -271,10 +271,13 @@ struct mv_cesa_op_ctx { /* TDMA descriptor flags */ #define CESA_TDMA_DST_IN_SRAM BIT(31) #define CESA_TDMA_SRC_IN_SRAM BIT(30) -#define CESA_TDMA_TYPE_MSK GENMASK(29, 0) +#define CESA_TDMA_END_OF_REQ BIT(29) +#define CESA_TDMA_BREAK_CHAIN BIT(28) +#define CESA_TDMA_TYPE_MSK GENMASK(27, 0) #define CESA_TDMA_DUMMY 0 #define CESA_TDMA_DATA 1 #define CESA_TDMA_OP 2 +#define CESA_TDMA_IV 3 /** * struct mv_cesa_tdma_desc - TDMA descriptor @@ -390,6 +393,7 @@ struct mv_cesa_dev_dma { struct dma_pool *op_pool; struct dma_pool *cache_pool; struct dma_pool *padding_pool; + struct dma_pool *iv_pool; }; /** @@ -398,7 +402,6 @@ struct mv_cesa_dev_dma { * @regs: device registers * @sram_size: usable SRAM size * @lock: device lock - * @queue: crypto request queue * @engines: array of engines * @dma: dma pools * @@ -410,7 +413,6 @@ struct mv_cesa_dev { struct device *dev; unsigned int sram_size; spinlock_t lock; - struct crypto_queue queue; struct mv_cesa_engine *engines; struct mv_cesa_dev_dma *dma; }; @@ -429,6 +431,11 @@ struct mv_cesa_dev { * @int_mask: interrupt mask cache * @pool: memory pool pointing to the memory region reserved in * SRAM + * @queue: fifo of the pending crypto requests + * @load: engine load counter, useful for load balancing + * @chain: list of the current tdma descriptors being processed + * by this engine. + * @complete_queue: fifo of the processed requests by the engine * * Structure storing CESA engine information. */ @@ -444,23 +451,27 @@ struct mv_cesa_engine { size_t max_req_len; u32 int_mask; struct gen_pool *pool; + struct crypto_queue queue; + atomic_t load; + struct mv_cesa_tdma_chain chain; + struct list_head complete_queue; }; /** * struct mv_cesa_req_ops - CESA request operations - * @prepare: prepare a request to be executed on the specified engine * @process: process a request chunk result (should return 0 if the * operation, -EINPROGRESS if it needs more steps or an error * code) * @step: launch the crypto operation on the next chunk * @cleanup: cleanup the crypto request (release associated data) + * @complete: complete the request, i.e copy result or context from sram when + * needed. */ struct mv_cesa_req_ops { - void (*prepare)(struct crypto_async_request *req, - struct mv_cesa_engine *engine); int (*process)(struct crypto_async_request *req, u32 status); void (*step)(struct crypto_async_request *req); void (*cleanup)(struct crypto_async_request *req); + void (*complete)(struct crypto_async_request *req); }; /** @@ -507,21 +518,11 @@ enum mv_cesa_req_type { /** * struct mv_cesa_req - CESA request - * @type: request type * @engine: engine associated with this request + * @chain: list of tdma descriptors associated with this request */ struct mv_cesa_req { - enum mv_cesa_req_type type; struct mv_cesa_engine *engine; -}; - -/** - * struct mv_cesa_tdma_req - CESA TDMA request - * @base: base information - * @chain: TDMA chain - */ -struct mv_cesa_tdma_req { - struct mv_cesa_req base; struct mv_cesa_tdma_chain chain; }; @@ -538,13 +539,11 @@ struct mv_cesa_sg_std_iter { /** * struct mv_cesa_ablkcipher_std_req - cipher standard request - * @base: base information * @op: operation context * @offset: current operation offset * @size: size of the crypto operation */ struct mv_cesa_ablkcipher_std_req { - struct mv_cesa_req base; struct mv_cesa_op_ctx op; unsigned int offset; unsigned int size; @@ -558,34 +557,27 @@ struct mv_cesa_ablkcipher_std_req { * @dst_nents: number of entries in the dest sg list */ struct mv_cesa_ablkcipher_req { - union { - struct mv_cesa_req base; - struct mv_cesa_tdma_req dma; - struct mv_cesa_ablkcipher_std_req std; - } req; + struct mv_cesa_req base; + struct mv_cesa_ablkcipher_std_req std; int src_nents; int dst_nents; }; /** * struct mv_cesa_ahash_std_req - standard hash request - * @base: base information * @offset: current operation offset */ struct mv_cesa_ahash_std_req { - struct mv_cesa_req base; unsigned int offset; }; /** * struct mv_cesa_ahash_dma_req - DMA hash request - * @base: base information * @padding: padding buffer * @padding_dma: DMA address of the padding buffer * @cache_dma: DMA address of the cache buffer */ struct mv_cesa_ahash_dma_req { - struct mv_cesa_tdma_req base; u8 *padding; dma_addr_t padding_dma; u8 *cache; @@ -604,8 +596,8 @@ struct mv_cesa_ahash_dma_req { * @state: hash state */ struct mv_cesa_ahash_req { + struct mv_cesa_req base; union { - struct mv_cesa_req base; struct mv_cesa_ahash_dma_req dma; struct mv_cesa_ahash_std_req std; } req; @@ -623,6 +615,35 @@ struct mv_cesa_ahash_req { extern struct mv_cesa_dev *cesa_dev; + +static inline void +mv_cesa_engine_enqueue_complete_request(struct mv_cesa_engine *engine, + struct crypto_async_request *req) +{ + list_add_tail(&req->list, &engine->complete_queue); +} + +static inline struct crypto_async_request * +mv_cesa_engine_dequeue_complete_request(struct mv_cesa_engine *engine) +{ + struct crypto_async_request *req; + + req = list_first_entry_or_null(&engine->complete_queue, + struct crypto_async_request, + list); + if (req) + list_del(&req->list); + + return req; +} + + +static inline enum mv_cesa_req_type +mv_cesa_req_get_type(struct mv_cesa_req *req) +{ + return req->chain.first ? CESA_DMA_REQ : CESA_STD_REQ; +} + static inline void mv_cesa_update_op_cfg(struct mv_cesa_op_ctx *op, u32 cfg, u32 mask) { @@ -695,7 +716,32 @@ static inline bool mv_cesa_mac_op_is_first_frag(const struct mv_cesa_op_ctx *op) CESA_SA_DESC_CFG_FIRST_FRAG; } -int mv_cesa_queue_req(struct crypto_async_request *req); +int mv_cesa_queue_req(struct crypto_async_request *req, + struct mv_cesa_req *creq); + +struct crypto_async_request * +mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine, + struct crypto_async_request **backlog); + +static inline struct mv_cesa_engine *mv_cesa_select_engine(int weight) +{ + int i; + u32 min_load = U32_MAX; + struct mv_cesa_engine *selected = NULL; + + for (i = 0; i < cesa_dev->caps->nengines; i++) { + struct mv_cesa_engine *engine = cesa_dev->engines + i; + u32 load = atomic_read(&engine->load); + if (load < min_load) { + min_load = load; + selected = engine; + } + } + + atomic_add(weight, &selected->load); + + return selected; +} /* * Helper function that indicates whether a crypto request needs to be @@ -765,9 +811,9 @@ static inline bool mv_cesa_req_dma_iter_next_op(struct mv_cesa_dma_iter *iter) return iter->op_len; } -void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq); +void mv_cesa_dma_step(struct mv_cesa_req *dreq); -static inline int mv_cesa_dma_process(struct mv_cesa_tdma_req *dreq, +static inline int mv_cesa_dma_process(struct mv_cesa_req *dreq, u32 status) { if (!(status & CESA_SA_INT_ACC0_IDMA_DONE)) @@ -779,10 +825,13 @@ static inline int mv_cesa_dma_process(struct mv_cesa_tdma_req *dreq, return 0; } -void mv_cesa_dma_prepare(struct mv_cesa_tdma_req *dreq, +void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, struct mv_cesa_engine *engine); +void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq); +void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, + struct mv_cesa_req *dreq); +int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status); -void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq); static inline void mv_cesa_tdma_desc_iter_init(struct mv_cesa_tdma_chain *chain) @@ -790,6 +839,9 @@ mv_cesa_tdma_desc_iter_init(struct mv_cesa_tdma_chain *chain) memset(chain, 0, sizeof(*chain)); } +int mv_cesa_dma_add_iv_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, + u32 size, u32 flags, gfp_t gfp_flags); + struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, const struct mv_cesa_op_ctx *op_templ, bool skip_ctx, diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index dcf1fceb9336..48df03a06066 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -70,25 +70,28 @@ mv_cesa_ablkcipher_dma_cleanup(struct ablkcipher_request *req) dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_BIDIRECTIONAL); } - mv_cesa_dma_cleanup(&creq->req.dma); + mv_cesa_dma_cleanup(&creq->base); } static inline void mv_cesa_ablkcipher_cleanup(struct ablkcipher_request *req) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ablkcipher_dma_cleanup(req); } static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_engine *engine = creq->base.engine; size_t len = min_t(size_t, req->nbytes - sreq->offset, CESA_SA_SRAM_PAYLOAD_SIZE); + mv_cesa_adjust_op(engine, &sreq->op); + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + len = sg_pcopy_to_buffer(req->src, creq->src_nents, engine->sram + CESA_SA_DATA_SRAM_OFFSET, len, sreq->offset); @@ -106,6 +109,8 @@ static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + BUG_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } @@ -113,8 +118,8 @@ static int mv_cesa_ablkcipher_std_process(struct ablkcipher_request *req, u32 status) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_engine *engine = creq->base.engine; size_t len; len = sg_pcopy_from_buffer(req->dst, creq->dst_nents, @@ -133,21 +138,19 @@ static int mv_cesa_ablkcipher_process(struct crypto_async_request *req, { struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_req *basereq = &creq->base; + unsigned int ivsize; int ret; - if (creq->req.base.type == CESA_DMA_REQ) - ret = mv_cesa_dma_process(&creq->req.dma, status); - else - ret = mv_cesa_ablkcipher_std_process(ablkreq, status); + if (mv_cesa_req_get_type(basereq) == CESA_STD_REQ) + return mv_cesa_ablkcipher_std_process(ablkreq, status); + ret = mv_cesa_dma_process(basereq, status); if (ret) return ret; - memcpy_fromio(ablkreq->info, - engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, - crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq))); + ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq)); + memcpy_fromio(ablkreq->info, basereq->chain.last->data, ivsize); return 0; } @@ -157,8 +160,8 @@ static void mv_cesa_ablkcipher_step(struct crypto_async_request *req) struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); - if (creq->req.base.type == CESA_DMA_REQ) - mv_cesa_dma_step(&creq->req.dma); + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_dma_step(&creq->base); else mv_cesa_ablkcipher_std_step(ablkreq); } @@ -167,22 +170,19 @@ static inline void mv_cesa_ablkcipher_dma_prepare(struct ablkcipher_request *req) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_tdma_req *dreq = &creq->req.dma; + struct mv_cesa_req *basereq = &creq->base; - mv_cesa_dma_prepare(dreq, dreq->base.engine); + mv_cesa_dma_prepare(basereq, basereq->engine); } static inline void mv_cesa_ablkcipher_std_prepare(struct ablkcipher_request *req) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; sreq->size = 0; sreq->offset = 0; - mv_cesa_adjust_op(engine, &sreq->op); - memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); } static inline void mv_cesa_ablkcipher_prepare(struct crypto_async_request *req, @@ -190,9 +190,9 @@ static inline void mv_cesa_ablkcipher_prepare(struct crypto_async_request *req, { struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); - creq->req.base.engine = engine; + creq->base.engine = engine; - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ablkcipher_dma_prepare(ablkreq); else mv_cesa_ablkcipher_std_prepare(ablkreq); @@ -206,11 +206,34 @@ mv_cesa_ablkcipher_req_cleanup(struct crypto_async_request *req) mv_cesa_ablkcipher_cleanup(ablkreq); } +static void +mv_cesa_ablkcipher_complete(struct crypto_async_request *req) +{ + struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct mv_cesa_engine *engine = creq->base.engine; + unsigned int ivsize; + + atomic_sub(ablkreq->nbytes, &engine->load); + ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq)); + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) { + struct mv_cesa_req *basereq; + + basereq = &creq->base; + memcpy(ablkreq->info, basereq->chain.last->data, ivsize); + } else { + memcpy_fromio(ablkreq->info, + engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, + ivsize); + } +} + static const struct mv_cesa_req_ops mv_cesa_ablkcipher_req_ops = { .step = mv_cesa_ablkcipher_step, .process = mv_cesa_ablkcipher_process, - .prepare = mv_cesa_ablkcipher_prepare, .cleanup = mv_cesa_ablkcipher_req_cleanup, + .complete = mv_cesa_ablkcipher_complete, }; static int mv_cesa_ablkcipher_cra_init(struct crypto_tfm *tfm) @@ -295,15 +318,15 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - struct mv_cesa_tdma_req *dreq = &creq->req.dma; + struct mv_cesa_req *basereq = &creq->base; struct mv_cesa_ablkcipher_dma_iter iter; struct mv_cesa_tdma_chain chain; bool skip_ctx = false; int ret; + unsigned int ivsize; - dreq->base.type = CESA_DMA_REQ; - dreq->chain.first = NULL; - dreq->chain.last = NULL; + basereq->chain.first = NULL; + basereq->chain.last = NULL; if (req->src != req->dst) { ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, @@ -358,12 +381,21 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, } while (mv_cesa_ablkcipher_req_iter_next_op(&iter)); - dreq->chain = chain; + /* Add output data for IV */ + ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req)); + ret = mv_cesa_dma_add_iv_op(&chain, CESA_SA_CRYPT_IV_SRAM_OFFSET, + ivsize, CESA_TDMA_SRC_IN_SRAM, flags); + + if (ret) + goto err_free_tdma; + + basereq->chain = chain; + basereq->chain.last->flags |= CESA_TDMA_END_OF_REQ; return 0; err_free_tdma: - mv_cesa_dma_cleanup(dreq); + mv_cesa_dma_cleanup(basereq); if (req->dst != req->src) dma_unmap_sg(cesa_dev->dev, req->dst, creq->dst_nents, DMA_FROM_DEVICE); @@ -380,11 +412,13 @@ mv_cesa_ablkcipher_std_req_init(struct ablkcipher_request *req, const struct mv_cesa_op_ctx *op_templ) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; + struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_req *basereq = &creq->base; - sreq->base.type = CESA_STD_REQ; sreq->op = *op_templ; sreq->skip_ctx = false; + basereq->chain.first = NULL; + basereq->chain.last = NULL; return 0; } @@ -414,7 +448,6 @@ static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_OP_CRYPT_ONLY, CESA_SA_DESC_CFG_OP_MSK); - /* TODO: add a threshold for DMA usage */ if (cesa_dev->caps->has_tdma) ret = mv_cesa_ablkcipher_dma_req_init(req, tmpl); else @@ -423,28 +456,41 @@ static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, return ret; } -static int mv_cesa_des_op(struct ablkcipher_request *req, - struct mv_cesa_op_ctx *tmpl) +static int mv_cesa_ablkcipher_queue_req(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) { - struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm); int ret; - - mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_DES, - CESA_SA_DESC_CFG_CRYPTM_MSK); - - memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE); + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_engine *engine; ret = mv_cesa_ablkcipher_req_init(req, tmpl); if (ret) return ret; - ret = mv_cesa_queue_req(&req->base); + engine = mv_cesa_select_engine(req->nbytes); + mv_cesa_ablkcipher_prepare(&req->base, engine); + + ret = mv_cesa_queue_req(&req->base, &creq->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) mv_cesa_ablkcipher_cleanup(req); return ret; } +static int mv_cesa_des_op(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_DES, + CESA_SA_DESC_CFG_CRYPTM_MSK); + + memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE); + + return mv_cesa_ablkcipher_queue_req(req, tmpl); +} + static int mv_cesa_ecb_des_encrypt(struct ablkcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -547,22 +593,13 @@ static int mv_cesa_des3_op(struct ablkcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_des3_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - int ret; mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_3DES, CESA_SA_DESC_CFG_CRYPTM_MSK); memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES3_EDE_KEY_SIZE); - ret = mv_cesa_ablkcipher_req_init(req, tmpl); - if (ret) - return ret; - - ret = mv_cesa_queue_req(&req->base); - if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ablkcipher_cleanup(req); - - return ret; + return mv_cesa_ablkcipher_queue_req(req, tmpl); } static int mv_cesa_ecb_des3_ede_encrypt(struct ablkcipher_request *req) @@ -673,7 +710,7 @@ static int mv_cesa_aes_op(struct ablkcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - int ret, i; + int i; u32 *key; u32 cfg; @@ -696,15 +733,7 @@ static int mv_cesa_aes_op(struct ablkcipher_request *req, CESA_SA_DESC_CFG_CRYPTM_MSK | CESA_SA_DESC_CFG_AES_LEN_MSK); - ret = mv_cesa_ablkcipher_req_init(req, tmpl); - if (ret) - return ret; - - ret = mv_cesa_queue_req(&req->base); - if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ablkcipher_cleanup(req); - - return ret; + return mv_cesa_ablkcipher_queue_req(req, tmpl); } static int mv_cesa_ecb_aes_encrypt(struct ablkcipher_request *req) diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 7a5058da9151..c35912b4fffb 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -103,14 +103,14 @@ static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req) dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); mv_cesa_ahash_dma_free_cache(&creq->req.dma); - mv_cesa_dma_cleanup(&creq->req.dma.base); + mv_cesa_dma_cleanup(&creq->base); } static inline void mv_cesa_ahash_cleanup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ahash_dma_cleanup(req); } @@ -118,7 +118,7 @@ static void mv_cesa_ahash_last_cleanup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ahash_dma_last_cleanup(req); } @@ -157,11 +157,23 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_ahash_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_engine *engine = creq->base.engine; struct mv_cesa_op_ctx *op; unsigned int new_cache_ptr = 0; u32 frag_mode; size_t len; + unsigned int digsize; + int i; + + mv_cesa_adjust_op(engine, &creq->op_tmpl); + memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); + + digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + for (i = 0; i < digsize / 4; i++) + writel_relaxed(creq->state[i], engine->regs + CESA_IVDIG(i)); + + mv_cesa_adjust_op(engine, &creq->op_tmpl); + memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); if (creq->cache_ptr) memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, @@ -237,6 +249,8 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + BUG_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } @@ -254,20 +268,17 @@ static int mv_cesa_ahash_std_process(struct ahash_request *req, u32 status) static inline void mv_cesa_ahash_dma_prepare(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - struct mv_cesa_tdma_req *dreq = &creq->req.dma.base; + struct mv_cesa_req *basereq = &creq->base; - mv_cesa_dma_prepare(dreq, dreq->base.engine); + mv_cesa_dma_prepare(basereq, basereq->engine); } static void mv_cesa_ahash_std_prepare(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_ahash_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; sreq->offset = 0; - mv_cesa_adjust_op(engine, &creq->op_tmpl); - memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); } static void mv_cesa_ahash_step(struct crypto_async_request *req) @@ -275,8 +286,8 @@ static void mv_cesa_ahash_step(struct crypto_async_request *req) struct ahash_request *ahashreq = ahash_request_cast(req); struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); - if (creq->req.base.type == CESA_DMA_REQ) - mv_cesa_dma_step(&creq->req.dma.base); + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_dma_step(&creq->base); else mv_cesa_ahash_std_step(ahashreq); } @@ -285,17 +296,20 @@ static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) { struct ahash_request *ahashreq = ahash_request_cast(req); struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); - struct mv_cesa_engine *engine = creq->req.base.engine; - unsigned int digsize; - int ret, i; - if (creq->req.base.type == CESA_DMA_REQ) - ret = mv_cesa_dma_process(&creq->req.dma.base, status); - else - ret = mv_cesa_ahash_std_process(ahashreq, status); + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + return mv_cesa_dma_process(&creq->base, status); - if (ret == -EINPROGRESS) - return ret; + return mv_cesa_ahash_std_process(ahashreq, status); +} + +static void mv_cesa_ahash_complete(struct crypto_async_request *req) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + struct mv_cesa_engine *engine = creq->base.engine; + unsigned int digsize; + int i; digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); for (i = 0; i < digsize / 4; i++) @@ -325,7 +339,7 @@ static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) } } - return ret; + atomic_sub(ahashreq->nbytes, &engine->load); } static void mv_cesa_ahash_prepare(struct crypto_async_request *req, @@ -333,19 +347,13 @@ static void mv_cesa_ahash_prepare(struct crypto_async_request *req, { struct ahash_request *ahashreq = ahash_request_cast(req); struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); - unsigned int digsize; - int i; - creq->req.base.engine = engine; + creq->base.engine = engine; - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ahash_dma_prepare(ahashreq); else mv_cesa_ahash_std_prepare(ahashreq); - - digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); - for (i = 0; i < digsize / 4; i++) - writel_relaxed(creq->state[i], engine->regs + CESA_IVDIG(i)); } static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req) @@ -362,8 +370,8 @@ static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req) static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops = { .step = mv_cesa_ahash_step, .process = mv_cesa_ahash_process, - .prepare = mv_cesa_ahash_prepare, .cleanup = mv_cesa_ahash_req_cleanup, + .complete = mv_cesa_ahash_complete, }; static int mv_cesa_ahash_init(struct ahash_request *req, @@ -553,15 +561,14 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; - struct mv_cesa_tdma_req *dreq = &ahashdreq->base; + struct mv_cesa_req *basereq = &creq->base; struct mv_cesa_ahash_dma_iter iter; struct mv_cesa_op_ctx *op = NULL; unsigned int frag_len; int ret; - dreq->chain.first = NULL; - dreq->chain.last = NULL; + basereq->chain.first = NULL; + basereq->chain.last = NULL; if (creq->src_nents) { ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, @@ -572,14 +579,14 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) } } - mv_cesa_tdma_desc_iter_init(&dreq->chain); + mv_cesa_tdma_desc_iter_init(&basereq->chain); mv_cesa_ahash_req_iter_init(&iter, req); /* * Add the cache (left-over data from a previous block) first. * This will never overflow the SRAM size. */ - ret = mv_cesa_ahash_dma_add_cache(&dreq->chain, &iter, creq, flags); + ret = mv_cesa_ahash_dma_add_cache(&basereq->chain, &iter, creq, flags); if (ret) goto err_free_tdma; @@ -590,7 +597,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) * data. We intentionally do not add the final op block. */ while (true) { - ret = mv_cesa_dma_add_op_transfers(&dreq->chain, + ret = mv_cesa_dma_add_op_transfers(&basereq->chain, &iter.base, &iter.src, flags); if (ret) @@ -601,7 +608,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (!mv_cesa_ahash_req_iter_next_op(&iter)) break; - op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, + op = mv_cesa_dma_add_frag(&basereq->chain, &creq->op_tmpl, frag_len, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); @@ -619,10 +626,10 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) * operation, which depends whether this is the final request. */ if (creq->last_req) - op = mv_cesa_ahash_dma_last_req(&dreq->chain, &iter, creq, + op = mv_cesa_ahash_dma_last_req(&basereq->chain, &iter, creq, frag_len, flags); else if (frag_len) - op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, + op = mv_cesa_dma_add_frag(&basereq->chain, &creq->op_tmpl, frag_len, flags); if (IS_ERR(op)) { @@ -632,7 +639,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (op) { /* Add dummy desc to wait for crypto operation end */ - ret = mv_cesa_dma_add_dummy_end(&dreq->chain, flags); + ret = mv_cesa_dma_add_dummy_end(&basereq->chain, flags); if (ret) goto err_free_tdma; } @@ -643,10 +650,13 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) else creq->cache_ptr = 0; + basereq->chain.last->flags |= (CESA_TDMA_END_OF_REQ | + CESA_TDMA_BREAK_CHAIN); + return 0; err_free_tdma: - mv_cesa_dma_cleanup(dreq); + mv_cesa_dma_cleanup(basereq); dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); err: @@ -660,11 +670,6 @@ static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached) struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); int ret; - if (cesa_dev->caps->has_tdma) - creq->req.base.type = CESA_DMA_REQ; - else - creq->req.base.type = CESA_STD_REQ; - creq->src_nents = sg_nents_for_len(req->src, req->nbytes); if (creq->src_nents < 0) { dev_err(cesa_dev->dev, "Invalid number of src SG"); @@ -678,19 +683,19 @@ static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached) if (*cached) return 0; - if (creq->req.base.type == CESA_DMA_REQ) + if (cesa_dev->caps->has_tdma) ret = mv_cesa_ahash_dma_req_init(req); return ret; } -static int mv_cesa_ahash_update(struct ahash_request *req) +static int mv_cesa_ahash_queue_req(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_engine *engine; bool cached = false; int ret; - creq->len += req->nbytes; ret = mv_cesa_ahash_req_init(req, &cached); if (ret) return ret; @@ -698,61 +703,48 @@ static int mv_cesa_ahash_update(struct ahash_request *req) if (cached) return 0; - ret = mv_cesa_queue_req(&req->base); + engine = mv_cesa_select_engine(req->nbytes); + mv_cesa_ahash_prepare(&req->base, engine); + + ret = mv_cesa_queue_req(&req->base, &creq->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) mv_cesa_ahash_cleanup(req); return ret; } +static int mv_cesa_ahash_update(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + creq->len += req->nbytes; + + return mv_cesa_ahash_queue_req(req); +} + static int mv_cesa_ahash_final(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; - bool cached = false; - int ret; mv_cesa_set_mac_op_total_len(tmpl, creq->len); creq->last_req = true; req->nbytes = 0; - ret = mv_cesa_ahash_req_init(req, &cached); - if (ret) - return ret; - - if (cached) - return 0; - - ret = mv_cesa_queue_req(&req->base); - if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ahash_cleanup(req); - - return ret; + return mv_cesa_ahash_queue_req(req); } static int mv_cesa_ahash_finup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; - bool cached = false; - int ret; creq->len += req->nbytes; mv_cesa_set_mac_op_total_len(tmpl, creq->len); creq->last_req = true; - ret = mv_cesa_ahash_req_init(req, &cached); - if (ret) - return ret; - - if (cached) - return 0; - - ret = mv_cesa_queue_req(&req->base); - if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ahash_cleanup(req); - - return ret; + return mv_cesa_ahash_queue_req(req); } static int mv_cesa_ahash_export(struct ahash_request *req, void *hash, diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index 0ad8f1ecf175..86a065bcc187 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -37,9 +37,9 @@ bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter, return true; } -void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq) +void mv_cesa_dma_step(struct mv_cesa_req *dreq) { - struct mv_cesa_engine *engine = dreq->base.engine; + struct mv_cesa_engine *engine = dreq->engine; writel_relaxed(0, engine->regs + CESA_SA_CFG); @@ -53,19 +53,25 @@ void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq) engine->regs + CESA_SA_CFG); writel_relaxed(dreq->chain.first->cur_dma, engine->regs + CESA_TDMA_NEXT_ADDR); + BUG_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } -void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq) +void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq) { struct mv_cesa_tdma_desc *tdma; for (tdma = dreq->chain.first; tdma;) { struct mv_cesa_tdma_desc *old_tdma = tdma; + u32 type = tdma->flags & CESA_TDMA_TYPE_MSK; - if (tdma->flags & CESA_TDMA_OP) + if (type == CESA_TDMA_OP) dma_pool_free(cesa_dev->dma->op_pool, tdma->op, le32_to_cpu(tdma->src)); + else if (type == CESA_TDMA_IV) + dma_pool_free(cesa_dev->dma->iv_pool, tdma->data, + le32_to_cpu(tdma->dst)); tdma = tdma->next; dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma, @@ -76,7 +82,7 @@ void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq) dreq->chain.last = NULL; } -void mv_cesa_dma_prepare(struct mv_cesa_tdma_req *dreq, +void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, struct mv_cesa_engine *engine) { struct mv_cesa_tdma_desc *tdma; @@ -88,11 +94,97 @@ void mv_cesa_dma_prepare(struct mv_cesa_tdma_req *dreq, if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) tdma->src = cpu_to_le32(tdma->src + engine->sram_dma); - if (tdma->flags & CESA_TDMA_OP) + if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP) mv_cesa_adjust_op(engine, tdma->op); } } +void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, + struct mv_cesa_req *dreq) +{ + if (engine->chain.first == NULL && engine->chain.last == NULL) { + engine->chain.first = dreq->chain.first; + engine->chain.last = dreq->chain.last; + } else { + struct mv_cesa_tdma_desc *last; + + last = engine->chain.last; + last->next = dreq->chain.first; + engine->chain.last = dreq->chain.last; + + if (!(last->flags & CESA_TDMA_BREAK_CHAIN)) + last->next_dma = dreq->chain.first->cur_dma; + } +} + +int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) +{ + struct crypto_async_request *req = NULL; + struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL; + dma_addr_t tdma_cur; + int res = 0; + + tdma_cur = readl(engine->regs + CESA_TDMA_CUR); + + for (tdma = engine->chain.first; tdma; tdma = next) { + spin_lock_bh(&engine->lock); + next = tdma->next; + spin_unlock_bh(&engine->lock); + + if (tdma->flags & CESA_TDMA_END_OF_REQ) { + struct crypto_async_request *backlog = NULL; + struct mv_cesa_ctx *ctx; + u32 current_status; + + spin_lock_bh(&engine->lock); + /* + * if req is NULL, this means we're processing the + * request in engine->req. + */ + if (!req) + req = engine->req; + else + req = mv_cesa_dequeue_req_locked(engine, + &backlog); + + /* Re-chaining to the next request */ + engine->chain.first = tdma->next; + tdma->next = NULL; + + /* If this is the last request, clear the chain */ + if (engine->chain.first == NULL) + engine->chain.last = NULL; + spin_unlock_bh(&engine->lock); + + ctx = crypto_tfm_ctx(req->tfm); + current_status = (tdma->cur_dma == tdma_cur) ? + status : CESA_SA_INT_ACC0_IDMA_DONE; + res = ctx->ops->process(req, current_status); + ctx->ops->complete(req); + + if (res == 0) + mv_cesa_engine_enqueue_complete_request(engine, + req); + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + } + + if (res || tdma->cur_dma == tdma_cur) + break; + } + + /* Save the last request in error to engine->req, so that the core + * knows which request was fautly */ + if (res) { + spin_lock_bh(&engine->lock); + engine->req = req; + spin_unlock_bh(&engine->lock); + } + + return res; +} + static struct mv_cesa_tdma_desc * mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) { @@ -117,6 +209,32 @@ mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) return new_tdma; } +int mv_cesa_dma_add_iv_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, + u32 size, u32 flags, gfp_t gfp_flags) +{ + + struct mv_cesa_tdma_desc *tdma; + u8 *iv; + dma_addr_t dma_handle; + + tdma = mv_cesa_dma_add_desc(chain, gfp_flags); + if (IS_ERR(tdma)) + return PTR_ERR(tdma); + + iv = dma_pool_alloc(cesa_dev->dma->iv_pool, gfp_flags, &dma_handle); + if (!iv) + return -ENOMEM; + + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); + tdma->src = src; + tdma->dst = cpu_to_le32(dma_handle); + tdma->data = iv; + + flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); + tdma->flags = flags | CESA_TDMA_IV; + return 0; +} + struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, const struct mv_cesa_op_ctx *op_templ, bool skip_ctx, diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 59ed54e464a9..625ee50fd78b 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -11,7 +11,6 @@ * http://www.gnu.org/copyleft/gpl.html */ -#include #include #include #include @@ -25,6 +24,7 @@ #include #include #include +#include #define DCP_MAX_CHANS 4 #define DCP_BUF_SZ PAGE_SIZE @@ -84,7 +84,7 @@ struct dcp_async_ctx { unsigned int hot:1; /* Crypto-specific context */ - struct crypto_ablkcipher *fallback; + struct crypto_skcipher *fallback; unsigned int key_len; uint8_t key[AES_KEYSIZE_128]; }; @@ -374,20 +374,22 @@ static int dcp_chan_thread_aes(void *data) static int mxs_dcp_block_fallback(struct ablkcipher_request *req, int enc) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); - struct dcp_async_ctx *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct dcp_async_ctx *ctx = crypto_ablkcipher_ctx(tfm); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); int ret; - ablkcipher_request_set_tfm(req, ctx->fallback); + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); if (enc) - ret = crypto_ablkcipher_encrypt(req); + ret = crypto_skcipher_encrypt(subreq); else - ret = crypto_ablkcipher_decrypt(req); + ret = crypto_skcipher_decrypt(subreq); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + skcipher_request_zero(subreq); return ret; } @@ -453,28 +455,22 @@ static int mxs_dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, return 0; } - /* Check if the key size is supported by kernel at all. */ - if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { - tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; - return -EINVAL; - } - /* * If the requested AES key size is not supported by the hardware, * but is supported by in-kernel software implementation, we use * software fallback. */ - actx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - actx->fallback->base.crt_flags |= - tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK; + crypto_skcipher_clear_flags(actx->fallback, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(actx->fallback, + tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); - ret = crypto_ablkcipher_setkey(actx->fallback, key, len); + ret = crypto_skcipher_setkey(actx->fallback, key, len); if (!ret) return 0; tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm->base.crt_flags |= - actx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK; + tfm->base.crt_flags |= crypto_skcipher_get_flags(actx->fallback) & + CRYPTO_TFM_RES_MASK; return ret; } @@ -484,9 +480,9 @@ static int mxs_dcp_aes_fallback_init(struct crypto_tfm *tfm) const char *name = crypto_tfm_alg_name(tfm); const uint32_t flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK; struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm); - struct crypto_ablkcipher *blk; + struct crypto_skcipher *blk; - blk = crypto_alloc_ablkcipher(name, 0, flags); + blk = crypto_alloc_skcipher(name, 0, flags); if (IS_ERR(blk)) return PTR_ERR(blk); @@ -499,8 +495,7 @@ static void mxs_dcp_aes_fallback_exit(struct crypto_tfm *tfm) { struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm); - crypto_free_ablkcipher(actx->fallback); - actx->fallback = NULL; + crypto_free_skcipher(actx->fallback); } /* diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 0794f1cc0018..42f0f229f7f7 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -392,7 +392,7 @@ static void nx_of_update_msc(struct device *dev, ((bytes_so_far + sizeof(struct msc_triplet)) <= lenp) && i < msc->triplets; i++) { - if (msc->fc > NX_MAX_FC || msc->mode > NX_MAX_MODE) { + if (msc->fc >= NX_MAX_FC || msc->mode >= NX_MAX_MODE) { dev_err(dev, "unknown function code/mode " "combo: %d/%d (ignored)\n", msc->fc, msc->mode); diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index ce174d3b842c..4ab53a604312 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -528,8 +528,6 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) omap_aes_dma_stop(dd); - dmaengine_terminate_all(dd->dma_lch_in); - dmaengine_terminate_all(dd->dma_lch_out); return 0; } @@ -580,10 +578,12 @@ static int omap_aes_copy_sgs(struct omap_aes_dev *dd) sg_init_table(&dd->in_sgl, 1); sg_set_buf(&dd->in_sgl, buf_in, total); dd->in_sg = &dd->in_sgl; + dd->in_sg_len = 1; sg_init_table(&dd->out_sgl, 1); sg_set_buf(&dd->out_sgl, buf_out, total); dd->out_sg = &dd->out_sgl; + dd->out_sg_len = 1; return 0; } @@ -604,7 +604,6 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, crypto_ablkcipher_reqtfm(req)); struct omap_aes_dev *dd = omap_aes_find_dev(ctx); struct omap_aes_reqctx *rctx; - int len; if (!dd) return -ENODEV; @@ -616,6 +615,14 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, dd->in_sg = req->src; dd->out_sg = req->dst; + dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total); + if (dd->in_sg_len < 0) + return dd->in_sg_len; + + dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total); + if (dd->out_sg_len < 0) + return dd->out_sg_len; + if (omap_aes_check_aligned(dd->in_sg, dd->total) || omap_aes_check_aligned(dd->out_sg, dd->total)) { if (omap_aes_copy_sgs(dd)) @@ -625,11 +632,6 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, dd->sgs_copied = 0; } - len = ALIGN(dd->total, AES_BLOCK_SIZE); - dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, len); - dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, len); - BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); - rctx = ablkcipher_request_ctx(req); ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); rctx->mode &= FLAGS_MODE_MASK; @@ -1185,17 +1187,19 @@ static int omap_aes_probe(struct platform_device *pdev) spin_unlock(&list_lock); for (i = 0; i < dd->pdata->algs_info_size; i++) { - for (j = 0; j < dd->pdata->algs_info[i].size; j++) { - algp = &dd->pdata->algs_info[i].algs_list[j]; + if (!dd->pdata->algs_info[i].registered) { + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { + algp = &dd->pdata->algs_info[i].algs_list[j]; - pr_debug("reg alg: %s\n", algp->cra_name); - INIT_LIST_HEAD(&algp->cra_list); + pr_debug("reg alg: %s\n", algp->cra_name); + INIT_LIST_HEAD(&algp->cra_list); - err = crypto_register_alg(algp); - if (err) - goto err_algs; + err = crypto_register_alg(algp); + if (err) + goto err_algs; - dd->pdata->algs_info[i].registered++; + dd->pdata->algs_info[i].registered++; + } } } diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 3eedb03111ba..5691434ffb2d 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -560,10 +560,12 @@ static int omap_des_copy_sgs(struct omap_des_dev *dd) sg_init_table(&dd->in_sgl, 1); sg_set_buf(&dd->in_sgl, buf_in, dd->total); dd->in_sg = &dd->in_sgl; + dd->in_sg_len = 1; sg_init_table(&dd->out_sgl, 1); sg_set_buf(&dd->out_sgl, buf_out, dd->total); dd->out_sg = &dd->out_sgl; + dd->out_sg_len = 1; return 0; } @@ -595,6 +597,14 @@ static int omap_des_prepare_req(struct crypto_engine *engine, dd->in_sg = req->src; dd->out_sg = req->dst; + dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total); + if (dd->in_sg_len < 0) + return dd->in_sg_len; + + dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total); + if (dd->out_sg_len < 0) + return dd->out_sg_len; + if (omap_des_copy_needed(dd->in_sg) || omap_des_copy_needed(dd->out_sg)) { if (omap_des_copy_sgs(dd)) @@ -604,10 +614,6 @@ static int omap_des_prepare_req(struct crypto_engine *engine, dd->sgs_copied = 0; } - dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total); - dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total); - BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); - rctx = ablkcipher_request_ctx(req); ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); rctx->mode &= FLAGS_MODE_MASK; diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 63464e86f2b1..7fe4eef12fe2 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -100,6 +100,8 @@ #define DEFAULT_TIMEOUT_INTERVAL HZ +#define DEFAULT_AUTOSUSPEND_DELAY 1000 + /* mostly device flags */ #define FLAGS_BUSY 0 #define FLAGS_FINAL 1 @@ -173,7 +175,7 @@ struct omap_sham_ctx { struct omap_sham_hmac_ctx base[0]; }; -#define OMAP_SHAM_QUEUE_LENGTH 1 +#define OMAP_SHAM_QUEUE_LENGTH 10 struct omap_sham_algs_info { struct ahash_alg *algs_list; @@ -813,7 +815,6 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) { struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); - dmaengine_terminate_all(dd->dma_lch); if (ctx->flags & BIT(FLAGS_SG)) { dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); @@ -999,7 +1000,8 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); - pm_runtime_put(dd->dev); + pm_runtime_mark_last_busy(dd->dev); + pm_runtime_put_autosuspend(dd->dev); if (req->base.complete) req->base.complete(&req->base, err); @@ -1093,7 +1095,7 @@ static int omap_sham_update(struct ahash_request *req) ctx->offset = 0; if (ctx->flags & BIT(FLAGS_FINUP)) { - if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) { + if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 240) { /* * OMAP HW accel works only with buffers >= 9 * will switch to bypass in final() @@ -1149,9 +1151,13 @@ static int omap_sham_final(struct ahash_request *req) if (ctx->flags & BIT(FLAGS_ERROR)) return 0; /* uncompleted hash is not needed */ - /* OMAP HW accel works only with buffers >= 9 */ - /* HMAC is always >= 9 because ipad == block size */ - if ((ctx->digcnt + ctx->bufcnt) < 9) + /* + * OMAP HW accel works only with buffers >= 9. + * HMAC is always >= 9 because ipad == block size. + * If buffersize is less than 240, we use fallback SW encoding, + * as using DMA + HW in this case doesn't provide any benefit. + */ + if ((ctx->digcnt + ctx->bufcnt) < 240) return omap_sham_final_shash(req); else if (ctx->bufcnt) return omap_sham_enqueue(req, OP_FINAL); @@ -1328,7 +1334,7 @@ static struct ahash_alg algs_sha1_md5[] = { .halg.base = { .cra_name = "sha1", .cra_driver_name = "omap-sha1", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | @@ -1351,7 +1357,7 @@ static struct ahash_alg algs_sha1_md5[] = { .halg.base = { .cra_name = "md5", .cra_driver_name = "omap-md5", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | @@ -1375,7 +1381,7 @@ static struct ahash_alg algs_sha1_md5[] = { .halg.base = { .cra_name = "hmac(sha1)", .cra_driver_name = "omap-hmac-sha1", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | @@ -1400,7 +1406,7 @@ static struct ahash_alg algs_sha1_md5[] = { .halg.base = { .cra_name = "hmac(md5)", .cra_driver_name = "omap-hmac-md5", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | @@ -1428,7 +1434,7 @@ static struct ahash_alg algs_sha224_sha256[] = { .halg.base = { .cra_name = "sha224", .cra_driver_name = "omap-sha224", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1450,7 +1456,7 @@ static struct ahash_alg algs_sha224_sha256[] = { .halg.base = { .cra_name = "sha256", .cra_driver_name = "omap-sha256", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1473,7 +1479,7 @@ static struct ahash_alg algs_sha224_sha256[] = { .halg.base = { .cra_name = "hmac(sha224)", .cra_driver_name = "omap-hmac-sha224", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1497,7 +1503,7 @@ static struct ahash_alg algs_sha224_sha256[] = { .halg.base = { .cra_name = "hmac(sha256)", .cra_driver_name = "omap-hmac-sha256", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1523,7 +1529,7 @@ static struct ahash_alg algs_sha384_sha512[] = { .halg.base = { .cra_name = "sha384", .cra_driver_name = "omap-sha384", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1545,7 +1551,7 @@ static struct ahash_alg algs_sha384_sha512[] = { .halg.base = { .cra_name = "sha512", .cra_driver_name = "omap-sha512", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1568,7 +1574,7 @@ static struct ahash_alg algs_sha384_sha512[] = { .halg.base = { .cra_name = "hmac(sha384)", .cra_driver_name = "omap-hmac-sha384", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1592,7 +1598,7 @@ static struct ahash_alg algs_sha384_sha512[] = { .halg.base = { .cra_name = "hmac(sha512)", .cra_driver_name = "omap-hmac-sha512", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1946,6 +1952,9 @@ static int omap_sham_probe(struct platform_device *pdev) dd->flags |= dd->pdata->flags; + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); + pm_runtime_enable(dev); pm_runtime_irq_safe(dev); diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 3b1c7ecf078f..47576098831f 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -171,7 +171,7 @@ struct spacc_ablk_ctx { * The fallback cipher. If the operation can't be done in hardware, * fallback to a software version. */ - struct crypto_ablkcipher *sw_cipher; + struct crypto_skcipher *sw_cipher; }; /* AEAD cipher context. */ @@ -789,33 +789,35 @@ static int spacc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, * request for any other size (192 bits) then we need to do a software * fallback. */ - if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256 && - ctx->sw_cipher) { + if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) { + if (!ctx->sw_cipher) + return -EINVAL; + /* * Set the fallback transform to use the same request flags as * the hardware transform. */ - ctx->sw_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - ctx->sw_cipher->base.crt_flags |= - cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK; + crypto_skcipher_clear_flags(ctx->sw_cipher, + CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->sw_cipher, + cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + err = crypto_skcipher_setkey(ctx->sw_cipher, key, len); + + tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->crt_flags |= + crypto_skcipher_get_flags(ctx->sw_cipher) & + CRYPTO_TFM_RES_MASK; - err = crypto_ablkcipher_setkey(ctx->sw_cipher, key, len); if (err) goto sw_setkey_failed; - } else if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256 && - !ctx->sw_cipher) - err = -EINVAL; + } memcpy(ctx->key, key, len); ctx->key_len = len; sw_setkey_failed: - if (err && ctx->sw_cipher) { - tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm->crt_flags |= - ctx->sw_cipher->base.crt_flags & CRYPTO_TFM_RES_MASK; - } - return err; } @@ -910,20 +912,21 @@ static int spacc_ablk_do_fallback(struct ablkcipher_request *req, struct crypto_tfm *old_tfm = crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher); int err; - if (!ctx->sw_cipher) - return -EINVAL; - /* * Change the request to use the software fallback transform, and once * the ciphering has completed, put the old transform back into the * request. */ - ablkcipher_request_set_tfm(req, ctx->sw_cipher); - err = is_encrypt ? crypto_ablkcipher_encrypt(req) : - crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(old_tfm)); + skcipher_request_set_tfm(subreq, ctx->sw_cipher); + skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = is_encrypt ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -1015,12 +1018,13 @@ static int spacc_ablk_cra_init(struct crypto_tfm *tfm) ctx->generic.flags = spacc_alg->type; ctx->generic.engine = engine; if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { - ctx->sw_cipher = crypto_alloc_ablkcipher(alg->cra_name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + ctx->sw_cipher = crypto_alloc_skcipher( + alg->cra_name, 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->sw_cipher)) { dev_warn(engine->dev, "failed to allocate fallback for %s\n", alg->cra_name); - ctx->sw_cipher = NULL; + return PTR_ERR(ctx->sw_cipher); } } ctx->generic.key_offs = spacc_alg->key_offs; @@ -1035,9 +1039,7 @@ static void spacc_ablk_cra_exit(struct crypto_tfm *tfm) { struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->sw_cipher) - crypto_free_ablkcipher(ctx->sw_cipher); - ctx->sw_cipher = NULL; + crypto_free_skcipher(ctx->sw_cipher); } static int spacc_ablk_encrypt(struct ablkcipher_request *req) diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 85b44e577684..ce3cae40f949 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -4,12 +4,13 @@ config CRYPTO_DEV_QAT select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER select CRYPTO_AKCIPHER + select CRYPTO_DH select CRYPTO_HMAC + select CRYPTO_RSA select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 select FW_LOADER - select ASN1 config CRYPTO_DEV_QAT_DH895xCC tristate "Support for Intel(R) DH895xCC" diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index c5bd5a9abc4d..6bc68bc00d76 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -229,6 +229,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 879e04cae714..618cec360b39 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -239,6 +239,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 6d74b91f2152..92fb6ffdc062 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -1,11 +1,3 @@ -$(obj)/qat_rsapubkey-asn1.o: $(obj)/qat_rsapubkey-asn1.c \ - $(obj)/qat_rsapubkey-asn1.h -$(obj)/qat_rsaprivkey-asn1.o: $(obj)/qat_rsaprivkey-asn1.c \ - $(obj)/qat_rsaprivkey-asn1.h - -clean-files += qat_rsapubkey-asn1.c qat_rsapubkey-asn1.h -clean-files += qat_rsaprivkey-asn1.c qat_rsaprivkey-asn1.h - obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o intel_qat-objs := adf_cfg.o \ adf_isr.o \ @@ -19,8 +11,6 @@ intel_qat-objs := adf_cfg.o \ adf_hw_arbiter.o \ qat_crypto.o \ qat_algs.o \ - qat_rsapubkey-asn1.o \ - qat_rsaprivkey-asn1.o \ qat_asym_algs.o \ qat_uclo.o \ qat_hal.o diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 5a07208ce778..e8822536530b 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -176,6 +176,7 @@ struct adf_hw_device_data { void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); + void (*reset_device)(struct adf_accel_dev *accel_dev); const char *fw_name; const char *fw_mmp_name; uint32_t fuses; diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index b40d9c8dad96..2839fccdd84b 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -82,18 +82,12 @@ struct adf_reset_dev_data { struct work_struct reset_work; }; -void adf_dev_restore(struct adf_accel_dev *accel_dev) +void adf_reset_sbr(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); struct pci_dev *parent = pdev->bus->self; uint16_t bridge_ctl = 0; - if (accel_dev->is_vf) - return; - - dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", - accel_dev->accel_id); - if (!parent) parent = pdev; @@ -101,6 +95,8 @@ void adf_dev_restore(struct adf_accel_dev *accel_dev) dev_info(&GET_DEV(accel_dev), "Transaction still in progress. Proceeding\n"); + dev_info(&GET_DEV(accel_dev), "Secondary bus reset\n"); + pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET; pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); @@ -108,8 +104,40 @@ void adf_dev_restore(struct adf_accel_dev *accel_dev) bridge_ctl &= ~PCI_BRIDGE_CTL_BUS_RESET; pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); msleep(100); - pci_restore_state(pdev); - pci_save_state(pdev); +} +EXPORT_SYMBOL_GPL(adf_reset_sbr); + +void adf_reset_flr(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + u16 control = 0; + int pos = 0; + + dev_info(&GET_DEV(accel_dev), "Function level reset\n"); + pos = pci_pcie_cap(pdev); + if (!pos) { + dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); + return; + } + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &control); + control |= PCI_EXP_DEVCTL_BCR_FLR; + pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, control); + msleep(100); +} +EXPORT_SYMBOL_GPL(adf_reset_flr); + +void adf_dev_restore(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + if (hw_device->reset_device) { + dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", + accel_dev->accel_id); + hw_device->reset_device(accel_dev); + pci_restore_state(pdev); + pci_save_state(pdev); + } } static void adf_device_reset_worker(struct work_struct *work) @@ -243,7 +271,8 @@ EXPORT_SYMBOL_GPL(adf_disable_aer); int adf_init_aer(void) { - device_reset_wq = create_workqueue("qat_device_reset_wq"); + device_reset_wq = alloc_workqueue("qat_device_reset_wq", + WQ_MEM_RECLAIM, 0); return !device_reset_wq ? -EFAULT : 0; } diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 75faa39bc8d0..980e07475012 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -141,6 +141,8 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev); int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf); void adf_disable_aer(struct adf_accel_dev *accel_dev); +void adf_reset_sbr(struct adf_accel_dev *accel_dev); +void adf_reset_flr(struct adf_accel_dev *accel_dev); void adf_dev_restore(struct adf_accel_dev *accel_dev); int adf_init_aer(void); void adf_exit_aer(void); diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 4a526e2f1d7f..9320ae1d005b 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -292,7 +292,7 @@ EXPORT_SYMBOL_GPL(adf_sriov_configure); int __init adf_init_pf_wq(void) { /* Workqueue for PF2VF responses */ - pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq"); + pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", WQ_MEM_RECLAIM, 0); return !pf2vf_resp_wq ? -ENOMEM : 0; } diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index aa689cabedb4..bf99e11a3403 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -321,7 +321,7 @@ EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc); int __init adf_init_vf_wq(void) { - adf_vf_stop_wq = create_workqueue("adf_vf_stop_wq"); + adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0); return !adf_vf_stop_wq ? -EFAULT : 0; } diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 1e8852a8a057..769148dbaeb3 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -947,13 +947,13 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, return 0; out_free_all: - memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd)); - dma_free_coherent(dev, sizeof(*ctx->enc_cd), + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + dma_free_coherent(dev, sizeof(*ctx->dec_cd), ctx->dec_cd, ctx->dec_cd_paddr); ctx->dec_cd = NULL; out_free_enc: - memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd)); - dma_free_coherent(dev, sizeof(*ctx->dec_cd), + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + dma_free_coherent(dev, sizeof(*ctx->enc_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; return -ENOMEM; diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 05f49d4f94b2..0d35dca2e925 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -49,11 +49,12 @@ #include #include #include +#include +#include +#include #include #include #include -#include "qat_rsapubkey-asn1.h" -#include "qat_rsaprivkey-asn1.h" #include "icp_qat_fw_pke.h" #include "adf_accel_devices.h" #include "adf_transport.h" @@ -75,6 +76,14 @@ struct qat_rsa_input_params { dma_addr_t d; dma_addr_t n; } dec; + struct { + dma_addr_t c; + dma_addr_t p; + dma_addr_t q; + dma_addr_t dp; + dma_addr_t dq; + dma_addr_t qinv; + } dec_crt; u64 in_tab[8]; }; } __packed __aligned(64); @@ -95,71 +104,480 @@ struct qat_rsa_ctx { char *n; char *e; char *d; + char *p; + char *q; + char *dp; + char *dq; + char *qinv; dma_addr_t dma_n; dma_addr_t dma_e; dma_addr_t dma_d; + dma_addr_t dma_p; + dma_addr_t dma_q; + dma_addr_t dma_dp; + dma_addr_t dma_dq; + dma_addr_t dma_qinv; unsigned int key_sz; + bool crt_mode; + struct qat_crypto_instance *inst; +} __packed __aligned(64); + +struct qat_dh_input_params { + union { + struct { + dma_addr_t b; + dma_addr_t xa; + dma_addr_t p; + } in; + struct { + dma_addr_t xa; + dma_addr_t p; + } in_g2; + u64 in_tab[8]; + }; +} __packed __aligned(64); + +struct qat_dh_output_params { + union { + dma_addr_t r; + u64 out_tab[8]; + }; +} __packed __aligned(64); + +struct qat_dh_ctx { + char *g; + char *xa; + char *p; + dma_addr_t dma_g; + dma_addr_t dma_xa; + dma_addr_t dma_p; + unsigned int p_size; + bool g2; struct qat_crypto_instance *inst; } __packed __aligned(64); -struct qat_rsa_request { - struct qat_rsa_input_params in; - struct qat_rsa_output_params out; +struct qat_asym_request { + union { + struct qat_rsa_input_params rsa; + struct qat_dh_input_params dh; + } in; + union { + struct qat_rsa_output_params rsa; + struct qat_dh_output_params dh; + } out; dma_addr_t phy_in; dma_addr_t phy_out; char *src_align; char *dst_align; struct icp_qat_fw_pke_request req; - struct qat_rsa_ctx *ctx; + union { + struct qat_rsa_ctx *rsa; + struct qat_dh_ctx *dh; + } ctx; + union { + struct akcipher_request *rsa; + struct kpp_request *dh; + } areq; int err; + void (*cb)(struct icp_qat_fw_pke_resp *resp); } __aligned(64); -static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) +static void qat_dh_cb(struct icp_qat_fw_pke_resp *resp) { - struct akcipher_request *areq = (void *)(__force long)resp->opaque; - struct qat_rsa_request *req = PTR_ALIGN(akcipher_request_ctx(areq), 64); - struct device *dev = &GET_DEV(req->ctx->inst->accel_dev); + struct qat_asym_request *req = (void *)(__force long)resp->opaque; + struct kpp_request *areq = req->areq.dh; + struct device *dev = &GET_DEV(req->ctx.dh->inst->accel_dev); int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( resp->pke_resp_hdr.comn_resp_flags); err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; - if (req->src_align) - dma_free_coherent(dev, req->ctx->key_sz, req->src_align, - req->in.enc.m); - else - dma_unmap_single(dev, req->in.enc.m, req->ctx->key_sz, - DMA_TO_DEVICE); + if (areq->src) { + if (req->src_align) + dma_free_coherent(dev, req->ctx.dh->p_size, + req->src_align, req->in.dh.in.b); + else + dma_unmap_single(dev, req->in.dh.in.b, + req->ctx.dh->p_size, DMA_TO_DEVICE); + } - areq->dst_len = req->ctx->key_sz; + areq->dst_len = req->ctx.dh->p_size; if (req->dst_align) { - char *ptr = req->dst_align; + scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, + areq->dst_len, 1); - while (!(*ptr) && areq->dst_len) { - areq->dst_len--; - ptr++; - } + dma_free_coherent(dev, req->ctx.dh->p_size, req->dst_align, + req->out.dh.r); + } else { + dma_unmap_single(dev, req->out.dh.r, req->ctx.dh->p_size, + DMA_FROM_DEVICE); + } - if (areq->dst_len != req->ctx->key_sz) - memmove(req->dst_align, ptr, areq->dst_len); + dma_unmap_single(dev, req->phy_in, sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); + dma_unmap_single(dev, req->phy_out, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); - scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, - areq->dst_len, 1); + kpp_request_complete(areq, err); +} + +#define PKE_DH_1536 0x390c1a49 +#define PKE_DH_G2_1536 0x2e0b1a3e +#define PKE_DH_2048 0x4d0c1a60 +#define PKE_DH_G2_2048 0x3e0b1a55 +#define PKE_DH_3072 0x510c1a77 +#define PKE_DH_G2_3072 0x3a0b1a6c +#define PKE_DH_4096 0x690c1a8e +#define PKE_DH_G2_4096 0x4a0b1a83 + +static unsigned long qat_dh_fn_id(unsigned int len, bool g2) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 1536: + return g2 ? PKE_DH_G2_1536 : PKE_DH_1536; + case 2048: + return g2 ? PKE_DH_G2_2048 : PKE_DH_2048; + case 3072: + return g2 ? PKE_DH_G2_3072 : PKE_DH_3072; + case 4096: + return g2 ? PKE_DH_G2_4096 : PKE_DH_4096; + default: + return 0; + }; +} + +static inline struct qat_dh_ctx *qat_dh_get_params(struct crypto_kpp *tfm) +{ + return kpp_tfm_ctx(tfm); +} + +static int qat_dh_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_asym_request *qat_req = + PTR_ALIGN(kpp_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + int ret, ctr = 0; + int n_input_params = 0; + + if (unlikely(!ctx->xa)) + return -EINVAL; + + if (req->dst_len < ctx->p_size) { + req->dst_len = ctx->p_size; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + + msg->pke_hdr.cd_pars.func_id = qat_dh_fn_id(ctx->p_size, + !req->src && ctx->g2); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->cb = qat_dh_cb; + qat_req->ctx.dh = ctx; + qat_req->areq.dh = req; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, + QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - dma_free_coherent(dev, req->ctx->key_sz, req->dst_align, - req->out.enc.c); + /* + * If no source is provided use g as base + */ + if (req->src) { + qat_req->in.dh.in.xa = ctx->dma_xa; + qat_req->in.dh.in.p = ctx->dma_p; + n_input_params = 3; } else { - char *ptr = sg_virt(areq->dst); + if (ctx->g2) { + qat_req->in.dh.in_g2.xa = ctx->dma_xa; + qat_req->in.dh.in_g2.p = ctx->dma_p; + n_input_params = 2; + } else { + qat_req->in.dh.in.b = ctx->dma_g; + qat_req->in.dh.in.xa = ctx->dma_xa; + qat_req->in.dh.in.p = ctx->dma_p; + n_input_params = 3; + } + } - while (!(*ptr) && areq->dst_len) { - areq->dst_len--; - ptr++; + ret = -ENOMEM; + if (req->src) { + /* + * src can be of any size in valid range, but HW expects it to + * be the same as modulo p so in case it is different we need + * to allocate a new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->src) && req->src_len == ctx->p_size) { + qat_req->src_align = NULL; + qat_req->in.dh.in.b = dma_map_single(dev, + sg_virt(req->src), + req->src_len, + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, + qat_req->in.dh.in.b))) + return ret; + + } else { + int shift = ctx->p_size - req->src_len; + + qat_req->src_align = dma_zalloc_coherent(dev, + ctx->p_size, + &qat_req->in.dh.in.b, + GFP_KERNEL); + if (unlikely(!qat_req->src_align)) + return ret; + + scatterwalk_map_and_copy(qat_req->src_align + shift, + req->src, 0, req->src_len, 0); } + } + /* + * dst can be of any size in valid range, but HW expects it to be the + * same as modulo m so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->dst) && req->dst_len == ctx->p_size) { + qat_req->dst_align = NULL; + qat_req->out.dh.r = dma_map_single(dev, sg_virt(req->dst), + req->dst_len, + DMA_FROM_DEVICE); - if (sg_virt(areq->dst) != ptr && areq->dst_len) - memmove(sg_virt(areq->dst), ptr, areq->dst_len); + if (unlikely(dma_mapping_error(dev, qat_req->out.dh.r))) + goto unmap_src; + + } else { + qat_req->dst_align = dma_zalloc_coherent(dev, ctx->p_size, + &qat_req->out.dh.r, + GFP_KERNEL); + if (unlikely(!qat_req->dst_align)) + goto unmap_src; + } - dma_unmap_single(dev, req->out.enc.c, req->ctx->key_sz, + qat_req->in.dh.in_tab[n_input_params] = 0; + qat_req->out.dh.out_tab[1] = 0; + /* Mapping in.in.b or in.in_g2.xa is the same */ + qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh.in.b, + sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) + goto unmap_dst; + + qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh.r, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) + goto unmap_in_params; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (uint64_t)(__force long)qat_req; + msg->input_param_count = n_input_params; + msg->output_param_count = 1; + + do { + ret = adf_send_message(ctx->inst->pke_tx, (uint32_t *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; + + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); +unmap_dst: + if (qat_req->dst_align) + dma_free_coherent(dev, ctx->p_size, qat_req->dst_align, + qat_req->out.dh.r); + else + if (!dma_mapping_error(dev, qat_req->out.dh.r)) + dma_unmap_single(dev, qat_req->out.dh.r, ctx->p_size, + DMA_FROM_DEVICE); +unmap_src: + if (req->src) { + if (qat_req->src_align) + dma_free_coherent(dev, ctx->p_size, qat_req->src_align, + qat_req->in.dh.in.b); + else + if (!dma_mapping_error(dev, qat_req->in.dh.in.b)) + dma_unmap_single(dev, qat_req->in.dh.in.b, + ctx->p_size, + DMA_TO_DEVICE); + } + return ret; +} + +static int qat_dh_check_params_length(unsigned int p_len) +{ + switch (p_len) { + case 1536: + case 2048: + case 3072: + case 4096: + return 0; + } + return -EINVAL; +} + +static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + + if (unlikely(!params->p || !params->g)) + return -EINVAL; + + if (qat_dh_check_params_length(params->p_size << 3)) + return -EINVAL; + + ctx->p_size = params->p_size; + ctx->p = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_p, GFP_KERNEL); + if (!ctx->p) + return -ENOMEM; + memcpy(ctx->p, params->p, ctx->p_size); + + /* If g equals 2 don't copy it */ + if (params->g_size == 1 && *(char *)params->g == 0x02) { + ctx->g2 = true; + return 0; + } + + ctx->g = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_g, GFP_KERNEL); + if (!ctx->g) { + dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); + ctx->p = NULL; + return -ENOMEM; + } + memcpy(ctx->g + (ctx->p_size - params->g_size), params->g, + params->g_size); + + return 0; +} + +static void qat_dh_clear_ctx(struct device *dev, struct qat_dh_ctx *ctx) +{ + if (ctx->g) { + dma_free_coherent(dev, ctx->p_size, ctx->g, ctx->dma_g); + ctx->g = NULL; + } + if (ctx->xa) { + dma_free_coherent(dev, ctx->p_size, ctx->xa, ctx->dma_xa); + ctx->xa = NULL; + } + if (ctx->p) { + dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); + ctx->p = NULL; + } + ctx->p_size = 0; + ctx->g2 = false; +} + +static int qat_dh_set_secret(struct crypto_kpp *tfm, void *buf, + unsigned int len) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + struct dh params; + int ret; + + if (crypto_dh_decode_key(buf, len, ¶ms) < 0) + return -EINVAL; + + /* Free old secret if any */ + qat_dh_clear_ctx(dev, ctx); + + ret = qat_dh_set_params(ctx, ¶ms); + if (ret < 0) + return ret; + + ctx->xa = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_xa, + GFP_KERNEL); + if (!ctx->xa) { + qat_dh_clear_ctx(dev, ctx); + return -ENOMEM; + } + memcpy(ctx->xa + (ctx->p_size - params.key_size), params.key, + params.key_size); + + return 0; +} + +static int qat_dh_max_size(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + + return ctx->p ? ctx->p_size : -EINVAL; +} + +static int qat_dh_init_tfm(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(get_current_node()); + + if (!inst) + return -EINVAL; + + ctx->p_size = 0; + ctx->g2 = false; + ctx->inst = inst; + return 0; +} + +static void qat_dh_exit_tfm(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + qat_dh_clear_ctx(dev, ctx); + qat_crypto_put_instance(ctx->inst); +} + +static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) +{ + struct qat_asym_request *req = (void *)(__force long)resp->opaque; + struct akcipher_request *areq = req->areq.rsa; + struct device *dev = &GET_DEV(req->ctx.rsa->inst->accel_dev); + int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( + resp->pke_resp_hdr.comn_resp_flags); + + err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; + + if (req->src_align) + dma_free_coherent(dev, req->ctx.rsa->key_sz, req->src_align, + req->in.rsa.enc.m); + else + dma_unmap_single(dev, req->in.rsa.enc.m, req->ctx.rsa->key_sz, + DMA_TO_DEVICE); + + areq->dst_len = req->ctx.rsa->key_sz; + if (req->dst_align) { + scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, + areq->dst_len, 1); + + dma_free_coherent(dev, req->ctx.rsa->key_sz, req->dst_align, + req->out.rsa.enc.c); + } else { + dma_unmap_single(dev, req->out.rsa.enc.c, req->ctx.rsa->key_sz, DMA_FROM_DEVICE); } @@ -175,8 +593,9 @@ static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) void qat_alg_asym_callback(void *_resp) { struct icp_qat_fw_pke_resp *resp = _resp; + struct qat_asym_request *areq = (void *)(__force long)resp->opaque; - qat_rsa_cb(resp); + areq->cb(resp); } #define PKE_RSA_EP_512 0x1c161b21 @@ -237,13 +656,42 @@ static unsigned long qat_rsa_dec_fn_id(unsigned int len) }; } +#define PKE_RSA_DP2_512 0x1c131b57 +#define PKE_RSA_DP2_1024 0x26131c2d +#define PKE_RSA_DP2_1536 0x45111d12 +#define PKE_RSA_DP2_2048 0x59121dfa +#define PKE_RSA_DP2_3072 0x81121ed9 +#define PKE_RSA_DP2_4096 0xb1111fb2 + +static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_DP2_512; + case 1024: + return PKE_RSA_DP2_1024; + case 1536: + return PKE_RSA_DP2_1536; + case 2048: + return PKE_RSA_DP2_2048; + case 3072: + return PKE_RSA_DP2_3072; + case 4096: + return PKE_RSA_DP2_4096; + default: + return 0; + }; +} + static int qat_rsa_enc(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); - struct qat_rsa_request *qat_req = + struct qat_asym_request *qat_req = PTR_ALIGN(akcipher_request_ctx(req), 64); struct icp_qat_fw_pke_request *msg = &qat_req->req; int ret, ctr = 0; @@ -262,14 +710,16 @@ static int qat_rsa_enc(struct akcipher_request *req) if (unlikely(!msg->pke_hdr.cd_pars.func_id)) return -EINVAL; - qat_req->ctx = ctx; + qat_req->cb = qat_rsa_cb; + qat_req->ctx.rsa = ctx; + qat_req->areq.rsa = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - qat_req->in.enc.e = ctx->dma_e; - qat_req->in.enc.n = ctx->dma_n; + qat_req->in.rsa.enc.e = ctx->dma_e; + qat_req->in.rsa.enc.n = ctx->dma_n; ret = -ENOMEM; /* @@ -281,16 +731,16 @@ static int qat_rsa_enc(struct akcipher_request *req) */ if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { qat_req->src_align = NULL; - qat_req->in.enc.m = dma_map_single(dev, sg_virt(req->src), + qat_req->in.rsa.enc.m = dma_map_single(dev, sg_virt(req->src), req->src_len, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->in.enc.m))) + if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.enc.m))) return ret; } else { int shift = ctx->key_sz - req->src_len; qat_req->src_align = dma_zalloc_coherent(dev, ctx->key_sz, - &qat_req->in.enc.m, + &qat_req->in.rsa.enc.m, GFP_KERNEL); if (unlikely(!qat_req->src_align)) return ret; @@ -300,30 +750,30 @@ static int qat_rsa_enc(struct akcipher_request *req) } if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { qat_req->dst_align = NULL; - qat_req->out.enc.c = dma_map_single(dev, sg_virt(req->dst), - req->dst_len, - DMA_FROM_DEVICE); + qat_req->out.rsa.enc.c = dma_map_single(dev, sg_virt(req->dst), + req->dst_len, + DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->out.enc.c))) + if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.enc.c))) goto unmap_src; } else { qat_req->dst_align = dma_zalloc_coherent(dev, ctx->key_sz, - &qat_req->out.enc.c, + &qat_req->out.rsa.enc.c, GFP_KERNEL); if (unlikely(!qat_req->dst_align)) goto unmap_src; } - qat_req->in.in_tab[3] = 0; - qat_req->out.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.enc.m, + qat_req->in.rsa.in_tab[3] = 0; + qat_req->out.rsa.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.enc.m, sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.enc.c, + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.enc.c, sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) @@ -331,7 +781,7 @@ static int qat_rsa_enc(struct akcipher_request *req) msg->pke_mid.src_data_addr = qat_req->phy_in; msg->pke_mid.dest_data_addr = qat_req->phy_out; - msg->pke_mid.opaque = (uint64_t)(__force long)req; + msg->pke_mid.opaque = (uint64_t)(__force long)qat_req; msg->input_param_count = 3; msg->output_param_count = 1; do { @@ -353,19 +803,19 @@ unmap_in_params: unmap_dst: if (qat_req->dst_align) dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align, - qat_req->out.enc.c); + qat_req->out.rsa.enc.c); else - if (!dma_mapping_error(dev, qat_req->out.enc.c)) - dma_unmap_single(dev, qat_req->out.enc.c, ctx->key_sz, - DMA_FROM_DEVICE); + if (!dma_mapping_error(dev, qat_req->out.rsa.enc.c)) + dma_unmap_single(dev, qat_req->out.rsa.enc.c, + ctx->key_sz, DMA_FROM_DEVICE); unmap_src: if (qat_req->src_align) dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, - qat_req->in.enc.m); + qat_req->in.rsa.enc.m); else - if (!dma_mapping_error(dev, qat_req->in.enc.m)) - dma_unmap_single(dev, qat_req->in.enc.m, ctx->key_sz, - DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->in.rsa.enc.m)) + dma_unmap_single(dev, qat_req->in.rsa.enc.m, + ctx->key_sz, DMA_TO_DEVICE); return ret; } @@ -375,7 +825,7 @@ static int qat_rsa_dec(struct akcipher_request *req) struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); - struct qat_rsa_request *qat_req = + struct qat_asym_request *qat_req = PTR_ALIGN(akcipher_request_ctx(req), 64); struct icp_qat_fw_pke_request *msg = &qat_req->req; int ret, ctr = 0; @@ -390,18 +840,30 @@ static int qat_rsa_dec(struct akcipher_request *req) memset(msg, '\0', sizeof(*msg)); ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, ICP_QAT_FW_COMN_REQ_FLAG_SET); - msg->pke_hdr.cd_pars.func_id = qat_rsa_dec_fn_id(ctx->key_sz); + msg->pke_hdr.cd_pars.func_id = ctx->crt_mode ? + qat_rsa_dec_fn_id_crt(ctx->key_sz) : + qat_rsa_dec_fn_id(ctx->key_sz); if (unlikely(!msg->pke_hdr.cd_pars.func_id)) return -EINVAL; - qat_req->ctx = ctx; + qat_req->cb = qat_rsa_cb; + qat_req->ctx.rsa = ctx; + qat_req->areq.rsa = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - qat_req->in.dec.d = ctx->dma_d; - qat_req->in.dec.n = ctx->dma_n; + if (ctx->crt_mode) { + qat_req->in.rsa.dec_crt.p = ctx->dma_p; + qat_req->in.rsa.dec_crt.q = ctx->dma_q; + qat_req->in.rsa.dec_crt.dp = ctx->dma_dp; + qat_req->in.rsa.dec_crt.dq = ctx->dma_dq; + qat_req->in.rsa.dec_crt.qinv = ctx->dma_qinv; + } else { + qat_req->in.rsa.dec.d = ctx->dma_d; + qat_req->in.rsa.dec.n = ctx->dma_n; + } ret = -ENOMEM; /* @@ -413,16 +875,16 @@ static int qat_rsa_dec(struct akcipher_request *req) */ if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { qat_req->src_align = NULL; - qat_req->in.dec.c = dma_map_single(dev, sg_virt(req->src), + qat_req->in.rsa.dec.c = dma_map_single(dev, sg_virt(req->src), req->dst_len, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->in.dec.c))) + if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.dec.c))) return ret; } else { int shift = ctx->key_sz - req->src_len; qat_req->src_align = dma_zalloc_coherent(dev, ctx->key_sz, - &qat_req->in.dec.c, + &qat_req->in.rsa.dec.c, GFP_KERNEL); if (unlikely(!qat_req->src_align)) return ret; @@ -432,31 +894,34 @@ static int qat_rsa_dec(struct akcipher_request *req) } if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { qat_req->dst_align = NULL; - qat_req->out.dec.m = dma_map_single(dev, sg_virt(req->dst), + qat_req->out.rsa.dec.m = dma_map_single(dev, sg_virt(req->dst), req->dst_len, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->out.dec.m))) + if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.dec.m))) goto unmap_src; } else { qat_req->dst_align = dma_zalloc_coherent(dev, ctx->key_sz, - &qat_req->out.dec.m, + &qat_req->out.rsa.dec.m, GFP_KERNEL); if (unlikely(!qat_req->dst_align)) goto unmap_src; } - qat_req->in.in_tab[3] = 0; - qat_req->out.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.dec.c, + if (ctx->crt_mode) + qat_req->in.rsa.in_tab[6] = 0; + else + qat_req->in.rsa.in_tab[3] = 0; + qat_req->out.rsa.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.dec.c, sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.dec.m, + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.dec.m, sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) @@ -464,8 +929,12 @@ static int qat_rsa_dec(struct akcipher_request *req) msg->pke_mid.src_data_addr = qat_req->phy_in; msg->pke_mid.dest_data_addr = qat_req->phy_out; - msg->pke_mid.opaque = (uint64_t)(__force long)req; - msg->input_param_count = 3; + msg->pke_mid.opaque = (uint64_t)(__force long)qat_req; + if (ctx->crt_mode) + msg->input_param_count = 6; + else + msg->input_param_count = 3; + msg->output_param_count = 1; do { ret = adf_send_message(ctx->inst->pke_tx, (uint32_t *)msg); @@ -486,26 +955,24 @@ unmap_in_params: unmap_dst: if (qat_req->dst_align) dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align, - qat_req->out.dec.m); + qat_req->out.rsa.dec.m); else - if (!dma_mapping_error(dev, qat_req->out.dec.m)) - dma_unmap_single(dev, qat_req->out.dec.m, ctx->key_sz, - DMA_FROM_DEVICE); + if (!dma_mapping_error(dev, qat_req->out.rsa.dec.m)) + dma_unmap_single(dev, qat_req->out.rsa.dec.m, + ctx->key_sz, DMA_FROM_DEVICE); unmap_src: if (qat_req->src_align) dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, - qat_req->in.dec.c); + qat_req->in.rsa.dec.c); else - if (!dma_mapping_error(dev, qat_req->in.dec.c)) - dma_unmap_single(dev, qat_req->in.dec.c, ctx->key_sz, - DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->in.rsa.dec.c)) + dma_unmap_single(dev, qat_req->in.rsa.dec.c, + ctx->key_sz, DMA_TO_DEVICE); return ret; } -int qat_rsa_get_n(void *context, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen) +int qat_rsa_set_n(struct qat_rsa_ctx *ctx, const char *value, size_t vlen) { - struct qat_rsa_ctx *ctx = context; struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); const char *ptr = value; @@ -518,11 +985,6 @@ int qat_rsa_get_n(void *context, size_t hdrlen, unsigned char tag, ctx->key_sz = vlen; ret = -EINVAL; - /* In FIPS mode only allow key size 2K & 3K */ - if (fips_enabled && (ctx->key_sz != 256 && ctx->key_sz != 384)) { - pr_err("QAT: RSA: key size not allowed in FIPS mode\n"); - goto err; - } /* invalid key size provided */ if (!qat_rsa_enc_fn_id(ctx->key_sz)) goto err; @@ -540,10 +1002,8 @@ err: return ret; } -int qat_rsa_get_e(void *context, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen) +int qat_rsa_set_e(struct qat_rsa_ctx *ctx, const char *value, size_t vlen) { - struct qat_rsa_ctx *ctx = context; struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); const char *ptr = value; @@ -559,18 +1019,15 @@ int qat_rsa_get_e(void *context, size_t hdrlen, unsigned char tag, } ctx->e = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_e, GFP_KERNEL); - if (!ctx->e) { - ctx->e = NULL; + if (!ctx->e) return -ENOMEM; - } + memcpy(ctx->e + (ctx->key_sz - vlen), ptr, vlen); return 0; } -int qat_rsa_get_d(void *context, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen) +int qat_rsa_set_d(struct qat_rsa_ctx *ctx, const char *value, size_t vlen) { - struct qat_rsa_ctx *ctx = context; struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); const char *ptr = value; @@ -585,12 +1042,6 @@ int qat_rsa_get_d(void *context, size_t hdrlen, unsigned char tag, if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) goto err; - /* In FIPS mode only allow key size 2K & 3K */ - if (fips_enabled && (vlen != 256 && vlen != 384)) { - pr_err("QAT: RSA: key size not allowed in FIPS mode\n"); - goto err; - } - ret = -ENOMEM; ctx->d = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_d, GFP_KERNEL); if (!ctx->d) @@ -603,12 +1054,106 @@ err: return ret; } -static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, - unsigned int keylen, bool private) +static void qat_rsa_drop_leading_zeros(const char **ptr, unsigned int *len) { - struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct device *dev = &GET_DEV(ctx->inst->accel_dev); - int ret; + while (!**ptr && *len) { + (*ptr)++; + (*len)--; + } +} + +static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr; + unsigned int len; + unsigned int half_key_sz = ctx->key_sz / 2; + + /* p */ + ptr = rsa_key->p; + len = rsa_key->p_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto err; + ctx->p = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_p, GFP_KERNEL); + if (!ctx->p) + goto err; + memcpy(ctx->p + (half_key_sz - len), ptr, len); + + /* q */ + ptr = rsa_key->q; + len = rsa_key->q_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_p; + ctx->q = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_q, GFP_KERNEL); + if (!ctx->q) + goto free_p; + memcpy(ctx->q + (half_key_sz - len), ptr, len); + + /* dp */ + ptr = rsa_key->dp; + len = rsa_key->dp_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_q; + ctx->dp = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_dp, + GFP_KERNEL); + if (!ctx->dp) + goto free_q; + memcpy(ctx->dp + (half_key_sz - len), ptr, len); + + /* dq */ + ptr = rsa_key->dq; + len = rsa_key->dq_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_dp; + ctx->dq = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_dq, + GFP_KERNEL); + if (!ctx->dq) + goto free_dp; + memcpy(ctx->dq + (half_key_sz - len), ptr, len); + + /* qinv */ + ptr = rsa_key->qinv; + len = rsa_key->qinv_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_dq; + ctx->qinv = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_qinv, + GFP_KERNEL); + if (!ctx->qinv) + goto free_dq; + memcpy(ctx->qinv + (half_key_sz - len), ptr, len); + + ctx->crt_mode = true; + return; + +free_dq: + memset(ctx->dq, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); + ctx->dq = NULL; +free_dp: + memset(ctx->dp, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); + ctx->dp = NULL; +free_q: + memset(ctx->q, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); + ctx->q = NULL; +free_p: + memset(ctx->p, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); + ctx->p = NULL; +err: + ctx->crt_mode = false; +} + +static void qat_rsa_clear_ctx(struct device *dev, struct qat_rsa_ctx *ctx) +{ + unsigned int half_key_sz = ctx->key_sz / 2; /* Free the old key if any */ if (ctx->n) @@ -619,19 +1164,68 @@ static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, memset(ctx->d, '\0', ctx->key_sz); dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); } + if (ctx->p) { + memset(ctx->p, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); + } + if (ctx->q) { + memset(ctx->q, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); + } + if (ctx->dp) { + memset(ctx->dp, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); + } + if (ctx->dq) { + memset(ctx->dq, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); + } + if (ctx->qinv) { + memset(ctx->qinv, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->qinv, ctx->dma_qinv); + } ctx->n = NULL; ctx->e = NULL; ctx->d = NULL; + ctx->p = NULL; + ctx->q = NULL; + ctx->dp = NULL; + ctx->dq = NULL; + ctx->qinv = NULL; + ctx->crt_mode = false; + ctx->key_sz = 0; +} + +static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + struct rsa_key rsa_key; + int ret; + + qat_rsa_clear_ctx(dev, ctx); if (private) - ret = asn1_ber_decoder(&qat_rsaprivkey_decoder, ctx, key, - keylen); + ret = rsa_parse_priv_key(&rsa_key, key, keylen); else - ret = asn1_ber_decoder(&qat_rsapubkey_decoder, ctx, key, - keylen); + ret = rsa_parse_pub_key(&rsa_key, key, keylen); + if (ret < 0) + goto free; + + ret = qat_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz); if (ret < 0) goto free; + ret = qat_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); + if (ret < 0) + goto free; + if (private) { + ret = qat_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); + if (ret < 0) + goto free; + qat_rsa_setkey_crt(ctx, &rsa_key); + } if (!ctx->n || !ctx->e) { /* invalid key provided */ @@ -646,20 +1240,7 @@ static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, return 0; free: - if (ctx->d) { - memset(ctx->d, '\0', ctx->key_sz); - dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); - ctx->d = NULL; - } - if (ctx->e) { - dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); - ctx->e = NULL; - } - if (ctx->n) { - dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); - ctx->n = NULL; - ctx->key_sz = 0; - } + qat_rsa_clear_ctx(dev, ctx); return ret; } @@ -725,7 +1306,7 @@ static struct akcipher_alg rsa = { .max_size = qat_rsa_max_size, .init = qat_rsa_init_tfm, .exit = qat_rsa_exit_tfm, - .reqsize = sizeof(struct qat_rsa_request) + 64, + .reqsize = sizeof(struct qat_asym_request) + 64, .base = { .cra_name = "rsa", .cra_driver_name = "qat-rsa", @@ -735,6 +1316,23 @@ static struct akcipher_alg rsa = { }, }; +static struct kpp_alg dh = { + .set_secret = qat_dh_set_secret, + .generate_public_key = qat_dh_compute_value, + .compute_shared_secret = qat_dh_compute_value, + .max_size = qat_dh_max_size, + .init = qat_dh_init_tfm, + .exit = qat_dh_exit_tfm, + .reqsize = sizeof(struct qat_asym_request) + 64, + .base = { + .cra_name = "dh", + .cra_driver_name = "qat-dh", + .cra_priority = 1000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct qat_dh_ctx), + }, +}; + int qat_asym_algs_register(void) { int ret = 0; @@ -743,7 +1341,11 @@ int qat_asym_algs_register(void) if (++active_devs == 1) { rsa.base.cra_flags = 0; ret = crypto_register_akcipher(&rsa); + if (ret) + goto unlock; + ret = crypto_register_kpp(&dh); } +unlock: mutex_unlock(&algs_lock); return ret; } @@ -751,7 +1353,9 @@ int qat_asym_algs_register(void) void qat_asym_algs_unregister(void) { mutex_lock(&algs_lock); - if (--active_devs == 0) + if (--active_devs == 0) { crypto_unregister_akcipher(&rsa); + crypto_unregister_kpp(&dh); + } mutex_unlock(&algs_lock); } diff --git a/drivers/crypto/qat/qat_common/qat_rsaprivkey.asn1 b/drivers/crypto/qat/qat_common/qat_rsaprivkey.asn1 deleted file mode 100644 index f0066adb79b8..000000000000 --- a/drivers/crypto/qat/qat_common/qat_rsaprivkey.asn1 +++ /dev/null @@ -1,11 +0,0 @@ -RsaPrivKey ::= SEQUENCE { - version INTEGER, - n INTEGER ({ qat_rsa_get_n }), - e INTEGER ({ qat_rsa_get_e }), - d INTEGER ({ qat_rsa_get_d }), - prime1 INTEGER, - prime2 INTEGER, - exponent1 INTEGER, - exponent2 INTEGER, - coefficient INTEGER -} diff --git a/drivers/crypto/qat/qat_common/qat_rsapubkey.asn1 b/drivers/crypto/qat/qat_common/qat_rsapubkey.asn1 deleted file mode 100644 index bd667b31a21a..000000000000 --- a/drivers/crypto/qat/qat_common/qat_rsapubkey.asn1 +++ /dev/null @@ -1,4 +0,0 @@ -RsaPubKey ::= SEQUENCE { - n INTEGER ({ qat_rsa_get_n }), - e INTEGER ({ qat_rsa_get_e }) -} diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 6e1d5e185526..1dfcab317bed 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -252,6 +252,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_sbr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c index dbcbbe242bd6..b04b42f48366 100644 --- a/drivers/crypto/qce/ablkcipher.c +++ b/drivers/crypto/qce/ablkcipher.c @@ -15,8 +15,8 @@ #include #include #include -#include #include +#include #include "cipher.h" @@ -189,7 +189,7 @@ static int qce_ablkcipher_setkey(struct crypto_ablkcipher *ablk, const u8 *key, memcpy(ctx->enc_key, key, keylen); return 0; fallback: - ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen); + ret = crypto_skcipher_setkey(ctx->fallback, key, keylen); if (!ret) ctx->enc_keylen = keylen; return ret; @@ -212,10 +212,16 @@ static int qce_ablkcipher_crypt(struct ablkcipher_request *req, int encrypt) if (IS_AES(rctx->flags) && ctx->enc_keylen != AES_KEYSIZE_128 && ctx->enc_keylen != AES_KEYSIZE_256) { - ablkcipher_request_set_tfm(req, ctx->fallback); - ret = encrypt ? crypto_ablkcipher_encrypt(req) : - crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + ret = encrypt ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return ret; } @@ -239,10 +245,9 @@ static int qce_ablkcipher_init(struct crypto_tfm *tfm) memset(ctx, 0, sizeof(*ctx)); tfm->crt_ablkcipher.reqsize = sizeof(struct qce_cipher_reqctx); - ctx->fallback = crypto_alloc_ablkcipher(crypto_tfm_alg_name(tfm), - CRYPTO_ALG_TYPE_ABLKCIPHER, - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK); + ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->fallback)) return PTR_ERR(ctx->fallback); @@ -253,7 +258,7 @@ static void qce_ablkcipher_exit(struct crypto_tfm *tfm) { struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_ablkcipher(ctx->fallback); + crypto_free_skcipher(ctx->fallback); } struct qce_ablkcipher_def { diff --git a/drivers/crypto/qce/cipher.h b/drivers/crypto/qce/cipher.h index 5c6a5f8633e5..2b0278bb6e92 100644 --- a/drivers/crypto/qce/cipher.h +++ b/drivers/crypto/qce/cipher.h @@ -22,7 +22,7 @@ struct qce_cipher_ctx { u8 enc_key[QCE_MAX_KEY_SIZE]; unsigned int enc_keylen; - struct crypto_ablkcipher *fallback; + struct crypto_skcipher *fallback; }; /** diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 2b3a0cfe3331..dce1af0ce85c 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -155,43 +155,43 @@ * expansion of its usage. */ struct samsung_aes_variant { - unsigned int aes_offset; + unsigned int aes_offset; }; struct s5p_aes_reqctx { - unsigned long mode; + unsigned long mode; }; struct s5p_aes_ctx { - struct s5p_aes_dev *dev; + struct s5p_aes_dev *dev; - uint8_t aes_key[AES_MAX_KEY_SIZE]; - uint8_t nonce[CTR_RFC3686_NONCE_SIZE]; - int keylen; + uint8_t aes_key[AES_MAX_KEY_SIZE]; + uint8_t nonce[CTR_RFC3686_NONCE_SIZE]; + int keylen; }; struct s5p_aes_dev { - struct device *dev; - struct clk *clk; - void __iomem *ioaddr; - void __iomem *aes_ioaddr; - int irq_fc; + struct device *dev; + struct clk *clk; + void __iomem *ioaddr; + void __iomem *aes_ioaddr; + int irq_fc; - struct ablkcipher_request *req; - struct s5p_aes_ctx *ctx; - struct scatterlist *sg_src; - struct scatterlist *sg_dst; + struct ablkcipher_request *req; + struct s5p_aes_ctx *ctx; + struct scatterlist *sg_src; + struct scatterlist *sg_dst; /* In case of unaligned access: */ - struct scatterlist *sg_src_cpy; - struct scatterlist *sg_dst_cpy; + struct scatterlist *sg_src_cpy; + struct scatterlist *sg_dst_cpy; - struct tasklet_struct tasklet; - struct crypto_queue queue; - bool busy; - spinlock_t lock; + struct tasklet_struct tasklet; + struct crypto_queue queue; + bool busy; + spinlock_t lock; - struct samsung_aes_variant *variant; + struct samsung_aes_variant *variant; }; static struct s5p_aes_dev *s5p_dev; @@ -421,11 +421,11 @@ static bool s5p_aes_rx(struct s5p_aes_dev *dev) static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) { struct platform_device *pdev = dev_id; - struct s5p_aes_dev *dev = platform_get_drvdata(pdev); - uint32_t status; - unsigned long flags; - bool set_dma_tx = false; - bool set_dma_rx = false; + struct s5p_aes_dev *dev = platform_get_drvdata(pdev); + bool set_dma_tx = false; + bool set_dma_rx = false; + unsigned long flags; + uint32_t status; spin_lock_irqsave(&dev->lock, flags); @@ -538,10 +538,10 @@ static int s5p_set_outdata_start(struct s5p_aes_dev *dev, static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode) { - struct ablkcipher_request *req = dev->req; - uint32_t aes_control; - int err; - unsigned long flags; + struct ablkcipher_request *req = dev->req; + uint32_t aes_control; + unsigned long flags; + int err; aes_control = SSS_AES_KEY_CHANGE_MODE; if (mode & FLAGS_AES_DECRYPT) @@ -653,10 +653,10 @@ exit: static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); - struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req); - struct s5p_aes_dev *dev = ctx->dev; + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req); + struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct s5p_aes_dev *dev = ctx->dev; if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { dev_err(dev->dev, "request size is not exact amount of AES blocks\n"); @@ -671,7 +671,7 @@ static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode) static int s5p_aes_setkey(struct crypto_ablkcipher *cipher, const uint8_t *key, unsigned int keylen) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm); if (keylen != AES_KEYSIZE_128 && @@ -763,11 +763,11 @@ static struct crypto_alg algs[] = { static int s5p_aes_probe(struct platform_device *pdev) { - int i, j, err = -ENODEV; - struct s5p_aes_dev *pdata; - struct device *dev = &pdev->dev; - struct resource *res; + struct device *dev = &pdev->dev; + int i, j, err = -ENODEV; struct samsung_aes_variant *variant; + struct s5p_aes_dev *pdata; + struct resource *res; if (s5p_dev) return -EEXIST; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index c3f3d89e4831..0c49956ee0ce 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -14,10 +14,9 @@ * Based on omap-aes.c and tegra-aes.c */ -#include #include -#include #include +#include #include #include @@ -150,10 +149,7 @@ struct sahara_ctx { /* AES-specific context */ int keylen; u8 key[AES_KEYSIZE_128]; - struct crypto_ablkcipher *fallback; - - /* SHA-specific context */ - struct crypto_shash *shash_fallback; + struct crypto_skcipher *fallback; }; struct sahara_aes_reqctx { @@ -620,25 +616,21 @@ static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, return 0; } - if (keylen != AES_KEYSIZE_128 && - keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) + if (keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) return -EINVAL; /* * The requested key size is not supported by HW, do a fallback. */ - ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - ctx->fallback->base.crt_flags |= - (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); - ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen); - if (ret) { - struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm); + ret = crypto_skcipher_setkey(ctx->fallback, key, keylen); - tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm_aux->crt_flags |= - (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK); - } + tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->base.crt_flags |= crypto_skcipher_get_flags(ctx->fallback) & + CRYPTO_TFM_RES_MASK; return ret; } @@ -670,16 +662,20 @@ static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode) static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct sahara_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); int err; if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_encrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -688,16 +684,20 @@ static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req) static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct sahara_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); int err; if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -706,16 +706,20 @@ static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req) static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct sahara_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); int err; if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_encrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -724,16 +728,20 @@ static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req) static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct sahara_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); int err; if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -745,8 +753,9 @@ static int sahara_aes_cra_init(struct crypto_tfm *tfm) const char *name = crypto_tfm_alg_name(tfm); struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->fallback = crypto_alloc_ablkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + ctx->fallback = crypto_alloc_skcipher(name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->fallback)) { pr_err("Error allocating fallback algo %s\n", name); return PTR_ERR(ctx->fallback); @@ -761,9 +770,7 @@ static void sahara_aes_cra_exit(struct crypto_tfm *tfm) { struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->fallback) - crypto_free_ablkcipher(ctx->fallback); - ctx->fallback = NULL; + crypto_free_skcipher(ctx->fallback); } static u32 sahara_sha_init_hdr(struct sahara_dev *dev, @@ -1180,15 +1187,6 @@ static int sahara_sha_import(struct ahash_request *req, const void *in) static int sahara_sha_cra_init(struct crypto_tfm *tfm) { - const char *name = crypto_tfm_alg_name(tfm); - struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->shash_fallback = crypto_alloc_shash(name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->shash_fallback)) { - pr_err("Error allocating fallback algo %s\n", name); - return PTR_ERR(ctx->shash_fallback); - } crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct sahara_sha_reqctx) + SHA_BUFFER_LEN + SHA256_BLOCK_SIZE); @@ -1196,14 +1194,6 @@ static int sahara_sha_cra_init(struct crypto_tfm *tfm) return 0; } -static void sahara_sha_cra_exit(struct crypto_tfm *tfm) -{ - struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_shash(ctx->shash_fallback); - ctx->shash_fallback = NULL; -} - static struct crypto_alg aes_algs[] = { { .cra_name = "ecb(aes)", @@ -1272,7 +1262,6 @@ static struct ahash_alg sha_v3_algs[] = { .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = sahara_sha_cra_init, - .cra_exit = sahara_sha_cra_exit, } }, }; @@ -1300,7 +1289,6 @@ static struct ahash_alg sha_v4_algs[] = { .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = sahara_sha_cra_init, - .cra_exit = sahara_sha_cra_exit, } }, }; diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index b7ee8d30147d..0418a2f41dc0 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -91,10 +91,17 @@ static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, return be16_to_cpu(ptr->len); } -static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1) +static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, + bool is_sec1) { if (!is_sec1) - ptr->j_extent = 0; + ptr->j_extent = val; +} + +static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) +{ + if (!is_sec1) + ptr->j_extent |= val; } /* @@ -111,7 +118,7 @@ static void map_single_talitos_ptr(struct device *dev, to_talitos_ptr_len(ptr, len, is_sec1); to_talitos_ptr(ptr, dma_addr, is_sec1); - to_talitos_ptr_extent_clear(ptr, is_sec1); + to_talitos_ptr_ext_set(ptr, 0, is_sec1); } /* @@ -804,6 +811,11 @@ static void talitos_unregister_rng(struct device *dev) * crypto alg */ #define TALITOS_CRA_PRIORITY 3000 +/* + * Defines a priority for doing AEAD with descriptors type + * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP + */ +#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) #define TALITOS_MAX_KEY_SIZE 96 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ @@ -904,35 +916,59 @@ struct talitos_edesc { static void talitos_sg_unmap(struct device *dev, struct talitos_edesc *edesc, struct scatterlist *src, - struct scatterlist *dst) + struct scatterlist *dst, + unsigned int len, unsigned int offset) { + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); unsigned int src_nents = edesc->src_nents ? : 1; unsigned int dst_nents = edesc->dst_nents ? : 1; + if (is_sec1 && dst && dst_nents > 1) { + dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset, + len, DMA_FROM_DEVICE); + sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len, + offset); + } if (src != dst) { - dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + if (src_nents == 1 || !is_sec1) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); - if (dst) { + if (dst && (dst_nents == 1 || !is_sec1)) dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); - } - } else + } else if (src_nents == 1 || !is_sec1) { dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } } static void ipsec_esp_unmap(struct device *dev, struct talitos_edesc *edesc, struct aead_request *areq) { - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE); + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct talitos_ctx *ctx = crypto_aead_ctx(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + + if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) + unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], + DMA_FROM_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst); + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, + areq->assoclen); if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); + + if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) { + unsigned int dst_nents = edesc->dst_nents ? : 1; + + sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, + areq->assoclen + areq->cryptlen - ivsize); + } } /* @@ -942,6 +978,8 @@ static void ipsec_esp_encrypt_done(struct device *dev, struct talitos_desc *desc, void *context, int err) { + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); unsigned int authsize = crypto_aead_authsize(authenc); @@ -955,8 +993,11 @@ static void ipsec_esp_encrypt_done(struct device *dev, /* copy the generated ICV to dst */ if (edesc->icv_ool) { - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; + if (is_sec1) + icvdata = edesc->buf + areq->assoclen + areq->cryptlen; + else + icvdata = &edesc->link_tbl[edesc->src_nents + + edesc->dst_nents + 2]; sg = sg_last(areq->dst, edesc->dst_nents); memcpy((char *)sg_virt(sg) + sg->length - authsize, icvdata, authsize); @@ -977,6 +1018,8 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, struct talitos_edesc *edesc; struct scatterlist *sg; char *oicv, *icv; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); edesc = container_of(desc, struct talitos_edesc, desc); @@ -988,7 +1031,12 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, icv = (char *)sg_virt(sg) + sg->length - authsize; if (edesc->dma_len) { - oicv = (char *)&edesc->link_tbl[edesc->src_nents + + if (is_sec1) + oicv = (char *)&edesc->dma_link_tbl + + req->assoclen + req->cryptlen; + else + oicv = (char *) + &edesc->link_tbl[edesc->src_nents + edesc->dst_nents + 2]; if (edesc->icv_ool) icv = oicv + authsize; @@ -1050,8 +1098,8 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, to_talitos_ptr(link_tbl_ptr + count, sg_dma_address(sg) + offset, 0); - link_tbl_ptr[count].len = cpu_to_be16(len); - link_tbl_ptr[count].j_extent = 0; + to_talitos_ptr_len(link_tbl_ptr + count, len, 0); + to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; offset = 0; @@ -1062,17 +1110,43 @@ next: /* tag end of link table */ if (count > 0) - link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN; + to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, + DESC_PTR_LNKTBL_RETURN, 0); return count; } -static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count, - int cryptlen, - struct talitos_ptr *link_tbl_ptr) +int talitos_sg_map(struct device *dev, struct scatterlist *src, + unsigned int len, struct talitos_edesc *edesc, + struct talitos_ptr *ptr, + int sg_count, unsigned int offset, int tbl_off) { - return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen, - link_tbl_ptr); + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + to_talitos_ptr_len(ptr, len, is_sec1); + to_talitos_ptr_ext_set(ptr, 0, is_sec1); + + if (sg_count == 1) { + to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1); + return sg_count; + } + if (is_sec1) { + to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, is_sec1); + return sg_count; + } + sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, + &edesc->link_tbl[tbl_off]); + if (sg_count == 1) { + /* Only one segment now, so no link tbl needed*/ + copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); + return sg_count; + } + to_talitos_ptr(ptr, edesc->dma_link_tbl + + tbl_off * sizeof(struct talitos_ptr), is_sec1); + to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); + + return sg_count; } /* @@ -1093,42 +1167,52 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, int tbl_off = 0; int sg_count, ret; int sg_link_tbl_len; + bool sync_needed = false; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); /* hmac key */ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, DMA_TO_DEVICE); - sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1, - (areq->src == areq->dst) ? DMA_BIDIRECTIONAL - : DMA_TO_DEVICE); - /* hmac data */ - desc->ptr[1].len = cpu_to_be16(areq->assoclen); - if (sg_count > 1 && - (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0, - areq->assoclen, - &edesc->link_tbl[tbl_off])) > 1) { - to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off * - sizeof(struct talitos_ptr), 0); - desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP; + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(areq->src, sg_count, edesc->buf, + areq->assoclen + cryptlen); + else + sg_count = dma_map_sg(dev, areq->src, sg_count, + (areq->src == areq->dst) ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, DMA_BIDIRECTIONAL); + /* hmac data */ + ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc, + &desc->ptr[1], sg_count, 0, tbl_off); + if (ret > 1) { tbl_off += ret; - } else { - to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0); - desc->ptr[1].j_extent = 0; + sync_needed = true; } /* cipher iv */ - to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0); - desc->ptr[2].len = cpu_to_be16(ivsize); - desc->ptr[2].j_extent = 0; + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1); + to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1); + to_talitos_ptr_ext_set(&desc->ptr[2], 0, is_sec1); + } else { + to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1); + to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1); + to_talitos_ptr_ext_set(&desc->ptr[3], 0, is_sec1); + } /* cipher key */ - map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, - (char *)&ctx->key + ctx->authkeylen, - DMA_TO_DEVICE); + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, + (char *)&ctx->key + ctx->authkeylen, + DMA_TO_DEVICE); + else + map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen, + (char *)&ctx->key + ctx->authkeylen, + DMA_TO_DEVICE); /* * cipher in @@ -1136,78 +1220,82 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, * extent is bytes of HMAC postpended to ciphertext, * typically 12 for ipsec */ - desc->ptr[4].len = cpu_to_be16(cryptlen); - desc->ptr[4].j_extent = authsize; + to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1); + to_talitos_ptr_ext_set(&desc->ptr[4], 0, is_sec1); sg_link_tbl_len = cryptlen; - if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) - sg_link_tbl_len += authsize; - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) + - areq->assoclen, 0); - } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count, - areq->assoclen, sg_link_tbl_len, - &edesc->link_tbl[tbl_off])) > - 1) { - desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; - to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + - tbl_off * - sizeof(struct talitos_ptr), 0); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - tbl_off += ret; - } else { - copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0); + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1); + + if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) + sg_link_tbl_len += authsize; } - /* cipher out */ - desc->ptr[5].len = cpu_to_be16(cryptlen); - desc->ptr[5].j_extent = authsize; + sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, + &desc->ptr[4], sg_count, areq->assoclen, + tbl_off); - if (areq->src != areq->dst) - sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1, - DMA_FROM_DEVICE); + if (sg_count > 1) { + tbl_off += sg_count; + sync_needed = true; + } - edesc->icv_ool = false; + /* cipher out */ + if (areq->src != areq->dst) { + sg_count = edesc->dst_nents ? : 1; + if (!is_sec1 || sg_count == 1) + dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); + } - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) + - areq->assoclen, 0); - } else if ((sg_count = - sg_to_link_tbl_offset(areq->dst, sg_count, - areq->assoclen, cryptlen, - &edesc->link_tbl[tbl_off])) > 1) { - struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; - - to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + - tbl_off * sizeof(struct talitos_ptr), 0); - - /* Add an entry to the link table for ICV data */ - tbl_ptr += sg_count - 1; - tbl_ptr->j_extent = 0; - tbl_ptr++; - tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; - tbl_ptr->len = cpu_to_be16(authsize); - - /* icv data follows link tables */ - to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + - (edesc->src_nents + edesc->dst_nents + - 2) * sizeof(struct talitos_ptr) + - authsize, 0); - desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; - dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, - edesc->dma_len, DMA_BIDIRECTIONAL); + sg_count = talitos_sg_map(dev, areq->dst, cryptlen, edesc, + &desc->ptr[5], sg_count, areq->assoclen, + tbl_off); + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); + + if (sg_count > 1) { edesc->icv_ool = true; + sync_needed = true; + + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; + int offset = (edesc->src_nents + edesc->dst_nents + 2) * + sizeof(struct talitos_ptr) + authsize; + + /* Add an entry to the link table for ICV data */ + tbl_ptr += sg_count - 1; + to_talitos_ptr_ext_set(tbl_ptr, 0, is_sec1); + tbl_ptr++; + to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN, + is_sec1); + to_talitos_ptr_len(tbl_ptr, authsize, is_sec1); + + /* icv data follows link tables */ + to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset, + is_sec1); + } } else { - copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0); + edesc->icv_ool = false; + } + + /* ICV data */ + if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) { + to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1); + to_talitos_ptr(&desc->ptr[6], edesc->dma_link_tbl + + areq->assoclen + cryptlen, is_sec1); } /* iv out */ - map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, - DMA_FROM_DEVICE); + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, + DMA_FROM_DEVICE); + + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, + DMA_BIDIRECTIONAL); ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { @@ -1233,7 +1321,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, bool encrypt) { struct talitos_edesc *edesc; - int src_nents, dst_nents, alloc_len, dma_len; + int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; dma_addr_t iv_dma = 0; gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1251,8 +1339,8 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); if (!dst || dst == src) { - src_nents = sg_nents_for_len(src, - assoclen + cryptlen + authsize); + src_len = assoclen + cryptlen + authsize; + src_nents = sg_nents_for_len(src, src_len); if (src_nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); err = ERR_PTR(-EINVAL); @@ -1260,17 +1348,18 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, } src_nents = (src_nents == 1) ? 0 : src_nents; dst_nents = dst ? src_nents : 0; + dst_len = 0; } else { /* dst && dst != src*/ - src_nents = sg_nents_for_len(src, assoclen + cryptlen + - (encrypt ? 0 : authsize)); + src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); + src_nents = sg_nents_for_len(src, src_len); if (src_nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); err = ERR_PTR(-EINVAL); goto error_sg; } src_nents = (src_nents == 1) ? 0 : src_nents; - dst_nents = sg_nents_for_len(dst, assoclen + cryptlen + - (encrypt ? authsize : 0)); + dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); + dst_nents = sg_nents_for_len(dst, dst_len); if (dst_nents < 0) { dev_err(dev, "Invalid number of dst SG.\n"); err = ERR_PTR(-EINVAL); @@ -1287,8 +1376,8 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, alloc_len = sizeof(struct talitos_edesc); if (src_nents || dst_nents) { if (is_sec1) - dma_len = (src_nents ? cryptlen : 0) + - (dst_nents ? cryptlen : 0); + dma_len = (src_nents ? src_len : 0) + + (dst_nents ? dst_len : 0); else dma_len = (src_nents + dst_nents + 2) * sizeof(struct talitos_ptr) + authsize * 2; @@ -1412,40 +1501,13 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, return 0; } -static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src, - struct scatterlist *dst, unsigned int len, - struct talitos_edesc *edesc) -{ - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - if (is_sec1) { - if (!edesc->src_nents) { - dma_unmap_sg(dev, src, 1, - dst != src ? DMA_TO_DEVICE - : DMA_BIDIRECTIONAL); - } - if (dst && edesc->dst_nents) { - dma_sync_single_for_device(dev, - edesc->dma_link_tbl + len, - len, DMA_FROM_DEVICE); - sg_copy_from_buffer(dst, edesc->dst_nents ? : 1, - edesc->buf + len, len); - } else if (dst && dst != src) { - dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE); - } - } else { - talitos_sg_unmap(dev, edesc, src, dst); - } -} - static void common_nonsnoop_unmap(struct device *dev, struct talitos_edesc *edesc, struct ablkcipher_request *areq) { unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); - unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc); + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); @@ -1470,100 +1532,6 @@ static void ablkcipher_done(struct device *dev, areq->base.complete(&areq->base, err); } -int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src, - unsigned int len, struct talitos_edesc *edesc, - enum dma_data_direction dir, struct talitos_ptr *ptr) -{ - int sg_count; - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - to_talitos_ptr_len(ptr, len, is_sec1); - - if (is_sec1) { - sg_count = edesc->src_nents ? : 1; - - if (sg_count == 1) { - dma_map_sg(dev, src, 1, dir); - to_talitos_ptr(ptr, sg_dma_address(src), is_sec1); - } else { - sg_copy_to_buffer(src, sg_count, edesc->buf, len); - to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - len, DMA_TO_DEVICE); - } - } else { - to_talitos_ptr_extent_clear(ptr, is_sec1); - - sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir); - - if (sg_count == 1) { - to_talitos_ptr(ptr, sg_dma_address(src), is_sec1); - } else { - sg_count = sg_to_link_tbl(src, sg_count, len, - &edesc->link_tbl[0]); - if (sg_count > 1) { - to_talitos_ptr(ptr, edesc->dma_link_tbl, 0); - ptr->j_extent |= DESC_PTR_LNKTBL_JUMP; - dma_sync_single_for_device(dev, - edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - } else { - /* Only one segment now, so no link tbl needed*/ - to_talitos_ptr(ptr, sg_dma_address(src), - is_sec1); - } - } - } - return sg_count; -} - -void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst, - unsigned int len, struct talitos_edesc *edesc, - enum dma_data_direction dir, - struct talitos_ptr *ptr, int sg_count) -{ - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - if (dir != DMA_NONE) - sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir); - - to_talitos_ptr_len(ptr, len, is_sec1); - - if (is_sec1) { - if (sg_count == 1) { - if (dir != DMA_NONE) - dma_map_sg(dev, dst, 1, dir); - to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1); - } else { - to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1); - dma_sync_single_for_device(dev, - edesc->dma_link_tbl + len, - len, DMA_FROM_DEVICE); - } - } else { - to_talitos_ptr_extent_clear(ptr, is_sec1); - - if (sg_count == 1) { - to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1); - } else { - struct talitos_ptr *link_tbl_ptr = - &edesc->link_tbl[edesc->src_nents + 1]; - - to_talitos_ptr(ptr, edesc->dma_link_tbl + - (edesc->src_nents + 1) * - sizeof(struct talitos_ptr), 0); - ptr->j_extent |= DESC_PTR_LNKTBL_JUMP; - sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - } - } -} - static int common_nonsnoop(struct talitos_edesc *edesc, struct ablkcipher_request *areq, void (*callback) (struct device *dev, @@ -1577,6 +1545,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, unsigned int cryptlen = areq->nbytes; unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); int sg_count, ret; + bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); @@ -1586,25 +1555,39 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* cipher iv */ to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1); to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1); - to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1); + to_talitos_ptr_ext_set(&desc->ptr[1], 0, is_sec1); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, (char *)&ctx->key, DMA_TO_DEVICE); + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(areq->src, sg_count, edesc->buf, + cryptlen); + else + sg_count = dma_map_sg(dev, areq->src, sg_count, + (areq->src == areq->dst) ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE); /* * cipher in */ - sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc, - (areq->src == areq->dst) ? - DMA_BIDIRECTIONAL : DMA_TO_DEVICE, - &desc->ptr[3]); + sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, + &desc->ptr[3], sg_count, 0, 0); + if (sg_count > 1) + sync_needed = true; /* cipher out */ - map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc, - (areq->src == areq->dst) ? DMA_NONE - : DMA_FROM_DEVICE, - &desc->ptr[4], sg_count); + if (areq->src != areq->dst) { + sg_count = edesc->dst_nents ? : 1; + if (!is_sec1 || sg_count == 1) + dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); + } + + ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4], + sg_count, 0, (edesc->src_nents + 1)); + if (ret > 1) + sync_needed = true; /* iv out */ map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, @@ -1613,6 +1596,10 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* last DWORD empty */ desc->ptr[6] = zero_entry; + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, DMA_BIDIRECTIONAL); + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { common_nonsnoop_unmap(dev, edesc, areq); @@ -1676,7 +1663,7 @@ static void common_nonsnoop_hash_unmap(struct device *dev, unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); - unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc); + talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); /* When using hashctx-in, must unmap it. */ if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) @@ -1747,8 +1734,10 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; int ret; + bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); + int sg_count; /* first DWORD empty */ desc->ptr[0] = zero_entry; @@ -1773,11 +1762,19 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, else desc->ptr[2] = zero_entry; + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(areq->src, sg_count, edesc->buf, length); + else + sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, + DMA_TO_DEVICE); /* * data in */ - map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc, - DMA_TO_DEVICE, &desc->ptr[3]); + sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, + &desc->ptr[3], sg_count, 0, 0); + if (sg_count > 1) + sync_needed = true; /* fifth DWORD empty */ desc->ptr[4] = zero_entry; @@ -1798,6 +1795,10 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, DMA_BIDIRECTIONAL); + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { common_nonsnoop_hash_unmap(dev, edesc, areq); @@ -2124,6 +2125,7 @@ static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, struct talitos_alg_template { u32 type; + u32 priority; union { struct crypto_alg crypto; struct ahash_alg hash; @@ -2154,6 +2156,27 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA1_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2176,6 +2199,29 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA1_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2196,6 +2242,27 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA224_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2218,6 +2285,29 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA224_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2238,6 +2328,27 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA256_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA256_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2260,6 +2371,29 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA256_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA256_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2364,6 +2498,27 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_MD5_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_MD5_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2385,6 +2540,28 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_MD5_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_MD5_HMAC, + }, /* ABLKCIPHER algorithms. */ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .alg.crypto = { @@ -2901,7 +3078,10 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, } alg->cra_module = THIS_MODULE; - alg->cra_priority = TALITOS_CRA_PRIORITY; + if (t_alg->algt.priority) + alg->cra_priority = t_alg->algt.priority; + else + alg->cra_priority = TALITOS_CRA_PRIORITY; alg->cra_alignmask = 0; alg->cra_ctxsize = sizeof(struct talitos_ctx); alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; diff --git a/drivers/crypto/ux500/cryp/Makefile b/drivers/crypto/ux500/cryp/Makefile index e5d362a6f680..b497ae3dde07 100644 --- a/drivers/crypto/ux500/cryp/Makefile +++ b/drivers/crypto/ux500/cryp/Makefile @@ -4,9 +4,9 @@ # * License terms: GNU General Public License (GPL) version 2 */ ifdef CONFIG_CRYPTO_DEV_UX500_DEBUG -CFLAGS_cryp_core.o := -DDEBUG -O0 -CFLAGS_cryp.o := -DDEBUG -O0 -CFLAGS_cryp_irq.o := -DDEBUG -O0 +CFLAGS_cryp_core.o := -DDEBUG +CFLAGS_cryp.o := -DDEBUG +CFLAGS_cryp_irq.o := -DDEBUG endif obj-$(CONFIG_CRYPTO_DEV_UX500_CRYP) += ux500_cryp.o diff --git a/drivers/crypto/ux500/hash/Makefile b/drivers/crypto/ux500/hash/Makefile index b2f90d9bac72..784d9c0a8853 100644 --- a/drivers/crypto/ux500/hash/Makefile +++ b/drivers/crypto/ux500/hash/Makefile @@ -4,7 +4,7 @@ # License terms: GNU General Public License (GPL) version 2 # ifdef CONFIG_CRYPTO_DEV_UX500_DEBUG -CFLAGS_hash_core.o := -DDEBUG -O0 +CFLAGS_hash_core.o := -DDEBUG endif obj-$(CONFIG_CRYPTO_DEV_UX500_HASH) += ux500_hash.o diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 574e87c7f2b8..9acccad26928 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -781,7 +781,7 @@ static int hash_process_data(struct hash_device_data *device_data, &device_data->state); memmove(req_ctx->state.buffer, device_data->state.buffer, - HASH_BLOCK_SIZE / sizeof(u32)); + HASH_BLOCK_SIZE); if (ret) { dev_err(device_data->dev, "%s: hash_resume_state() failed!\n", @@ -832,7 +832,7 @@ static int hash_process_data(struct hash_device_data *device_data, memmove(device_data->state.buffer, req_ctx->state.buffer, - HASH_BLOCK_SIZE / sizeof(u32)); + HASH_BLOCK_SIZE); if (ret) { dev_err(device_data->dev, "%s: hash_save_state() failed!\n", __func__); diff --git a/drivers/crypto/vmx/.gitignore b/drivers/crypto/vmx/.gitignore new file mode 100644 index 000000000000..af4a7ce4738d --- /dev/null +++ b/drivers/crypto/vmx/.gitignore @@ -0,0 +1,2 @@ +aesp8-ppc.S +ghashp8-ppc.S diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile index d28ab96a2475..de6e241b0866 100644 --- a/drivers/crypto/vmx/Makefile +++ b/drivers/crypto/vmx/Makefile @@ -1,5 +1,5 @@ obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o -vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o ghash.o +vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) TARGET := linux-ppc64le diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c index 495577b6d31b..94ad5c0adbcb 100644 --- a/drivers/crypto/vmx/aes_cbc.c +++ b/drivers/crypto/vmx/aes_cbc.c @@ -182,7 +182,7 @@ struct crypto_alg p8_aes_cbc_alg = { .cra_name = "cbc(aes)", .cra_driver_name = "p8_aes_cbc", .cra_module = THIS_MODULE, - .cra_priority = 1000, + .cra_priority = 2000, .cra_type = &crypto_blkcipher_type, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_alignmask = 0, diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index 0a3c1b04cf3c..38ed10d761d0 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -166,7 +166,7 @@ struct crypto_alg p8_aes_ctr_alg = { .cra_name = "ctr(aes)", .cra_driver_name = "p8_aes_ctr", .cra_module = THIS_MODULE, - .cra_priority = 1000, + .cra_priority = 2000, .cra_type = &crypto_blkcipher_type, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_alignmask = 0, diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c new file mode 100644 index 000000000000..cfb25413917c --- /dev/null +++ b/drivers/crypto/vmx/aes_xts.c @@ -0,0 +1,190 @@ +/** + * AES XTS routines supporting VMX In-core instructions on Power 8 + * + * Copyright (C) 2015 International Business Machines Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundations; version 2 only. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY of FITNESS FOR A PARTICUPAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Author: Leonidas S. Barbosa + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aesp8-ppc.h" + +struct p8_aes_xts_ctx { + struct crypto_blkcipher *fallback; + struct aes_key enc_key; + struct aes_key dec_key; + struct aes_key tweak_key; +}; + +static int p8_aes_xts_init(struct crypto_tfm *tfm) +{ + const char *alg; + struct crypto_blkcipher *fallback; + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + if (!(alg = crypto_tfm_alg_name(tfm))) { + printk(KERN_ERR "Failed to get algorithm name.\n"); + return -ENOENT; + } + + fallback = + crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + printk(KERN_ERR + "Failed to allocate transformation for '%s': %ld\n", + alg, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + printk(KERN_INFO "Using '%s' as fallback implementation.\n", + crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); + + crypto_blkcipher_set_flags( + fallback, + crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm)); + ctx->fallback = fallback; + + return 0; +} + +static void p8_aes_xts_exit(struct crypto_tfm *tfm) +{ + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->fallback) { + crypto_free_blkcipher(ctx->fallback); + ctx->fallback = NULL; + } +} + +static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + int ret; + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + ret = xts_check_key(tfm, key, keylen); + if (ret) + return ret; + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key); + ret += aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key); + ret += aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); + return ret; +} + +static int p8_aes_xts_crypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, + unsigned int nbytes, int enc) +{ + int ret; + u8 tweak[AES_BLOCK_SIZE]; + u8 *iv; + struct blkcipher_walk walk; + struct p8_aes_xts_ctx *ctx = + crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); + struct blkcipher_desc fallback_desc = { + .tfm = ctx->fallback, + .info = desc->info, + .flags = desc->flags + }; + + if (in_interrupt()) { + ret = enc ? crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes) : + crypto_blkcipher_decrypt(&fallback_desc, dst, src, nbytes); + } else { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + + blkcipher_walk_init(&walk, dst, src, nbytes); + + iv = (u8 *)walk.iv; + ret = blkcipher_walk_virt(desc, &walk); + memset(tweak, 0, AES_BLOCK_SIZE); + aes_p8_encrypt(iv, tweak, &ctx->tweak_key); + + while ((nbytes = walk.nbytes)) { + if (enc) + aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, + nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak); + else + aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, + nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak); + + nbytes &= AES_BLOCK_SIZE - 1; + ret = blkcipher_walk_done(desc, &walk, nbytes); + } + + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + } + return ret; +} + +static int p8_aes_xts_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return p8_aes_xts_crypt(desc, dst, src, nbytes, 1); +} + +static int p8_aes_xts_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return p8_aes_xts_crypt(desc, dst, src, nbytes, 0); +} + +struct crypto_alg p8_aes_xts_alg = { + .cra_name = "xts(aes)", + .cra_driver_name = "p8_aes_xts", + .cra_module = THIS_MODULE, + .cra_priority = 2000, + .cra_type = &crypto_blkcipher_type, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = 0, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct p8_aes_xts_ctx), + .cra_init = p8_aes_xts_init, + .cra_exit = p8_aes_xts_exit, + .cra_blkcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .setkey = p8_aes_xts_setkey, + .encrypt = p8_aes_xts_encrypt, + .decrypt = p8_aes_xts_decrypt, + } +}; diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h index 4cd34ee54a94..01972e16a6c0 100644 --- a/drivers/crypto/vmx/aesp8-ppc.h +++ b/drivers/crypto/vmx/aesp8-ppc.h @@ -19,3 +19,7 @@ void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len, void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, size_t len, const struct aes_key *key, const u8 *iv); +void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len, + const struct aes_key *key1, const struct aes_key *key2, u8 *iv); +void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len, + const struct aes_key *key1, const struct aes_key *key2, u8 *iv); diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl index 228053921b3f..0b4a293b8a1e 100644 --- a/drivers/crypto/vmx/aesp8-ppc.pl +++ b/drivers/crypto/vmx/aesp8-ppc.pl @@ -1,4 +1,11 @@ -#!/usr/bin/env perl +#! /usr/bin/env perl +# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the OpenSSL license (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + # # ==================================================================== # Written by Andy Polyakov for the OpenSSL @@ -20,6 +27,19 @@ # instructions are interleaved. It's reckoned that eventual # misalignment penalties at page boundaries are in average lower # than additional overhead in pure AltiVec approach. +# +# May 2016 +# +# Add XTS subroutine, 9x on little- and 12x improvement on big-endian +# systems were measured. +# +###################################################################### +# Current large-block performance in cycles per byte processed with +# 128-bit key (less is better). +# +# CBC en-/decrypt CTR XTS +# POWER8[le] 3.96/0.72 0.74 1.1 +# POWER8[be] 3.75/0.65 0.66 1.0 $flavour = shift; @@ -1875,6 +1895,1845 @@ Lctr32_enc8x_done: ___ }} }}} +######################################################################### +{{{ # XTS procedures # +# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len, # +# const AES_KEY *key1, const AES_KEY *key2, # +# [const] unsigned char iv[16]); # +# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which # +# input tweak value is assumed to be encrypted already, and last tweak # +# value, one suitable for consecutive call on same chunk of data, is # +# written back to original buffer. In addition, in "tweak chaining" # +# mode only complete input blocks are processed. # + +my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10)); +my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2)); +my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7)); +my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12)); +my $taillen = $key2; + + ($inp,$idx) = ($idx,$inp); # reassign + +$code.=<<___; +.globl .${prefix}_xts_encrypt + mr $inp,r3 # reassign + li r3,-1 + ${UCMP}i $len,16 + bltlr- + + lis r0,0xfff0 + mfspr r12,256 # save vrsave + li r11,0 + mtspr 256,r0 + + vspltisb $seven,0x07 # 0x070707..07 + le?lvsl $leperm,r11,r11 + le?vspltisb $tmp,0x0f + le?vxor $leperm,$leperm,$seven + + li $idx,15 + lvx $tweak,0,$ivp # load [unaligned] iv + lvsl $inpperm,0,$ivp + lvx $inptail,$idx,$ivp + le?vxor $inpperm,$inpperm,$tmp + vperm $tweak,$tweak,$inptail,$inpperm + + neg r11,$inp + lvsr $inpperm,0,r11 # prepare for unaligned load + lvx $inout,0,$inp + addi $inp,$inp,15 # 15 is not typo + le?vxor $inpperm,$inpperm,$tmp + + ${UCMP}i $key2,0 # key2==NULL? + beq Lxts_enc_no_key2 + + ?lvsl $keyperm,0,$key2 # prepare for unaligned key + lwz $rounds,240($key2) + srwi $rounds,$rounds,1 + subi $rounds,$rounds,1 + li $idx,16 + + lvx $rndkey0,0,$key2 + lvx $rndkey1,$idx,$key2 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $tweak,$tweak,$rndkey0 + lvx $rndkey0,$idx,$key2 + addi $idx,$idx,16 + mtctr $rounds + +Ltweak_xts_enc: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $tweak,$tweak,$rndkey1 + lvx $rndkey1,$idx,$key2 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipher $tweak,$tweak,$rndkey0 + lvx $rndkey0,$idx,$key2 + addi $idx,$idx,16 + bdnz Ltweak_xts_enc + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $tweak,$tweak,$rndkey1 + lvx $rndkey1,$idx,$key2 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipherlast $tweak,$tweak,$rndkey0 + + li $ivp,0 # don't chain the tweak + b Lxts_enc + +Lxts_enc_no_key2: + li $idx,-16 + and $len,$len,$idx # in "tweak chaining" + # mode only complete + # blocks are processed +Lxts_enc: + lvx $inptail,0,$inp + addi $inp,$inp,16 + + ?lvsl $keyperm,0,$key1 # prepare for unaligned key + lwz $rounds,240($key1) + srwi $rounds,$rounds,1 + subi $rounds,$rounds,1 + li $idx,16 + + vslb $eighty7,$seven,$seven # 0x808080..80 + vor $eighty7,$eighty7,$seven # 0x878787..87 + vspltisb $tmp,1 # 0x010101..01 + vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01 + + ${UCMP}i $len,96 + bge _aesp8_xts_encrypt6x + + andi. $taillen,$len,15 + subic r0,$len,32 + subi $taillen,$taillen,16 + subfe r0,r0,r0 + and r0,r0,$taillen + add $inp,$inp,r0 + + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$tweak + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + mtctr $rounds + b Loop_xts_enc + +.align 5 +Loop_xts_enc: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + bdnz Loop_xts_enc + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + li $idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $rndkey0,$rndkey0,$tweak + vcipherlast $output,$inout,$rndkey0 + + le?vperm $tmp,$output,$output,$leperm + be?nop + le?stvx_u $tmp,0,$out + be?stvx_u $output,0,$out + addi $out,$out,16 + + subic. $len,$len,16 + beq Lxts_enc_done + + vmr $inout,$inptail + lvx $inptail,0,$inp + addi $inp,$inp,16 + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + + subic r0,$len,32 + subfe r0,r0,r0 + and r0,r0,$taillen + add $inp,$inp,r0 + + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak,$tweak,$tmp + + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$tweak + vxor $output,$output,$rndkey0 # just in case $len<16 + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + + mtctr $rounds + ${UCMP}i $len,16 + bge Loop_xts_enc + + vxor $output,$output,$tweak + lvsr $inpperm,0,$len # $inpperm is no longer needed + vxor $inptail,$inptail,$inptail # $inptail is no longer needed + vspltisb $tmp,-1 + vperm $inptail,$inptail,$tmp,$inpperm + vsel $inout,$inout,$output,$inptail + + subi r11,$out,17 + subi $out,$out,16 + mtctr $len + li $len,16 +Loop_xts_enc_steal: + lbzu r0,1(r11) + stb r0,16(r11) + bdnz Loop_xts_enc_steal + + mtctr $rounds + b Loop_xts_enc # one more time... + +Lxts_enc_done: + ${UCMP}i $ivp,0 + beq Lxts_enc_ret + + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak,$tweak,$tmp + + le?vperm $tweak,$tweak,$tweak,$leperm + stvx_u $tweak,0,$ivp + +Lxts_enc_ret: + mtspr 256,r12 # restore vrsave + li r3,0 + blr + .long 0 + .byte 0,12,0x04,0,0x80,6,6,0 + .long 0 +.size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt + +.globl .${prefix}_xts_decrypt + mr $inp,r3 # reassign + li r3,-1 + ${UCMP}i $len,16 + bltlr- + + lis r0,0xfff8 + mfspr r12,256 # save vrsave + li r11,0 + mtspr 256,r0 + + andi. r0,$len,15 + neg r0,r0 + andi. r0,r0,16 + sub $len,$len,r0 + + vspltisb $seven,0x07 # 0x070707..07 + le?lvsl $leperm,r11,r11 + le?vspltisb $tmp,0x0f + le?vxor $leperm,$leperm,$seven + + li $idx,15 + lvx $tweak,0,$ivp # load [unaligned] iv + lvsl $inpperm,0,$ivp + lvx $inptail,$idx,$ivp + le?vxor $inpperm,$inpperm,$tmp + vperm $tweak,$tweak,$inptail,$inpperm + + neg r11,$inp + lvsr $inpperm,0,r11 # prepare for unaligned load + lvx $inout,0,$inp + addi $inp,$inp,15 # 15 is not typo + le?vxor $inpperm,$inpperm,$tmp + + ${UCMP}i $key2,0 # key2==NULL? + beq Lxts_dec_no_key2 + + ?lvsl $keyperm,0,$key2 # prepare for unaligned key + lwz $rounds,240($key2) + srwi $rounds,$rounds,1 + subi $rounds,$rounds,1 + li $idx,16 + + lvx $rndkey0,0,$key2 + lvx $rndkey1,$idx,$key2 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $tweak,$tweak,$rndkey0 + lvx $rndkey0,$idx,$key2 + addi $idx,$idx,16 + mtctr $rounds + +Ltweak_xts_dec: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $tweak,$tweak,$rndkey1 + lvx $rndkey1,$idx,$key2 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipher $tweak,$tweak,$rndkey0 + lvx $rndkey0,$idx,$key2 + addi $idx,$idx,16 + bdnz Ltweak_xts_dec + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $tweak,$tweak,$rndkey1 + lvx $rndkey1,$idx,$key2 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipherlast $tweak,$tweak,$rndkey0 + + li $ivp,0 # don't chain the tweak + b Lxts_dec + +Lxts_dec_no_key2: + neg $idx,$len + andi. $idx,$idx,15 + add $len,$len,$idx # in "tweak chaining" + # mode only complete + # blocks are processed +Lxts_dec: + lvx $inptail,0,$inp + addi $inp,$inp,16 + + ?lvsl $keyperm,0,$key1 # prepare for unaligned key + lwz $rounds,240($key1) + srwi $rounds,$rounds,1 + subi $rounds,$rounds,1 + li $idx,16 + + vslb $eighty7,$seven,$seven # 0x808080..80 + vor $eighty7,$eighty7,$seven # 0x878787..87 + vspltisb $tmp,1 # 0x010101..01 + vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01 + + ${UCMP}i $len,96 + bge _aesp8_xts_decrypt6x + + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$tweak + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + mtctr $rounds + + ${UCMP}i $len,16 + blt Ltail_xts_dec + be?b Loop_xts_dec + +.align 5 +Loop_xts_dec: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vncipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + bdnz Loop_xts_dec + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + li $idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $rndkey0,$rndkey0,$tweak + vncipherlast $output,$inout,$rndkey0 + + le?vperm $tmp,$output,$output,$leperm + be?nop + le?stvx_u $tmp,0,$out + be?stvx_u $output,0,$out + addi $out,$out,16 + + subic. $len,$len,16 + beq Lxts_dec_done + + vmr $inout,$inptail + lvx $inptail,0,$inp + addi $inp,$inp,16 + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak,$tweak,$tmp + + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$tweak + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + + mtctr $rounds + ${UCMP}i $len,16 + bge Loop_xts_dec + +Ltail_xts_dec: + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak1,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak1,$tweak1,$tmp + + subi $inp,$inp,16 + add $inp,$inp,$len + + vxor $inout,$inout,$tweak # :-( + vxor $inout,$inout,$tweak1 # :-) + +Loop_xts_dec_short: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vncipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + bdnz Loop_xts_dec_short + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + li $idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $rndkey0,$rndkey0,$tweak1 + vncipherlast $output,$inout,$rndkey0 + + le?vperm $tmp,$output,$output,$leperm + be?nop + le?stvx_u $tmp,0,$out + be?stvx_u $output,0,$out + + vmr $inout,$inptail + lvx $inptail,0,$inp + #addi $inp,$inp,16 + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + + lvsr $inpperm,0,$len # $inpperm is no longer needed + vxor $inptail,$inptail,$inptail # $inptail is no longer needed + vspltisb $tmp,-1 + vperm $inptail,$inptail,$tmp,$inpperm + vsel $inout,$inout,$output,$inptail + + vxor $rndkey0,$rndkey0,$tweak + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + + subi r11,$out,1 + mtctr $len + li $len,16 +Loop_xts_dec_steal: + lbzu r0,1(r11) + stb r0,16(r11) + bdnz Loop_xts_dec_steal + + mtctr $rounds + b Loop_xts_dec # one more time... + +Lxts_dec_done: + ${UCMP}i $ivp,0 + beq Lxts_dec_ret + + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak,$tweak,$tmp + + le?vperm $tweak,$tweak,$tweak,$leperm + stvx_u $tweak,0,$ivp + +Lxts_dec_ret: + mtspr 256,r12 # restore vrsave + li r3,0 + blr + .long 0 + .byte 0,12,0x04,0,0x80,6,6,0 + .long 0 +.size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt +___ +######################################################################### +{{ # Optimized XTS procedures # +my $key_=$key2; +my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31)); + $x00=0 if ($flavour =~ /osx/); +my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5)); +my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16)); +my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22)); +my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys + # v26-v31 last 6 round keys +my ($keyperm)=($out0); # aliases with "caller", redundant assignment +my $taillen=$x70; + +$code.=<<___; +.align 5 +_aesp8_xts_encrypt6x: + $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) + mflr r11 + li r7,`$FRAME+8*16+15` + li r3,`$FRAME+8*16+31` + $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp) + stvx v20,r7,$sp # ABI says so + addi r7,r7,32 + stvx v21,r3,$sp + addi r3,r3,32 + stvx v22,r7,$sp + addi r7,r7,32 + stvx v23,r3,$sp + addi r3,r3,32 + stvx v24,r7,$sp + addi r7,r7,32 + stvx v25,r3,$sp + addi r3,r3,32 + stvx v26,r7,$sp + addi r7,r7,32 + stvx v27,r3,$sp + addi r3,r3,32 + stvx v28,r7,$sp + addi r7,r7,32 + stvx v29,r3,$sp + addi r3,r3,32 + stvx v30,r7,$sp + stvx v31,r3,$sp + li r0,-1 + stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave + li $x10,0x10 + $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) + li $x20,0x20 + $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) + li $x30,0x30 + $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) + li $x40,0x40 + $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) + li $x50,0x50 + $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) + li $x60,0x60 + $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) + li $x70,0x70 + mtspr 256,r0 + + subi $rounds,$rounds,3 # -4 in total + + lvx $rndkey0,$x00,$key1 # load key schedule + lvx v30,$x10,$key1 + addi $key1,$key1,0x20 + lvx v31,$x00,$key1 + ?vperm $rndkey0,$rndkey0,v30,$keyperm + addi $key_,$sp,$FRAME+15 + mtctr $rounds + +Load_xts_enc_key: + ?vperm v24,v30,v31,$keyperm + lvx v30,$x10,$key1 + addi $key1,$key1,0x20 + stvx v24,$x00,$key_ # off-load round[1] + ?vperm v25,v31,v30,$keyperm + lvx v31,$x00,$key1 + stvx v25,$x10,$key_ # off-load round[2] + addi $key_,$key_,0x20 + bdnz Load_xts_enc_key + + lvx v26,$x10,$key1 + ?vperm v24,v30,v31,$keyperm + lvx v27,$x20,$key1 + stvx v24,$x00,$key_ # off-load round[3] + ?vperm v25,v31,v26,$keyperm + lvx v28,$x30,$key1 + stvx v25,$x10,$key_ # off-load round[4] + addi $key_,$sp,$FRAME+15 # rewind $key_ + ?vperm v26,v26,v27,$keyperm + lvx v29,$x40,$key1 + ?vperm v27,v27,v28,$keyperm + lvx v30,$x50,$key1 + ?vperm v28,v28,v29,$keyperm + lvx v31,$x60,$key1 + ?vperm v29,v29,v30,$keyperm + lvx $twk5,$x70,$key1 # borrow $twk5 + ?vperm v30,v30,v31,$keyperm + lvx v24,$x00,$key_ # pre-load round[1] + ?vperm v31,v31,$twk5,$keyperm + lvx v25,$x10,$key_ # pre-load round[2] + + vperm $in0,$inout,$inptail,$inpperm + subi $inp,$inp,31 # undo "caller" + vxor $twk0,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $out0,$in0,$twk0 + vxor $tweak,$tweak,$tmp + + lvx_u $in1,$x10,$inp + vxor $twk1,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in1,$in1,$in1,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out1,$in1,$twk1 + vxor $tweak,$tweak,$tmp + + lvx_u $in2,$x20,$inp + andi. $taillen,$len,15 + vxor $twk2,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in2,$in2,$in2,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out2,$in2,$twk2 + vxor $tweak,$tweak,$tmp + + lvx_u $in3,$x30,$inp + sub $len,$len,$taillen + vxor $twk3,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in3,$in3,$in3,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out3,$in3,$twk3 + vxor $tweak,$tweak,$tmp + + lvx_u $in4,$x40,$inp + subi $len,$len,0x60 + vxor $twk4,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in4,$in4,$in4,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out4,$in4,$twk4 + vxor $tweak,$tweak,$tmp + + lvx_u $in5,$x50,$inp + addi $inp,$inp,0x60 + vxor $twk5,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in5,$in5,$in5,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out5,$in5,$twk5 + vxor $tweak,$tweak,$tmp + + vxor v31,v31,$rndkey0 + mtctr $rounds + b Loop_xts_enc6x + +.align 5 +Loop_xts_enc6x: + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vcipher $out4,$out4,v24 + vcipher $out5,$out5,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vcipher $out4,$out4,v25 + vcipher $out5,$out5,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_xts_enc6x + + subic $len,$len,96 # $len-=96 + vxor $in0,$twk0,v31 # xor with last round key + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk0,$tweak,$rndkey0 + vaddubm $tweak,$tweak,$tweak + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vsldoi $tmp,$tmp,$tmp,15 + vcipher $out4,$out4,v24 + vcipher $out5,$out5,v24 + + subfe. r0,r0,r0 # borrow?-1:0 + vand $tmp,$tmp,$eighty7 + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vxor $tweak,$tweak,$tmp + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vxor $in1,$twk1,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk1,$tweak,$rndkey0 + vcipher $out4,$out4,v25 + vcipher $out5,$out5,v25 + + and r0,r0,$len + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vcipher $out0,$out0,v26 + vcipher $out1,$out1,v26 + vand $tmp,$tmp,$eighty7 + vcipher $out2,$out2,v26 + vcipher $out3,$out3,v26 + vxor $tweak,$tweak,$tmp + vcipher $out4,$out4,v26 + vcipher $out5,$out5,v26 + + add $inp,$inp,r0 # $inp is adjusted in such + # way that at exit from the + # loop inX-in5 are loaded + # with last "words" + vxor $in2,$twk2,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk2,$tweak,$rndkey0 + vaddubm $tweak,$tweak,$tweak + vcipher $out0,$out0,v27 + vcipher $out1,$out1,v27 + vsldoi $tmp,$tmp,$tmp,15 + vcipher $out2,$out2,v27 + vcipher $out3,$out3,v27 + vand $tmp,$tmp,$eighty7 + vcipher $out4,$out4,v27 + vcipher $out5,$out5,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vxor $tweak,$tweak,$tmp + vcipher $out0,$out0,v28 + vcipher $out1,$out1,v28 + vxor $in3,$twk3,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk3,$tweak,$rndkey0 + vcipher $out2,$out2,v28 + vcipher $out3,$out3,v28 + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vcipher $out4,$out4,v28 + vcipher $out5,$out5,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + vand $tmp,$tmp,$eighty7 + + vcipher $out0,$out0,v29 + vcipher $out1,$out1,v29 + vxor $tweak,$tweak,$tmp + vcipher $out2,$out2,v29 + vcipher $out3,$out3,v29 + vxor $in4,$twk4,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk4,$tweak,$rndkey0 + vcipher $out4,$out4,v29 + vcipher $out5,$out5,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + + vcipher $out0,$out0,v30 + vcipher $out1,$out1,v30 + vand $tmp,$tmp,$eighty7 + vcipher $out2,$out2,v30 + vcipher $out3,$out3,v30 + vxor $tweak,$tweak,$tmp + vcipher $out4,$out4,v30 + vcipher $out5,$out5,v30 + vxor $in5,$twk5,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk5,$tweak,$rndkey0 + + vcipherlast $out0,$out0,$in0 + lvx_u $in0,$x00,$inp # load next input block + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vcipherlast $out1,$out1,$in1 + lvx_u $in1,$x10,$inp + vcipherlast $out2,$out2,$in2 + le?vperm $in0,$in0,$in0,$leperm + lvx_u $in2,$x20,$inp + vand $tmp,$tmp,$eighty7 + vcipherlast $out3,$out3,$in3 + le?vperm $in1,$in1,$in1,$leperm + lvx_u $in3,$x30,$inp + vcipherlast $out4,$out4,$in4 + le?vperm $in2,$in2,$in2,$leperm + lvx_u $in4,$x40,$inp + vxor $tweak,$tweak,$tmp + vcipherlast $tmp,$out5,$in5 # last block might be needed + # in stealing mode + le?vperm $in3,$in3,$in3,$leperm + lvx_u $in5,$x50,$inp + addi $inp,$inp,0x60 + le?vperm $in4,$in4,$in4,$leperm + le?vperm $in5,$in5,$in5,$leperm + + le?vperm $out0,$out0,$out0,$leperm + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk0 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + vxor $out1,$in1,$twk1 + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + vxor $out2,$in2,$twk2 + le?vperm $out4,$out4,$out4,$leperm + stvx_u $out3,$x30,$out + vxor $out3,$in3,$twk3 + le?vperm $out5,$tmp,$tmp,$leperm + stvx_u $out4,$x40,$out + vxor $out4,$in4,$twk4 + le?stvx_u $out5,$x50,$out + be?stvx_u $tmp, $x50,$out + vxor $out5,$in5,$twk5 + addi $out,$out,0x60 + + mtctr $rounds + beq Loop_xts_enc6x # did $len-=96 borrow? + + addic. $len,$len,0x60 + beq Lxts_enc6x_zero + cmpwi $len,0x20 + blt Lxts_enc6x_one + nop + beq Lxts_enc6x_two + cmpwi $len,0x40 + blt Lxts_enc6x_three + nop + beq Lxts_enc6x_four + +Lxts_enc6x_five: + vxor $out0,$in1,$twk0 + vxor $out1,$in2,$twk1 + vxor $out2,$in3,$twk2 + vxor $out3,$in4,$twk3 + vxor $out4,$in5,$twk4 + + bl _aesp8_xts_enc5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk5 # unused tweak + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + vxor $tmp,$out4,$twk5 # last block prep for stealing + le?vperm $out4,$out4,$out4,$leperm + stvx_u $out3,$x30,$out + stvx_u $out4,$x40,$out + addi $out,$out,0x50 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_four: + vxor $out0,$in2,$twk0 + vxor $out1,$in3,$twk1 + vxor $out2,$in4,$twk2 + vxor $out3,$in5,$twk3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_enc5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk4 # unused tweak + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + vxor $tmp,$out3,$twk4 # last block prep for stealing + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + stvx_u $out3,$x30,$out + addi $out,$out,0x40 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_three: + vxor $out0,$in3,$twk0 + vxor $out1,$in4,$twk1 + vxor $out2,$in5,$twk2 + vxor $out3,$out3,$out3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_enc5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk3 # unused tweak + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $tmp,$out2,$twk3 # last block prep for stealing + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + stvx_u $out2,$x20,$out + addi $out,$out,0x30 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_two: + vxor $out0,$in4,$twk0 + vxor $out1,$in5,$twk1 + vxor $out2,$out2,$out2 + vxor $out3,$out3,$out3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_enc5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk2 # unused tweak + vxor $tmp,$out1,$twk2 # last block prep for stealing + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + stvx_u $out1,$x10,$out + addi $out,$out,0x20 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_one: + vxor $out0,$in5,$twk0 + nop +Loop_xts_enc1x: + vcipher $out0,$out0,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vcipher $out0,$out0,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_xts_enc1x + + add $inp,$inp,$taillen + cmpwi $taillen,0 + vcipher $out0,$out0,v24 + + subi $inp,$inp,16 + vcipher $out0,$out0,v25 + + lvsr $inpperm,0,$taillen + vcipher $out0,$out0,v26 + + lvx_u $in0,0,$inp + vcipher $out0,$out0,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vcipher $out0,$out0,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + + vcipher $out0,$out0,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $twk0,$twk0,v31 + + le?vperm $in0,$in0,$in0,$leperm + vcipher $out0,$out0,v30 + + vperm $in0,$in0,$in0,$inpperm + vcipherlast $out0,$out0,$twk0 + + vmr $twk0,$twk1 # unused tweak + vxor $tmp,$out0,$twk1 # last block prep for stealing + le?vperm $out0,$out0,$out0,$leperm + stvx_u $out0,$x00,$out # store output + addi $out,$out,0x10 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_zero: + cmpwi $taillen,0 + beq Lxts_enc6x_done + + add $inp,$inp,$taillen + subi $inp,$inp,16 + lvx_u $in0,0,$inp + lvsr $inpperm,0,$taillen # $in5 is no more + le?vperm $in0,$in0,$in0,$leperm + vperm $in0,$in0,$in0,$inpperm + vxor $tmp,$tmp,$twk0 +Lxts_enc6x_steal: + vxor $in0,$in0,$twk0 + vxor $out0,$out0,$out0 + vspltisb $out1,-1 + vperm $out0,$out0,$out1,$inpperm + vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember? + + subi r30,$out,17 + subi $out,$out,16 + mtctr $taillen +Loop_xts_enc6x_steal: + lbzu r0,1(r30) + stb r0,16(r30) + bdnz Loop_xts_enc6x_steal + + li $taillen,0 + mtctr $rounds + b Loop_xts_enc1x # one more time... + +.align 4 +Lxts_enc6x_done: + ${UCMP}i $ivp,0 + beq Lxts_enc6x_ret + + vxor $tweak,$twk0,$rndkey0 + le?vperm $tweak,$tweak,$tweak,$leperm + stvx_u $tweak,0,$ivp + +Lxts_enc6x_ret: + mtlr r11 + li r10,`$FRAME+15` + li r11,`$FRAME+31` + stvx $seven,r10,$sp # wipe copies of round keys + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + + mtspr 256,$vrsave + lvx v20,r10,$sp # ABI says so + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) + $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) + $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) + $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) + $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) + $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) + addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` + blr + .long 0 + .byte 0,12,0x04,1,0x80,6,6,0 + .long 0 + +.align 5 +_aesp8_xts_enc5x: + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vcipher $out4,$out4,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vcipher $out4,$out4,v25 + lvx v25,$x10,$key_ # round[4] + bdnz _aesp8_xts_enc5x + + add $inp,$inp,$taillen + cmpwi $taillen,0 + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vcipher $out4,$out4,v24 + + subi $inp,$inp,16 + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vcipher $out4,$out4,v25 + vxor $twk0,$twk0,v31 + + vcipher $out0,$out0,v26 + lvsr $inpperm,r0,$taillen # $in5 is no more + vcipher $out1,$out1,v26 + vcipher $out2,$out2,v26 + vcipher $out3,$out3,v26 + vcipher $out4,$out4,v26 + vxor $in1,$twk1,v31 + + vcipher $out0,$out0,v27 + lvx_u $in0,0,$inp + vcipher $out1,$out1,v27 + vcipher $out2,$out2,v27 + vcipher $out3,$out3,v27 + vcipher $out4,$out4,v27 + vxor $in2,$twk2,v31 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vcipher $out0,$out0,v28 + vcipher $out1,$out1,v28 + vcipher $out2,$out2,v28 + vcipher $out3,$out3,v28 + vcipher $out4,$out4,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + vxor $in3,$twk3,v31 + + vcipher $out0,$out0,v29 + le?vperm $in0,$in0,$in0,$leperm + vcipher $out1,$out1,v29 + vcipher $out2,$out2,v29 + vcipher $out3,$out3,v29 + vcipher $out4,$out4,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $in4,$twk4,v31 + + vcipher $out0,$out0,v30 + vperm $in0,$in0,$in0,$inpperm + vcipher $out1,$out1,v30 + vcipher $out2,$out2,v30 + vcipher $out3,$out3,v30 + vcipher $out4,$out4,v30 + + vcipherlast $out0,$out0,$twk0 + vcipherlast $out1,$out1,$in1 + vcipherlast $out2,$out2,$in2 + vcipherlast $out3,$out3,$in3 + vcipherlast $out4,$out4,$in4 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +.align 5 +_aesp8_xts_decrypt6x: + $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) + mflr r11 + li r7,`$FRAME+8*16+15` + li r3,`$FRAME+8*16+31` + $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp) + stvx v20,r7,$sp # ABI says so + addi r7,r7,32 + stvx v21,r3,$sp + addi r3,r3,32 + stvx v22,r7,$sp + addi r7,r7,32 + stvx v23,r3,$sp + addi r3,r3,32 + stvx v24,r7,$sp + addi r7,r7,32 + stvx v25,r3,$sp + addi r3,r3,32 + stvx v26,r7,$sp + addi r7,r7,32 + stvx v27,r3,$sp + addi r3,r3,32 + stvx v28,r7,$sp + addi r7,r7,32 + stvx v29,r3,$sp + addi r3,r3,32 + stvx v30,r7,$sp + stvx v31,r3,$sp + li r0,-1 + stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave + li $x10,0x10 + $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) + li $x20,0x20 + $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) + li $x30,0x30 + $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) + li $x40,0x40 + $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) + li $x50,0x50 + $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) + li $x60,0x60 + $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) + li $x70,0x70 + mtspr 256,r0 + + subi $rounds,$rounds,3 # -4 in total + + lvx $rndkey0,$x00,$key1 # load key schedule + lvx v30,$x10,$key1 + addi $key1,$key1,0x20 + lvx v31,$x00,$key1 + ?vperm $rndkey0,$rndkey0,v30,$keyperm + addi $key_,$sp,$FRAME+15 + mtctr $rounds + +Load_xts_dec_key: + ?vperm v24,v30,v31,$keyperm + lvx v30,$x10,$key1 + addi $key1,$key1,0x20 + stvx v24,$x00,$key_ # off-load round[1] + ?vperm v25,v31,v30,$keyperm + lvx v31,$x00,$key1 + stvx v25,$x10,$key_ # off-load round[2] + addi $key_,$key_,0x20 + bdnz Load_xts_dec_key + + lvx v26,$x10,$key1 + ?vperm v24,v30,v31,$keyperm + lvx v27,$x20,$key1 + stvx v24,$x00,$key_ # off-load round[3] + ?vperm v25,v31,v26,$keyperm + lvx v28,$x30,$key1 + stvx v25,$x10,$key_ # off-load round[4] + addi $key_,$sp,$FRAME+15 # rewind $key_ + ?vperm v26,v26,v27,$keyperm + lvx v29,$x40,$key1 + ?vperm v27,v27,v28,$keyperm + lvx v30,$x50,$key1 + ?vperm v28,v28,v29,$keyperm + lvx v31,$x60,$key1 + ?vperm v29,v29,v30,$keyperm + lvx $twk5,$x70,$key1 # borrow $twk5 + ?vperm v30,v30,v31,$keyperm + lvx v24,$x00,$key_ # pre-load round[1] + ?vperm v31,v31,$twk5,$keyperm + lvx v25,$x10,$key_ # pre-load round[2] + + vperm $in0,$inout,$inptail,$inpperm + subi $inp,$inp,31 # undo "caller" + vxor $twk0,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $out0,$in0,$twk0 + vxor $tweak,$tweak,$tmp + + lvx_u $in1,$x10,$inp + vxor $twk1,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in1,$in1,$in1,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out1,$in1,$twk1 + vxor $tweak,$tweak,$tmp + + lvx_u $in2,$x20,$inp + andi. $taillen,$len,15 + vxor $twk2,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in2,$in2,$in2,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out2,$in2,$twk2 + vxor $tweak,$tweak,$tmp + + lvx_u $in3,$x30,$inp + sub $len,$len,$taillen + vxor $twk3,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in3,$in3,$in3,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out3,$in3,$twk3 + vxor $tweak,$tweak,$tmp + + lvx_u $in4,$x40,$inp + subi $len,$len,0x60 + vxor $twk4,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in4,$in4,$in4,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out4,$in4,$twk4 + vxor $tweak,$tweak,$tmp + + lvx_u $in5,$x50,$inp + addi $inp,$inp,0x60 + vxor $twk5,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in5,$in5,$in5,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out5,$in5,$twk5 + vxor $tweak,$tweak,$tmp + + vxor v31,v31,$rndkey0 + mtctr $rounds + b Loop_xts_dec6x + +.align 5 +Loop_xts_dec6x: + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + vncipher $out5,$out5,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + vncipher $out5,$out5,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_xts_dec6x + + subic $len,$len,96 # $len-=96 + vxor $in0,$twk0,v31 # xor with last round key + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk0,$tweak,$rndkey0 + vaddubm $tweak,$tweak,$tweak + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vsldoi $tmp,$tmp,$tmp,15 + vncipher $out4,$out4,v24 + vncipher $out5,$out5,v24 + + subfe. r0,r0,r0 # borrow?-1:0 + vand $tmp,$tmp,$eighty7 + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vxor $tweak,$tweak,$tmp + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vxor $in1,$twk1,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk1,$tweak,$rndkey0 + vncipher $out4,$out4,v25 + vncipher $out5,$out5,v25 + + and r0,r0,$len + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vncipher $out0,$out0,v26 + vncipher $out1,$out1,v26 + vand $tmp,$tmp,$eighty7 + vncipher $out2,$out2,v26 + vncipher $out3,$out3,v26 + vxor $tweak,$tweak,$tmp + vncipher $out4,$out4,v26 + vncipher $out5,$out5,v26 + + add $inp,$inp,r0 # $inp is adjusted in such + # way that at exit from the + # loop inX-in5 are loaded + # with last "words" + vxor $in2,$twk2,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk2,$tweak,$rndkey0 + vaddubm $tweak,$tweak,$tweak + vncipher $out0,$out0,v27 + vncipher $out1,$out1,v27 + vsldoi $tmp,$tmp,$tmp,15 + vncipher $out2,$out2,v27 + vncipher $out3,$out3,v27 + vand $tmp,$tmp,$eighty7 + vncipher $out4,$out4,v27 + vncipher $out5,$out5,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vxor $tweak,$tweak,$tmp + vncipher $out0,$out0,v28 + vncipher $out1,$out1,v28 + vxor $in3,$twk3,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk3,$tweak,$rndkey0 + vncipher $out2,$out2,v28 + vncipher $out3,$out3,v28 + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vncipher $out4,$out4,v28 + vncipher $out5,$out5,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + vand $tmp,$tmp,$eighty7 + + vncipher $out0,$out0,v29 + vncipher $out1,$out1,v29 + vxor $tweak,$tweak,$tmp + vncipher $out2,$out2,v29 + vncipher $out3,$out3,v29 + vxor $in4,$twk4,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk4,$tweak,$rndkey0 + vncipher $out4,$out4,v29 + vncipher $out5,$out5,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + + vncipher $out0,$out0,v30 + vncipher $out1,$out1,v30 + vand $tmp,$tmp,$eighty7 + vncipher $out2,$out2,v30 + vncipher $out3,$out3,v30 + vxor $tweak,$tweak,$tmp + vncipher $out4,$out4,v30 + vncipher $out5,$out5,v30 + vxor $in5,$twk5,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk5,$tweak,$rndkey0 + + vncipherlast $out0,$out0,$in0 + lvx_u $in0,$x00,$inp # load next input block + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vncipherlast $out1,$out1,$in1 + lvx_u $in1,$x10,$inp + vncipherlast $out2,$out2,$in2 + le?vperm $in0,$in0,$in0,$leperm + lvx_u $in2,$x20,$inp + vand $tmp,$tmp,$eighty7 + vncipherlast $out3,$out3,$in3 + le?vperm $in1,$in1,$in1,$leperm + lvx_u $in3,$x30,$inp + vncipherlast $out4,$out4,$in4 + le?vperm $in2,$in2,$in2,$leperm + lvx_u $in4,$x40,$inp + vxor $tweak,$tweak,$tmp + vncipherlast $out5,$out5,$in5 + le?vperm $in3,$in3,$in3,$leperm + lvx_u $in5,$x50,$inp + addi $inp,$inp,0x60 + le?vperm $in4,$in4,$in4,$leperm + le?vperm $in5,$in5,$in5,$leperm + + le?vperm $out0,$out0,$out0,$leperm + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk0 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + vxor $out1,$in1,$twk1 + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + vxor $out2,$in2,$twk2 + le?vperm $out4,$out4,$out4,$leperm + stvx_u $out3,$x30,$out + vxor $out3,$in3,$twk3 + le?vperm $out5,$out5,$out5,$leperm + stvx_u $out4,$x40,$out + vxor $out4,$in4,$twk4 + stvx_u $out5,$x50,$out + vxor $out5,$in5,$twk5 + addi $out,$out,0x60 + + mtctr $rounds + beq Loop_xts_dec6x # did $len-=96 borrow? + + addic. $len,$len,0x60 + beq Lxts_dec6x_zero + cmpwi $len,0x20 + blt Lxts_dec6x_one + nop + beq Lxts_dec6x_two + cmpwi $len,0x40 + blt Lxts_dec6x_three + nop + beq Lxts_dec6x_four + +Lxts_dec6x_five: + vxor $out0,$in1,$twk0 + vxor $out1,$in2,$twk1 + vxor $out2,$in3,$twk2 + vxor $out3,$in4,$twk3 + vxor $out4,$in5,$twk4 + + bl _aesp8_xts_dec5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk5 # unused tweak + vxor $twk1,$tweak,$rndkey0 + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk1 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + le?vperm $out4,$out4,$out4,$leperm + stvx_u $out3,$x30,$out + stvx_u $out4,$x40,$out + addi $out,$out,0x50 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_four: + vxor $out0,$in2,$twk0 + vxor $out1,$in3,$twk1 + vxor $out2,$in4,$twk2 + vxor $out3,$in5,$twk3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_dec5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk4 # unused tweak + vmr $twk1,$twk5 + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk5 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + stvx_u $out3,$x30,$out + addi $out,$out,0x40 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_three: + vxor $out0,$in3,$twk0 + vxor $out1,$in4,$twk1 + vxor $out2,$in5,$twk2 + vxor $out3,$out3,$out3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_dec5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk3 # unused tweak + vmr $twk1,$twk4 + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk4 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + stvx_u $out2,$x20,$out + addi $out,$out,0x30 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_two: + vxor $out0,$in4,$twk0 + vxor $out1,$in5,$twk1 + vxor $out2,$out2,$out2 + vxor $out3,$out3,$out3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_dec5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk2 # unused tweak + vmr $twk1,$twk3 + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk3 + stvx_u $out1,$x10,$out + addi $out,$out,0x20 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_one: + vxor $out0,$in5,$twk0 + nop +Loop_xts_dec1x: + vncipher $out0,$out0,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_xts_dec1x + + subi r0,$taillen,1 + vncipher $out0,$out0,v24 + + andi. r0,r0,16 + cmpwi $taillen,0 + vncipher $out0,$out0,v25 + + sub $inp,$inp,r0 + vncipher $out0,$out0,v26 + + lvx_u $in0,0,$inp + vncipher $out0,$out0,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vncipher $out0,$out0,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + + vncipher $out0,$out0,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $twk0,$twk0,v31 + + le?vperm $in0,$in0,$in0,$leperm + vncipher $out0,$out0,v30 + + mtctr $rounds + vncipherlast $out0,$out0,$twk0 + + vmr $twk0,$twk1 # unused tweak + vmr $twk1,$twk2 + le?vperm $out0,$out0,$out0,$leperm + stvx_u $out0,$x00,$out # store output + addi $out,$out,0x10 + vxor $out0,$in0,$twk2 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_zero: + cmpwi $taillen,0 + beq Lxts_dec6x_done + + lvx_u $in0,0,$inp + le?vperm $in0,$in0,$in0,$leperm + vxor $out0,$in0,$twk1 +Lxts_dec6x_steal: + vncipher $out0,$out0,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Lxts_dec6x_steal + + add $inp,$inp,$taillen + vncipher $out0,$out0,v24 + + cmpwi $taillen,0 + vncipher $out0,$out0,v25 + + lvx_u $in0,0,$inp + vncipher $out0,$out0,v26 + + lvsr $inpperm,0,$taillen # $in5 is no more + vncipher $out0,$out0,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vncipher $out0,$out0,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + + vncipher $out0,$out0,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $twk1,$twk1,v31 + + le?vperm $in0,$in0,$in0,$leperm + vncipher $out0,$out0,v30 + + vperm $in0,$in0,$in0,$inpperm + vncipherlast $tmp,$out0,$twk1 + + le?vperm $out0,$tmp,$tmp,$leperm + le?stvx_u $out0,0,$out + be?stvx_u $tmp,0,$out + + vxor $out0,$out0,$out0 + vspltisb $out1,-1 + vperm $out0,$out0,$out1,$inpperm + vsel $out0,$in0,$tmp,$out0 + vxor $out0,$out0,$twk0 + + subi r30,$out,1 + mtctr $taillen +Loop_xts_dec6x_steal: + lbzu r0,1(r30) + stb r0,16(r30) + bdnz Loop_xts_dec6x_steal + + li $taillen,0 + mtctr $rounds + b Loop_xts_dec1x # one more time... + +.align 4 +Lxts_dec6x_done: + ${UCMP}i $ivp,0 + beq Lxts_dec6x_ret + + vxor $tweak,$twk0,$rndkey0 + le?vperm $tweak,$tweak,$tweak,$leperm + stvx_u $tweak,0,$ivp + +Lxts_dec6x_ret: + mtlr r11 + li r10,`$FRAME+15` + li r11,`$FRAME+31` + stvx $seven,r10,$sp # wipe copies of round keys + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + + mtspr 256,$vrsave + lvx v20,r10,$sp # ABI says so + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) + $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) + $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) + $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) + $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) + $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) + addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` + blr + .long 0 + .byte 0,12,0x04,1,0x80,6,6,0 + .long 0 + +.align 5 +_aesp8_xts_dec5x: + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + lvx v25,$x10,$key_ # round[4] + bdnz _aesp8_xts_dec5x + + subi r0,$taillen,1 + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + + andi. r0,r0,16 + cmpwi $taillen,0 + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + vxor $twk0,$twk0,v31 + + sub $inp,$inp,r0 + vncipher $out0,$out0,v26 + vncipher $out1,$out1,v26 + vncipher $out2,$out2,v26 + vncipher $out3,$out3,v26 + vncipher $out4,$out4,v26 + vxor $in1,$twk1,v31 + + vncipher $out0,$out0,v27 + lvx_u $in0,0,$inp + vncipher $out1,$out1,v27 + vncipher $out2,$out2,v27 + vncipher $out3,$out3,v27 + vncipher $out4,$out4,v27 + vxor $in2,$twk2,v31 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vncipher $out0,$out0,v28 + vncipher $out1,$out1,v28 + vncipher $out2,$out2,v28 + vncipher $out3,$out3,v28 + vncipher $out4,$out4,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + vxor $in3,$twk3,v31 + + vncipher $out0,$out0,v29 + le?vperm $in0,$in0,$in0,$leperm + vncipher $out1,$out1,v29 + vncipher $out2,$out2,v29 + vncipher $out3,$out3,v29 + vncipher $out4,$out4,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $in4,$twk4,v31 + + vncipher $out0,$out0,v30 + vncipher $out1,$out1,v30 + vncipher $out2,$out2,v30 + vncipher $out3,$out3,v30 + vncipher $out4,$out4,v30 + + vncipherlast $out0,$out0,$twk0 + vncipherlast $out1,$out1,$in1 + vncipherlast $out2,$out2,$in2 + vncipherlast $out3,$out3,$in3 + vncipherlast $out4,$out4,$in4 + mtctr $rounds + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 +___ +}} }}} + my $consts=1; foreach(split("\n",$code)) { s/\`([^\`]*)\`/eval($1)/geo; @@ -1898,7 +3757,7 @@ foreach(split("\n",$code)) { if ($flavour =~ /le$/o) { SWITCH: for($conv) { /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; }; - /\?rev/ && do { @bytes=reverse(@bytes); last; }; + /\?rev/ && do { @bytes=reverse(@bytes); last; }; } } diff --git a/drivers/crypto/vmx/ppc-xlate.pl b/drivers/crypto/vmx/ppc-xlate.pl index 9f4994cabcc7..b18e67d0e065 100644 --- a/drivers/crypto/vmx/ppc-xlate.pl +++ b/drivers/crypto/vmx/ppc-xlate.pl @@ -141,7 +141,7 @@ my $vmr = sub { # Some ABIs specify vrsave, special-purpose register #256, as reserved # for system use. -my $no_vrsave = ($flavour =~ /aix|linux64le/); +my $no_vrsave = ($flavour =~ /linux-ppc64le/); my $mtspr = sub { my ($f,$idx,$ra) = @_; if ($idx == 256 && $no_vrsave) { diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c index e163d5770438..f688c32fbcc7 100644 --- a/drivers/crypto/vmx/vmx.c +++ b/drivers/crypto/vmx/vmx.c @@ -31,10 +31,12 @@ extern struct shash_alg p8_ghash_alg; extern struct crypto_alg p8_aes_alg; extern struct crypto_alg p8_aes_cbc_alg; extern struct crypto_alg p8_aes_ctr_alg; +extern struct crypto_alg p8_aes_xts_alg; static struct crypto_alg *algs[] = { &p8_aes_alg, &p8_aes_cbc_alg, &p8_aes_ctr_alg, + &p8_aes_xts_alg, NULL, }; diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig index 78dac0e9da11..a5be56ec57f2 100644 --- a/drivers/devfreq/Kconfig +++ b/drivers/devfreq/Kconfig @@ -75,7 +75,7 @@ config DEVFREQ_GOV_PASSIVE comment "DEVFREQ Drivers" config ARM_EXYNOS_BUS_DEVFREQ - bool "ARM EXYNOS Generic Memory Bus DEVFREQ Driver" + tristate "ARM EXYNOS Generic Memory Bus DEVFREQ Driver" depends on ARCH_EXYNOS select DEVFREQ_GOV_SIMPLE_ONDEMAND select DEVFREQ_GOV_PASSIVE diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c index 39b048eda2ce..9aea2c7ecbe6 100644 --- a/drivers/devfreq/devfreq-event.c +++ b/drivers/devfreq/devfreq-event.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -481,13 +481,3 @@ static int __init devfreq_event_init(void) return 0; } subsys_initcall(devfreq_event_init); - -static void __exit devfreq_event_exit(void) -{ - class_destroy(devfreq_event_class); -} -module_exit(devfreq_event_exit); - -MODULE_AUTHOR("Chanwoo Choi "); -MODULE_DESCRIPTION("DEVFREQ-Event class support"); -MODULE_LICENSE("GPL"); diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index 1d6c803804d5..478006b7764a 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -268,8 +268,11 @@ int update_devfreq(struct devfreq *devfreq) devfreq_notify_transition(devfreq, &freqs, DEVFREQ_PRECHANGE); err = devfreq->profile->target(devfreq->dev.parent, &freq, flags); - if (err) + if (err) { + freqs.new = cur_freq; + devfreq_notify_transition(devfreq, &freqs, DEVFREQ_POSTCHANGE); return err; + } freqs.new = freq; devfreq_notify_transition(devfreq, &freqs, DEVFREQ_POSTCHANGE); @@ -552,6 +555,7 @@ struct devfreq *devfreq_add_device(struct device *dev, devfreq->profile = profile; strncpy(devfreq->governor_name, governor_name, DEVFREQ_NAME_LEN); devfreq->previous_freq = profile->initial_freq; + devfreq->last_status.current_frequency = profile->initial_freq; devfreq->data = data; devfreq->nb.notifier_call = devfreq_notifier_call; @@ -561,23 +565,22 @@ struct devfreq *devfreq_add_device(struct device *dev, mutex_lock(&devfreq->lock); } - devfreq->trans_table = devm_kzalloc(dev, sizeof(unsigned int) * - devfreq->profile->max_state * - devfreq->profile->max_state, - GFP_KERNEL); - devfreq->time_in_state = devm_kzalloc(dev, sizeof(unsigned long) * - devfreq->profile->max_state, - GFP_KERNEL); - devfreq->last_stat_updated = jiffies; - dev_set_name(&devfreq->dev, "%s", dev_name(dev)); err = device_register(&devfreq->dev); if (err) { - put_device(&devfreq->dev); mutex_unlock(&devfreq->lock); goto err_out; } + devfreq->trans_table = devm_kzalloc(&devfreq->dev, sizeof(unsigned int) * + devfreq->profile->max_state * + devfreq->profile->max_state, + GFP_KERNEL); + devfreq->time_in_state = devm_kzalloc(&devfreq->dev, sizeof(unsigned long) * + devfreq->profile->max_state, + GFP_KERNEL); + devfreq->last_stat_updated = jiffies; + srcu_init_notifier_head(&devfreq->transition_notifier_list); mutex_unlock(&devfreq->lock); @@ -603,7 +606,6 @@ struct devfreq *devfreq_add_device(struct device *dev, err_init: list_del(&devfreq->node); device_unregister(&devfreq->dev); - kfree(devfreq); err_out: return ERR_PTR(err); } @@ -621,7 +623,6 @@ int devfreq_remove_device(struct devfreq *devfreq) return -EINVAL; device_unregister(&devfreq->dev); - put_device(&devfreq->dev); return 0; } @@ -706,10 +707,12 @@ struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index) if (devfreq->dev.parent && devfreq->dev.parent->of_node == node) { mutex_unlock(&devfreq_list_lock); + of_node_put(node); return devfreq; } } mutex_unlock(&devfreq_list_lock); + of_node_put(node); return ERR_PTR(-EPROBE_DEFER); } @@ -1198,13 +1201,6 @@ static int __init devfreq_init(void) } subsys_initcall(devfreq_init); -static void __exit devfreq_exit(void) -{ - class_destroy(devfreq_class); - destroy_workqueue(devfreq_wq); -} -module_exit(devfreq_exit); - /* * The followings are helper functions for devfreq user device drivers with * OPP framework. @@ -1470,7 +1466,3 @@ void devm_devfreq_unregister_notifier(struct device *dev, devm_devfreq_dev_match, devfreq)); } EXPORT_SYMBOL(devm_devfreq_unregister_notifier); - -MODULE_AUTHOR("MyungJoo Ham "); -MODULE_DESCRIPTION("devfreq class support"); -MODULE_LICENSE("GPL"); diff --git a/drivers/devfreq/event/Kconfig b/drivers/devfreq/event/Kconfig index 1e8b4f469f38..eb6f74a2b6b9 100644 --- a/drivers/devfreq/event/Kconfig +++ b/drivers/devfreq/event/Kconfig @@ -14,7 +14,7 @@ menuconfig PM_DEVFREQ_EVENT if PM_DEVFREQ_EVENT config DEVFREQ_EVENT_EXYNOS_NOCP - bool "EXYNOS NoC (Network On Chip) Probe DEVFREQ event Driver" + tristate "EXYNOS NoC (Network On Chip) Probe DEVFREQ event Driver" depends on ARCH_EXYNOS select PM_OPP help @@ -22,7 +22,7 @@ config DEVFREQ_EVENT_EXYNOS_NOCP (Network on Chip) Probe counters to measure the bandwidth of AXI bus. config DEVFREQ_EVENT_EXYNOS_PPMU - bool "EXYNOS PPMU (Platform Performance Monitoring Unit) DEVFREQ event Driver" + tristate "EXYNOS PPMU (Platform Performance Monitoring Unit) DEVFREQ event Driver" depends on ARCH_EXYNOS select PM_OPP help diff --git a/drivers/devfreq/event/exynos-nocp.c b/drivers/devfreq/event/exynos-nocp.c index 6b6a5f310486..a5841403bde8 100644 --- a/drivers/devfreq/event/exynos-nocp.c +++ b/drivers/devfreq/event/exynos-nocp.c @@ -220,9 +220,6 @@ static int exynos_nocp_parse_dt(struct platform_device *pdev, /* Maps the memory mapped IO to control nocp register */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (IS_ERR(res)) - return PTR_ERR(res); - base = devm_ioremap_resource(dev, res); if (IS_ERR(base)) return PTR_ERR(base); diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c index f312485f1451..845bf25fb9fb 100644 --- a/drivers/devfreq/event/exynos-ppmu.c +++ b/drivers/devfreq/event/exynos-ppmu.c @@ -482,7 +482,8 @@ static int exynos_ppmu_probe(struct platform_device *pdev) if (!info->edev) { dev_err(&pdev->dev, "failed to allocate memory devfreq-event devices\n"); - return -ENOMEM; + ret = -ENOMEM; + goto err; } edev = info->edev; platform_set_drvdata(pdev, info); diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index 2363d0a189b7..29866f7e6d7e 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -383,7 +383,7 @@ err_clk: static int exynos_bus_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; + struct device_node *np = dev->of_node, *node; struct devfreq_dev_profile *profile; struct devfreq_simple_ondemand_data *ondemand_data; struct devfreq_passive_data *passive_data; @@ -407,7 +407,7 @@ static int exynos_bus_probe(struct platform_device *pdev) /* Parse the device-tree to get the resource information */ ret = exynos_bus_parse_of(np, bus); if (ret < 0) - goto err; + return ret; profile = devm_kzalloc(dev, sizeof(*profile), GFP_KERNEL); if (!profile) { @@ -415,10 +415,13 @@ static int exynos_bus_probe(struct platform_device *pdev) goto err; } - if (of_parse_phandle(dev->of_node, "devfreq", 0)) + node = of_parse_phandle(dev->of_node, "devfreq", 0); + if (node) { + of_node_put(node); goto passive; - else + } else { ret = exynos_bus_parent_parse_of(np, bus); + } if (ret < 0) goto err; diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c index f8c5cd53307c..c5f21efd6090 100644 --- a/drivers/dma/hsu/hsu.c +++ b/drivers/dma/hsu/hsu.c @@ -126,28 +126,33 @@ static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc) hsu_dma_start_channel(hsuc); } -static u32 hsu_dma_chan_get_sr(struct hsu_dma_chan *hsuc) -{ - unsigned long flags; - u32 sr; - - spin_lock_irqsave(&hsuc->vchan.lock, flags); - sr = hsu_chan_readl(hsuc, HSU_CH_SR); - spin_unlock_irqrestore(&hsuc->vchan.lock, flags); - - return sr & ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY); -} - -irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr) +/* + * hsu_dma_get_status() - get DMA channel status + * @chip: HSUART DMA chip + * @nr: DMA channel number + * @status: pointer for DMA Channel Status Register value + * + * Description: + * The function reads and clears the DMA Channel Status Register, checks + * if it was a timeout interrupt and returns a corresponding value. + * + * Caller should provide a valid pointer for the DMA Channel Status + * Register value that will be returned in @status. + * + * Return: + * 1 for DMA timeout status, 0 for other DMA status, or error code for + * invalid parameters or no interrupt pending. + */ +int hsu_dma_get_status(struct hsu_dma_chip *chip, unsigned short nr, + u32 *status) { struct hsu_dma_chan *hsuc; - struct hsu_dma_desc *desc; unsigned long flags; u32 sr; /* Sanity check */ if (nr >= chip->hsu->nr_channels) - return IRQ_NONE; + return -EINVAL; hsuc = &chip->hsu->chan[nr]; @@ -155,22 +160,65 @@ irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr) * No matter what situation, need read clear the IRQ status * There is a bug, see Errata 5, HSD 2900918 */ - sr = hsu_dma_chan_get_sr(hsuc); + spin_lock_irqsave(&hsuc->vchan.lock, flags); + sr = hsu_chan_readl(hsuc, HSU_CH_SR); + spin_unlock_irqrestore(&hsuc->vchan.lock, flags); + + /* Check if any interrupt is pending */ + sr &= ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY); if (!sr) - return IRQ_NONE; + return -EIO; /* Timeout IRQ, need wait some time, see Errata 2 */ if (sr & HSU_CH_SR_DESCTO_ANY) udelay(2); + /* + * At this point, at least one of Descriptor Time Out, Channel Error + * or Descriptor Done bits must be set. Clear the Descriptor Time Out + * bits and if sr is still non-zero, it must be channel error or + * descriptor done which are higher priority than timeout and handled + * in hsu_dma_do_irq(). Else, it must be a timeout. + */ sr &= ~HSU_CH_SR_DESCTO_ANY; - if (!sr) - return IRQ_HANDLED; + + *status = sr; + + return sr ? 0 : 1; +} +EXPORT_SYMBOL_GPL(hsu_dma_get_status); + +/* + * hsu_dma_do_irq() - DMA interrupt handler + * @chip: HSUART DMA chip + * @nr: DMA channel number + * @status: Channel Status Register value + * + * Description: + * This function handles Channel Error and Descriptor Done interrupts. + * This function should be called after determining that the DMA interrupt + * is not a normal timeout interrupt, ie. hsu_dma_get_status() returned 0. + * + * Return: + * IRQ_NONE for invalid channel number, IRQ_HANDLED otherwise. + */ +irqreturn_t hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr, + u32 status) +{ + struct hsu_dma_chan *hsuc; + struct hsu_dma_desc *desc; + unsigned long flags; + + /* Sanity check */ + if (nr >= chip->hsu->nr_channels) + return IRQ_NONE; + + hsuc = &chip->hsu->chan[nr]; spin_lock_irqsave(&hsuc->vchan.lock, flags); desc = hsuc->desc; if (desc) { - if (sr & HSU_CH_SR_CHE) { + if (status & HSU_CH_SR_CHE) { desc->status = DMA_ERROR; } else if (desc->active < desc->nents) { hsu_dma_start_channel(hsuc); @@ -184,7 +232,7 @@ irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr) return IRQ_HANDLED; } -EXPORT_SYMBOL_GPL(hsu_dma_irq); +EXPORT_SYMBOL_GPL(hsu_dma_do_irq); static struct hsu_dma_desc *hsu_dma_alloc_desc(unsigned int nents) { diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c index e2db76bd56d8..9916058531d9 100644 --- a/drivers/dma/hsu/pci.c +++ b/drivers/dma/hsu/pci.c @@ -27,13 +27,20 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev) { struct hsu_dma_chip *chip = dev; u32 dmaisr; + u32 status; unsigned short i; irqreturn_t ret = IRQ_NONE; + int err; dmaisr = readl(chip->regs + HSU_PCI_DMAISR); for (i = 0; i < chip->hsu->nr_channels; i++) { - if (dmaisr & 0x1) - ret |= hsu_dma_irq(chip, i); + if (dmaisr & 0x1) { + err = hsu_dma_get_status(chip, i, &status); + if (err > 0) + ret |= IRQ_HANDLED; + else if (err == 0) + ret |= hsu_dma_do_irq(chip, i, status); + } dmaisr >>= 1; } diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 6ca7474baf4a..d0c1dab9b435 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -391,6 +391,13 @@ config EDAC_ALTERA_OCRAM Support for error detection and correction on the Altera On-Chip RAM Memory for Altera SoCs. +config EDAC_ALTERA_ETHERNET + bool "Altera Ethernet FIFO ECC" + depends on EDAC_ALTERA=y + help + Support for error detection and correction on the + Altera Ethernet FIFO Memory for Altera SoCs. + config EDAC_SYNOPSYS tristate "Synopsys DDR Memory Controller" depends on EDAC_MM_EDAC && ARCH_ZYNQ diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 5b4d223d6d68..2398d0701f5b 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -19,12 +19,15 @@ #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -548,10 +551,10 @@ module_platform_driver(altr_edac_driver); * trigger testing are different for each memory. */ -const struct edac_device_prv_data ocramecc_data; -const struct edac_device_prv_data l2ecc_data; -const struct edac_device_prv_data a10_ocramecc_data; -const struct edac_device_prv_data a10_l2ecc_data; +static const struct edac_device_prv_data ocramecc_data; +static const struct edac_device_prv_data l2ecc_data; +static const struct edac_device_prv_data a10_ocramecc_data; +static const struct edac_device_prv_data a10_l2ecc_data; static irqreturn_t altr_edac_device_handler(int irq, void *dev_id) { @@ -686,11 +689,9 @@ static void altr_create_edacdev_dbgfs(struct edac_device_ctl_info *edac_dci, static const struct of_device_id altr_edac_device_of_match[] = { #ifdef CONFIG_EDAC_ALTERA_L2C { .compatible = "altr,socfpga-l2-ecc", .data = &l2ecc_data }, - { .compatible = "altr,socfpga-a10-l2-ecc", .data = &a10_l2ecc_data }, #endif #ifdef CONFIG_EDAC_ALTERA_OCRAM { .compatible = "altr,socfpga-ocram-ecc", .data = &ocramecc_data }, - { .compatible = "altr,socfpga-a10-ocram-ecc", .data = &a10_ocramecc_data }, #endif {}, }; @@ -825,16 +826,16 @@ static struct platform_driver altr_edac_device_driver = { }; module_platform_driver(altr_edac_device_driver); -/*********************** OCRAM EDAC Device Functions *********************/ +/******************* Arria10 Device ECC Shared Functions *****************/ -#ifdef CONFIG_EDAC_ALTERA_OCRAM /* * Test for memory's ECC dependencies upon entry because platform specific * startup should have initialized the memory and enabled the ECC. * Can't turn on ECC here because accessing un-initialized memory will * cause CE/UE errors possibly causing an ABORT. */ -static int altr_check_ecc_deps(struct altr_edac_device_dev *device) +static int __maybe_unused +altr_check_ecc_deps(struct altr_edac_device_dev *device) { void __iomem *base = device->base; const struct edac_device_prv_data *prv = device->data; @@ -848,6 +849,227 @@ static int altr_check_ecc_deps(struct altr_edac_device_dev *device) return -ENODEV; } +static irqreturn_t __maybe_unused altr_edac_a10_ecc_irq(int irq, void *dev_id) +{ + struct altr_edac_device_dev *dci = dev_id; + void __iomem *base = dci->base; + + if (irq == dci->sb_irq) { + writel(ALTR_A10_ECC_SERRPENA, + base + ALTR_A10_ECC_INTSTAT_OFST); + edac_device_handle_ce(dci->edac_dev, 0, 0, dci->edac_dev_name); + + return IRQ_HANDLED; + } else if (irq == dci->db_irq) { + writel(ALTR_A10_ECC_DERRPENA, + base + ALTR_A10_ECC_INTSTAT_OFST); + edac_device_handle_ue(dci->edac_dev, 0, 0, dci->edac_dev_name); + if (dci->data->panic) + panic("\nEDAC:ECC_DEVICE[Uncorrectable errors]\n"); + + return IRQ_HANDLED; + } + + WARN_ON(1); + + return IRQ_NONE; +} + +/******************* Arria10 Memory Buffer Functions *********************/ + +static inline int a10_get_irq_mask(struct device_node *np) +{ + int irq; + const u32 *handle = of_get_property(np, "interrupts", NULL); + + if (!handle) + return -ENODEV; + irq = be32_to_cpup(handle); + return irq; +} + +static inline void ecc_set_bits(u32 bit_mask, void __iomem *ioaddr) +{ + u32 value = readl(ioaddr); + + value |= bit_mask; + writel(value, ioaddr); +} + +static inline void ecc_clear_bits(u32 bit_mask, void __iomem *ioaddr) +{ + u32 value = readl(ioaddr); + + value &= ~bit_mask; + writel(value, ioaddr); +} + +static inline int ecc_test_bits(u32 bit_mask, void __iomem *ioaddr) +{ + u32 value = readl(ioaddr); + + return (value & bit_mask) ? 1 : 0; +} + +/* + * This function uses the memory initialization block in the Arria10 ECC + * controller to initialize/clear the entire memory data and ECC data. + */ +static int __maybe_unused altr_init_memory_port(void __iomem *ioaddr, int port) +{ + int limit = ALTR_A10_ECC_INIT_WATCHDOG_10US; + u32 init_mask, stat_mask, clear_mask; + int ret = 0; + + if (port) { + init_mask = ALTR_A10_ECC_INITB; + stat_mask = ALTR_A10_ECC_INITCOMPLETEB; + clear_mask = ALTR_A10_ECC_ERRPENB_MASK; + } else { + init_mask = ALTR_A10_ECC_INITA; + stat_mask = ALTR_A10_ECC_INITCOMPLETEA; + clear_mask = ALTR_A10_ECC_ERRPENA_MASK; + } + + ecc_set_bits(init_mask, (ioaddr + ALTR_A10_ECC_CTRL_OFST)); + while (limit--) { + if (ecc_test_bits(stat_mask, + (ioaddr + ALTR_A10_ECC_INITSTAT_OFST))) + break; + udelay(1); + } + if (limit < 0) + ret = -EBUSY; + + /* Clear any pending ECC interrupts */ + writel(clear_mask, (ioaddr + ALTR_A10_ECC_INTSTAT_OFST)); + + return ret; +} + +static __init int __maybe_unused +altr_init_a10_ecc_block(struct device_node *np, u32 irq_mask, + u32 ecc_ctrl_en_mask, bool dual_port) +{ + int ret = 0; + void __iomem *ecc_block_base; + struct regmap *ecc_mgr_map; + char *ecc_name; + struct device_node *np_eccmgr; + + ecc_name = (char *)np->name; + + /* Get the ECC Manager - parent of the device EDACs */ + np_eccmgr = of_get_parent(np); + ecc_mgr_map = syscon_regmap_lookup_by_phandle(np_eccmgr, + "altr,sysmgr-syscon"); + of_node_put(np_eccmgr); + if (IS_ERR(ecc_mgr_map)) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "Unable to get syscon altr,sysmgr-syscon\n"); + return -ENODEV; + } + + /* Map the ECC Block */ + ecc_block_base = of_iomap(np, 0); + if (!ecc_block_base) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "Unable to map %s ECC block\n", ecc_name); + return -ENODEV; + } + + /* Disable ECC */ + regmap_write(ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_SET_OFST, irq_mask); + writel(ALTR_A10_ECC_SERRINTEN, + (ecc_block_base + ALTR_A10_ECC_ERRINTENR_OFST)); + ecc_clear_bits(ecc_ctrl_en_mask, + (ecc_block_base + ALTR_A10_ECC_CTRL_OFST)); + /* Ensure all writes complete */ + wmb(); + /* Use HW initialization block to initialize memory for ECC */ + ret = altr_init_memory_port(ecc_block_base, 0); + if (ret) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "ECC: cannot init %s PORTA memory\n", ecc_name); + goto out; + } + + if (dual_port) { + ret = altr_init_memory_port(ecc_block_base, 1); + if (ret) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "ECC: cannot init %s PORTB memory\n", + ecc_name); + goto out; + } + } + + /* Interrupt mode set to every SBERR */ + regmap_write(ecc_mgr_map, ALTR_A10_ECC_INTMODE_OFST, + ALTR_A10_ECC_INTMODE); + /* Enable ECC */ + ecc_set_bits(ecc_ctrl_en_mask, (ecc_block_base + + ALTR_A10_ECC_CTRL_OFST)); + writel(ALTR_A10_ECC_SERRINTEN, + (ecc_block_base + ALTR_A10_ECC_ERRINTENS_OFST)); + regmap_write(ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_CLR_OFST, irq_mask); + /* Ensure all writes complete */ + wmb(); +out: + iounmap(ecc_block_base); + return ret; +} + +static int validate_parent_available(struct device_node *np); +static const struct of_device_id altr_edac_a10_device_of_match[]; +static int __init __maybe_unused altr_init_a10_ecc_device_type(char *compat) +{ + int irq; + struct device_node *child, *np = of_find_compatible_node(NULL, NULL, + "altr,socfpga-a10-ecc-manager"); + if (!np) { + edac_printk(KERN_ERR, EDAC_DEVICE, "ECC Manager not found\n"); + return -ENODEV; + } + + for_each_child_of_node(np, child) { + const struct of_device_id *pdev_id; + const struct edac_device_prv_data *prv; + + if (!of_device_is_available(child)) + continue; + if (!of_device_is_compatible(child, compat)) + continue; + + if (validate_parent_available(child)) + continue; + + irq = a10_get_irq_mask(child); + if (irq < 0) + continue; + + /* Get matching node and check for valid result */ + pdev_id = of_match_node(altr_edac_a10_device_of_match, child); + if (IS_ERR_OR_NULL(pdev_id)) + continue; + + /* Validate private data pointer before dereferencing */ + prv = pdev_id->data; + if (!prv) + continue; + + altr_init_a10_ecc_block(child, BIT(irq), + prv->ecc_enable_mask, 0); + } + + of_node_put(np); + return 0; +} + +/*********************** OCRAM EDAC Device Functions *********************/ + +#ifdef CONFIG_EDAC_ALTERA_OCRAM + static void *ocram_alloc_mem(size_t size, void **other) { struct device_node *np; @@ -882,25 +1104,7 @@ static void ocram_free_mem(void *p, size_t size, void *other) gen_pool_free((struct gen_pool *)other, (u32)p, size); } -static irqreturn_t altr_edac_a10_ecc_irq(struct altr_edac_device_dev *dci, - bool sberr) -{ - void __iomem *base = dci->base; - - if (sberr) { - writel(ALTR_A10_ECC_SERRPENA, - base + ALTR_A10_ECC_INTSTAT_OFST); - edac_device_handle_ce(dci->edac_dev, 0, 0, dci->edac_dev_name); - } else { - writel(ALTR_A10_ECC_DERRPENA, - base + ALTR_A10_ECC_INTSTAT_OFST); - edac_device_handle_ue(dci->edac_dev, 0, 0, dci->edac_dev_name); - panic("\nEDAC:ECC_DEVICE[Uncorrectable errors]\n"); - } - return IRQ_HANDLED; -} - -const struct edac_device_prv_data ocramecc_data = { +static const struct edac_device_prv_data ocramecc_data = { .setup = altr_check_ecc_deps, .ce_clear_mask = (ALTR_OCR_ECC_EN | ALTR_OCR_ECC_SERR), .ue_clear_mask = (ALTR_OCR_ECC_EN | ALTR_OCR_ECC_DERR), @@ -916,7 +1120,7 @@ const struct edac_device_prv_data ocramecc_data = { .inject_fops = &altr_edac_device_inject_fops, }; -const struct edac_device_prv_data a10_ocramecc_data = { +static const struct edac_device_prv_data a10_ocramecc_data = { .setup = altr_check_ecc_deps, .ce_clear_mask = ALTR_A10_ECC_SERRPENA, .ue_clear_mask = ALTR_A10_ECC_DERRPENA, @@ -929,6 +1133,12 @@ const struct edac_device_prv_data a10_ocramecc_data = { .set_err_ofst = ALTR_A10_ECC_INTTEST_OFST, .ecc_irq_handler = altr_edac_a10_ecc_irq, .inject_fops = &altr_edac_a10_device_inject_fops, + /* + * OCRAM panic on uncorrectable error because sleep/resume + * functions and FPGA contents are stored in OCRAM. Prefer + * a kernel panic over executing/loading corrupted data. + */ + .panic = true, }; #endif /* CONFIG_EDAC_ALTERA_OCRAM */ @@ -988,25 +1198,33 @@ static int altr_l2_check_deps(struct altr_edac_device_dev *device) return -ENODEV; } -static irqreturn_t altr_edac_a10_l2_irq(struct altr_edac_device_dev *dci, - bool sberr) +static irqreturn_t altr_edac_a10_l2_irq(int irq, void *dev_id) { - if (sberr) { + struct altr_edac_device_dev *dci = dev_id; + + if (irq == dci->sb_irq) { regmap_write(dci->edac->ecc_mgr_map, A10_SYSGMR_MPU_CLEAR_L2_ECC_OFST, A10_SYSGMR_MPU_CLEAR_L2_ECC_SB); edac_device_handle_ce(dci->edac_dev, 0, 0, dci->edac_dev_name); - } else { + + return IRQ_HANDLED; + } else if (irq == dci->db_irq) { regmap_write(dci->edac->ecc_mgr_map, A10_SYSGMR_MPU_CLEAR_L2_ECC_OFST, A10_SYSGMR_MPU_CLEAR_L2_ECC_MB); edac_device_handle_ue(dci->edac_dev, 0, 0, dci->edac_dev_name); panic("\nEDAC:ECC_DEVICE[Uncorrectable errors]\n"); + + return IRQ_HANDLED; } - return IRQ_HANDLED; + + WARN_ON(1); + + return IRQ_NONE; } -const struct edac_device_prv_data l2ecc_data = { +static const struct edac_device_prv_data l2ecc_data = { .setup = altr_l2_check_deps, .ce_clear_mask = 0, .ue_clear_mask = 0, @@ -1021,7 +1239,7 @@ const struct edac_device_prv_data l2ecc_data = { .inject_fops = &altr_edac_device_inject_fops, }; -const struct edac_device_prv_data a10_l2ecc_data = { +static const struct edac_device_prv_data a10_l2ecc_data = { .setup = altr_l2_check_deps, .ce_clear_mask = ALTR_A10_L2_ECC_SERR_CLR, .ue_clear_mask = ALTR_A10_L2_ECC_MERR_CLR, @@ -1040,7 +1258,49 @@ const struct edac_device_prv_data a10_l2ecc_data = { #endif /* CONFIG_EDAC_ALTERA_L2C */ +/********************* Ethernet Device Functions ********************/ + +#ifdef CONFIG_EDAC_ALTERA_ETHERNET + +static const struct edac_device_prv_data a10_enetecc_data = { + .setup = altr_check_ecc_deps, + .ce_clear_mask = ALTR_A10_ECC_SERRPENA, + .ue_clear_mask = ALTR_A10_ECC_DERRPENA, + .dbgfs_name = "altr_trigger", + .ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL, + .ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST, + .ce_set_mask = ALTR_A10_ECC_TSERRA, + .ue_set_mask = ALTR_A10_ECC_TDERRA, + .set_err_ofst = ALTR_A10_ECC_INTTEST_OFST, + .ecc_irq_handler = altr_edac_a10_ecc_irq, + .inject_fops = &altr_edac_a10_device_inject_fops, +}; + +static int __init socfpga_init_ethernet_ecc(void) +{ + return altr_init_a10_ecc_device_type("altr,socfpga-eth-mac-ecc"); +} + +early_initcall(socfpga_init_ethernet_ecc); + +#endif /* CONFIG_EDAC_ALTERA_ETHERNET */ + /********************* Arria10 EDAC Device Functions *************************/ +static const struct of_device_id altr_edac_a10_device_of_match[] = { +#ifdef CONFIG_EDAC_ALTERA_L2C + { .compatible = "altr,socfpga-a10-l2-ecc", .data = &a10_l2ecc_data }, +#endif +#ifdef CONFIG_EDAC_ALTERA_OCRAM + { .compatible = "altr,socfpga-a10-ocram-ecc", + .data = &a10_ocramecc_data }, +#endif +#ifdef CONFIG_EDAC_ALTERA_ETHERNET + { .compatible = "altr,socfpga-eth-mac-ecc", + .data = &a10_enetecc_data }, +#endif + {}, +}; +MODULE_DEVICE_TABLE(of, altr_edac_a10_device_of_match); /* * The Arria10 EDAC Device Functions differ from the Cyclone5/Arria5 @@ -1075,28 +1335,42 @@ static ssize_t altr_edac_a10_device_trig(struct file *file, return count; } -static irqreturn_t altr_edac_a10_irq_handler(int irq, void *dev_id) +static void altr_edac_a10_irq_handler(struct irq_desc *desc) { - irqreturn_t rc = IRQ_NONE; - struct altr_arria10_edac *edac = dev_id; - struct altr_edac_device_dev *dci; - int irq_status; - bool sberr = (irq == edac->sb_irq) ? 1 : 0; - int sm_offset = sberr ? A10_SYSMGR_ECC_INTSTAT_SERR_OFST : - A10_SYSMGR_ECC_INTSTAT_DERR_OFST; + int dberr, bit, sm_offset, irq_status; + struct altr_arria10_edac *edac = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + int irq = irq_desc_get_irq(desc); + + dberr = (irq == edac->db_irq) ? 1 : 0; + sm_offset = dberr ? A10_SYSMGR_ECC_INTSTAT_DERR_OFST : + A10_SYSMGR_ECC_INTSTAT_SERR_OFST; + + chained_irq_enter(chip, desc); regmap_read(edac->ecc_mgr_map, sm_offset, &irq_status); - if ((irq != edac->sb_irq) && (irq != edac->db_irq)) { - WARN_ON(1); - } else { - list_for_each_entry(dci, &edac->a10_ecc_devices, next) { - if (irq_status & dci->data->irq_status_mask) - rc = dci->data->ecc_irq_handler(dci, sberr); - } + for_each_set_bit(bit, (unsigned long *)&irq_status, 32) { + irq = irq_linear_revmap(edac->domain, dberr * 32 + bit); + if (irq) + generic_handle_irq(irq); } - return rc; + chained_irq_exit(chip, desc); +} + +static int validate_parent_available(struct device_node *np) +{ + struct device_node *parent; + int ret = 0; + + /* Ensure parent device is enabled if parent node exists */ + parent = of_parse_phandle(np, "altr,ecc-parent", 0); + if (parent && !of_device_is_available(parent)) + ret = -ENODEV; + + of_node_put(parent); + return ret; } static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, @@ -1111,7 +1385,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, const struct edac_device_prv_data *prv; /* Get matching node and check for valid result */ const struct of_device_id *pdev_id = - of_match_node(altr_edac_device_of_match, np); + of_match_node(altr_edac_a10_device_of_match, np); if (IS_ERR_OR_NULL(pdev_id)) return -ENODEV; @@ -1120,6 +1394,9 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, if (IS_ERR_OR_NULL(prv)) return -ENODEV; + if (validate_parent_available(np)) + return -ENODEV; + if (!devres_open_group(edac->dev, altr_edac_a10_device_add, GFP_KERNEL)) return -ENOMEM; @@ -1168,6 +1445,34 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } + altdev->sb_irq = irq_of_parse_and_map(np, 0); + if (!altdev->sb_irq) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Error allocating SBIRQ\n"); + rc = -ENODEV; + goto err_release_group1; + } + rc = devm_request_irq(edac->dev, altdev->sb_irq, + prv->ecc_irq_handler, + IRQF_SHARED, ecc_name, altdev); + if (rc) { + edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); + goto err_release_group1; + } + + altdev->db_irq = irq_of_parse_and_map(np, 1); + if (!altdev->db_irq) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Error allocating DBIRQ\n"); + rc = -ENODEV; + goto err_release_group1; + } + rc = devm_request_irq(edac->dev, altdev->db_irq, + prv->ecc_irq_handler, + IRQF_SHARED, ecc_name, altdev); + if (rc) { + edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); + goto err_release_group1; + } + rc = edac_device_add_device(dci); if (rc) { dev_err(edac->dev, "edac_device_add_device failed\n"); @@ -1186,7 +1491,6 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, err_release_group1: edac_device_free_ctl_info(dci); err_release_group: - edac_printk(KERN_ALERT, EDAC_DEVICE, "%s: %d\n", __func__, __LINE__); devres_release_group(edac->dev, NULL); edac_printk(KERN_ERR, EDAC_DEVICE, "%s:Error setting up EDAC device: %d\n", ecc_name, rc); @@ -1194,11 +1498,43 @@ err_release_group: return rc; } +static void a10_eccmgr_irq_mask(struct irq_data *d) +{ + struct altr_arria10_edac *edac = irq_data_get_irq_chip_data(d); + + regmap_write(edac->ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_SET_OFST, + BIT(d->hwirq)); +} + +static void a10_eccmgr_irq_unmask(struct irq_data *d) +{ + struct altr_arria10_edac *edac = irq_data_get_irq_chip_data(d); + + regmap_write(edac->ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_CLR_OFST, + BIT(d->hwirq)); +} + +static int a10_eccmgr_irqdomain_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) +{ + struct altr_arria10_edac *edac = d->host_data; + + irq_set_chip_and_handler(irq, &edac->irq_chip, handle_simple_irq); + irq_set_chip_data(irq, edac); + irq_set_noprobe(irq); + + return 0; +} + +struct irq_domain_ops a10_eccmgr_ic_ops = { + .map = a10_eccmgr_irqdomain_map, + .xlate = irq_domain_xlate_twocell, +}; + static int altr_edac_a10_probe(struct platform_device *pdev) { struct altr_arria10_edac *edac; struct device_node *child; - int rc; edac = devm_kzalloc(&pdev->dev, sizeof(*edac), GFP_KERNEL); if (!edac) @@ -1216,32 +1552,50 @@ static int altr_edac_a10_probe(struct platform_device *pdev) return PTR_ERR(edac->ecc_mgr_map); } + edac->irq_chip.name = pdev->dev.of_node->name; + edac->irq_chip.irq_mask = a10_eccmgr_irq_mask; + edac->irq_chip.irq_unmask = a10_eccmgr_irq_unmask; + edac->domain = irq_domain_add_linear(pdev->dev.of_node, 64, + &a10_eccmgr_ic_ops, edac); + if (!edac->domain) { + dev_err(&pdev->dev, "Error adding IRQ domain\n"); + return -ENOMEM; + } + edac->sb_irq = platform_get_irq(pdev, 0); - rc = devm_request_irq(&pdev->dev, edac->sb_irq, - altr_edac_a10_irq_handler, - IRQF_SHARED, dev_name(&pdev->dev), edac); - if (rc) { - edac_printk(KERN_ERR, EDAC_DEVICE, "No SBERR IRQ resource\n"); - return rc; + if (edac->sb_irq < 0) { + dev_err(&pdev->dev, "No SBERR IRQ resource\n"); + return edac->sb_irq; } + irq_set_chained_handler_and_data(edac->sb_irq, + altr_edac_a10_irq_handler, + edac); + edac->db_irq = platform_get_irq(pdev, 1); - rc = devm_request_irq(&pdev->dev, edac->db_irq, - altr_edac_a10_irq_handler, - IRQF_SHARED, dev_name(&pdev->dev), edac); - if (rc) { - edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); - return rc; + if (edac->db_irq < 0) { + dev_err(&pdev->dev, "No DBERR IRQ resource\n"); + return edac->db_irq; } + irq_set_chained_handler_and_data(edac->db_irq, + altr_edac_a10_irq_handler, + edac); for_each_child_of_node(pdev->dev.of_node, child) { if (!of_device_is_available(child)) continue; if (of_device_is_compatible(child, "altr,socfpga-a10-l2-ecc")) altr_edac_a10_device_add(edac, child); - else if (of_device_is_compatible(child, - "altr,socfpga-a10-ocram-ecc")) + else if ((of_device_is_compatible(child, + "altr,socfpga-a10-ocram-ecc")) || + (of_device_is_compatible(child, + "altr,socfpga-eth-mac-ecc"))) altr_edac_a10_device_add(edac, child); + else if (of_device_is_compatible(child, + "altr,sdram-edac-a10")) + of_platform_populate(pdev->dev.of_node, + altr_sdram_ctrl_of_match, + NULL, &pdev->dev); } return 0; diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h index 42090f36ba6e..687d8e754d36 100644 --- a/drivers/edac/altera_edac.h +++ b/drivers/edac/altera_edac.h @@ -230,8 +230,13 @@ struct altr_sdram_mc_data { #define ALTR_A10_ECC_INITCOMPLETEB BIT(8) #define ALTR_A10_ECC_ERRINTEN_OFST 0x10 +#define ALTR_A10_ECC_ERRINTENS_OFST 0x14 +#define ALTR_A10_ECC_ERRINTENR_OFST 0x18 #define ALTR_A10_ECC_SERRINTEN BIT(0) +#define ALTR_A10_ECC_INTMODE_OFST 0x1C +#define ALTR_A10_ECC_INTMODE BIT(0) + #define ALTR_A10_ECC_INTSTAT_OFST 0x20 #define ALTR_A10_ECC_SERRPENA BIT(0) #define ALTR_A10_ECC_DERRPENA BIT(8) @@ -280,6 +285,12 @@ struct altr_sdram_mc_data { /* Arria 10 OCRAM ECC Management Group Defines */ #define ALTR_A10_OCRAM_ECC_EN_CTL (BIT(1) | BIT(0)) +/* Arria 10 Ethernet ECC Management Group Defines */ +#define ALTR_A10_COMMON_ECC_EN_CTL BIT(0) + +/* A10 ECC Controller memory initialization timeout */ +#define ALTR_A10_ECC_INIT_WATCHDOG_10US 10000 + struct altr_edac_device_dev; struct edac_device_prv_data { @@ -295,10 +306,10 @@ struct edac_device_prv_data { int ce_set_mask; int ue_set_mask; int set_err_ofst; - irqreturn_t (*ecc_irq_handler)(struct altr_edac_device_dev *dci, - bool sb); + irqreturn_t (*ecc_irq_handler)(int irq, void *dev_id); int trig_alloc_sz; const struct file_operations *inject_fops; + bool panic; }; struct altr_edac_device_dev { @@ -320,6 +331,8 @@ struct altr_arria10_edac { struct regmap *ecc_mgr_map; int sb_irq; int db_irq; + struct irq_domain *domain; + struct irq_chip irq_chip; struct list_head a10_ecc_devices; }; diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 46784eb2edc6..8c0ec2128907 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -2966,11 +2966,11 @@ static int __init amd64_edac_init(void) int err = -ENODEV; int i; - opstate_init(); - if (amd_cache_northbridges() < 0) goto err_ret; + opstate_init(); + err = -ENOMEM; ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); if (!ecc_stngs) diff --git a/drivers/edac/edac_mc_sysfs.c b/drivers/edac/edac_mc_sysfs.c index 10c305b4a2e1..4e0f8e720ad9 100644 --- a/drivers/edac/edac_mc_sysfs.c +++ b/drivers/edac/edac_mc_sysfs.c @@ -313,7 +313,6 @@ static struct device_type csrow_attr_type = { * possible dynamic channel DIMM Label attribute files * */ - DEVICE_CHANNEL(ch0_dimm_label, S_IRUGO | S_IWUSR, channel_dimm_label_show, channel_dimm_label_store, 0); DEVICE_CHANNEL(ch1_dimm_label, S_IRUGO | S_IWUSR, @@ -326,6 +325,10 @@ DEVICE_CHANNEL(ch4_dimm_label, S_IRUGO | S_IWUSR, channel_dimm_label_show, channel_dimm_label_store, 4); DEVICE_CHANNEL(ch5_dimm_label, S_IRUGO | S_IWUSR, channel_dimm_label_show, channel_dimm_label_store, 5); +DEVICE_CHANNEL(ch6_dimm_label, S_IRUGO | S_IWUSR, + channel_dimm_label_show, channel_dimm_label_store, 6); +DEVICE_CHANNEL(ch7_dimm_label, S_IRUGO | S_IWUSR, + channel_dimm_label_show, channel_dimm_label_store, 7); /* Total possible dynamic DIMM Label attribute file table */ static struct attribute *dynamic_csrow_dimm_attr[] = { @@ -335,6 +338,8 @@ static struct attribute *dynamic_csrow_dimm_attr[] = { &dev_attr_legacy_ch3_dimm_label.attr.attr, &dev_attr_legacy_ch4_dimm_label.attr.attr, &dev_attr_legacy_ch5_dimm_label.attr.attr, + &dev_attr_legacy_ch6_dimm_label.attr.attr, + &dev_attr_legacy_ch7_dimm_label.attr.attr, NULL }; @@ -351,6 +356,10 @@ DEVICE_CHANNEL(ch4_ce_count, S_IRUGO, channel_ce_count_show, NULL, 4); DEVICE_CHANNEL(ch5_ce_count, S_IRUGO, channel_ce_count_show, NULL, 5); +DEVICE_CHANNEL(ch6_ce_count, S_IRUGO, + channel_ce_count_show, NULL, 6); +DEVICE_CHANNEL(ch7_ce_count, S_IRUGO, + channel_ce_count_show, NULL, 7); /* Total possible dynamic ce_count attribute file table */ static struct attribute *dynamic_csrow_ce_count_attr[] = { @@ -360,6 +369,8 @@ static struct attribute *dynamic_csrow_ce_count_attr[] = { &dev_attr_legacy_ch3_ce_count.attr.attr, &dev_attr_legacy_ch4_ce_count.attr.attr, &dev_attr_legacy_ch5_ce_count.attr.attr, + &dev_attr_legacy_ch6_ce_count.attr.attr, + &dev_attr_legacy_ch7_ce_count.attr.attr, NULL }; @@ -371,9 +382,16 @@ static umode_t csrow_dev_is_visible(struct kobject *kobj, if (idx >= csrow->nr_channels) return 0; + + if (idx >= ARRAY_SIZE(dynamic_csrow_ce_count_attr) - 1) { + WARN_ONCE(1, "idx: %d\n", idx); + return 0; + } + /* Only expose populated DIMMs */ if (!csrow->channels[idx]->dimm->nr_pages) return 0; + return attr->mode; } diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c index 6744d88bdea8..4fb2eb7c800d 100644 --- a/drivers/edac/sb_edac.c +++ b/drivers/edac/sb_edac.c @@ -2378,22 +2378,19 @@ static int sbridge_get_onedevice(struct pci_dev **prev, * @num_mc: pointer to the memory controllers count, to be incremented in case * of success. * @table: model specific table - * @allow_dups: allow for multiple devices to exist with the same device id - * (as implemented, this isn't expected to work correctly in the - * multi-socket case). - * @multi_bus: don't assume devices on different buses belong to different - * memory controllers. * * returns 0 in case of success or error code */ -static int sbridge_get_all_devices_full(u8 *num_mc, - const struct pci_id_table *table, - int allow_dups, - int multi_bus) +static int sbridge_get_all_devices(u8 *num_mc, + const struct pci_id_table *table) { int i, rc; struct pci_dev *pdev = NULL; + int allow_dups = 0; + int multi_bus = 0; + if (table->type == KNIGHTS_LANDING) + allow_dups = multi_bus = 1; while (table && table->descr) { for (i = 0; i < table->n_devs; i++) { if (!allow_dups || i == 0 || @@ -2420,11 +2417,6 @@ static int sbridge_get_all_devices_full(u8 *num_mc, return 0; } -#define sbridge_get_all_devices(num_mc, table) \ - sbridge_get_all_devices_full(num_mc, table, 0, 0) -#define sbridge_get_all_devices_knl(num_mc, table) \ - sbridge_get_all_devices_full(num_mc, table, 1, 1) - static int sbridge_mci_bind_devs(struct mem_ctl_info *mci, struct sbridge_dev *sbridge_dev) { diff --git a/drivers/extcon/Makefile b/drivers/extcon/Makefile index 2a0e4f45d5b2..972c813c375b 100644 --- a/drivers/extcon/Makefile +++ b/drivers/extcon/Makefile @@ -2,7 +2,8 @@ # Makefile for external connector class (extcon) devices # -obj-$(CONFIG_EXTCON) += extcon.o +obj-$(CONFIG_EXTCON) += extcon-core.o +extcon-core-objs += extcon.o devres.o obj-$(CONFIG_EXTCON_ADC_JACK) += extcon-adc-jack.o obj-$(CONFIG_EXTCON_ARIZONA) += extcon-arizona.o obj-$(CONFIG_EXTCON_AXP288) += extcon-axp288.o diff --git a/drivers/extcon/devres.c b/drivers/extcon/devres.c new file mode 100644 index 000000000000..e686acd1c459 --- /dev/null +++ b/drivers/extcon/devres.c @@ -0,0 +1,216 @@ +/* + * drivers/extcon/devres.c - EXTCON device's resource management + * + * Copyright (C) 2016 Samsung Electronics + * Author: Chanwoo Choi + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include + +static int devm_extcon_dev_match(struct device *dev, void *res, void *data) +{ + struct extcon_dev **r = res; + + if (WARN_ON(!r || !*r)) + return 0; + + return *r == data; +} + +static void devm_extcon_dev_release(struct device *dev, void *res) +{ + extcon_dev_free(*(struct extcon_dev **)res); +} + + +static void devm_extcon_dev_unreg(struct device *dev, void *res) +{ + extcon_dev_unregister(*(struct extcon_dev **)res); +} + +struct extcon_dev_notifier_devres { + struct extcon_dev *edev; + unsigned int id; + struct notifier_block *nb; +}; + +static void devm_extcon_dev_notifier_unreg(struct device *dev, void *res) +{ + struct extcon_dev_notifier_devres *this = res; + + extcon_unregister_notifier(this->edev, this->id, this->nb); +} + +/** + * devm_extcon_dev_allocate - Allocate managed extcon device + * @dev: device owning the extcon device being created + * @supported_cable: Array of supported extcon ending with EXTCON_NONE. + * If supported_cable is NULL, cable name related APIs + * are disabled. + * + * This function manages automatically the memory of extcon device using device + * resource management and simplify the control of freeing the memory of extcon + * device. + * + * Returns the pointer memory of allocated extcon_dev if success + * or ERR_PTR(err) if fail + */ +struct extcon_dev *devm_extcon_dev_allocate(struct device *dev, + const unsigned int *supported_cable) +{ + struct extcon_dev **ptr, *edev; + + ptr = devres_alloc(devm_extcon_dev_release, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + edev = extcon_dev_allocate(supported_cable); + if (IS_ERR(edev)) { + devres_free(ptr); + return edev; + } + + edev->dev.parent = dev; + + *ptr = edev; + devres_add(dev, ptr); + + return edev; +} +EXPORT_SYMBOL_GPL(devm_extcon_dev_allocate); + +/** + * devm_extcon_dev_free() - Resource-managed extcon_dev_unregister() + * @dev: device the extcon belongs to + * @edev: the extcon device to unregister + * + * Free the memory that is allocated with devm_extcon_dev_allocate() + * function. + */ +void devm_extcon_dev_free(struct device *dev, struct extcon_dev *edev) +{ + WARN_ON(devres_release(dev, devm_extcon_dev_release, + devm_extcon_dev_match, edev)); +} +EXPORT_SYMBOL_GPL(devm_extcon_dev_free); + +/** + * devm_extcon_dev_register() - Resource-managed extcon_dev_register() + * @dev: device to allocate extcon device + * @edev: the new extcon device to register + * + * Managed extcon_dev_register() function. If extcon device is attached with + * this function, that extcon device is automatically unregistered on driver + * detach. Internally this function calls extcon_dev_register() function. + * To get more information, refer that function. + * + * If extcon device is registered with this function and the device needs to be + * unregistered separately, devm_extcon_dev_unregister() should be used. + * + * Returns 0 if success or negaive error number if failure. + */ +int devm_extcon_dev_register(struct device *dev, struct extcon_dev *edev) +{ + struct extcon_dev **ptr; + int ret; + + ptr = devres_alloc(devm_extcon_dev_unreg, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + ret = extcon_dev_register(edev); + if (ret) { + devres_free(ptr); + return ret; + } + + *ptr = edev; + devres_add(dev, ptr); + + return 0; +} +EXPORT_SYMBOL_GPL(devm_extcon_dev_register); + +/** + * devm_extcon_dev_unregister() - Resource-managed extcon_dev_unregister() + * @dev: device the extcon belongs to + * @edev: the extcon device to unregister + * + * Unregister extcon device that is registered with devm_extcon_dev_register() + * function. + */ +void devm_extcon_dev_unregister(struct device *dev, struct extcon_dev *edev) +{ + WARN_ON(devres_release(dev, devm_extcon_dev_unreg, + devm_extcon_dev_match, edev)); +} +EXPORT_SYMBOL_GPL(devm_extcon_dev_unregister); + +/** + * devm_extcon_register_notifier() - Resource-managed extcon_register_notifier() + * @dev: device to allocate extcon device + * @edev: the extcon device that has the external connecotr. + * @id: the unique id of each external connector in extcon enumeration. + * @nb: a notifier block to be registered. + * + * This function manages automatically the notifier of extcon device using + * device resource management and simplify the control of unregistering + * the notifier of extcon device. + * + * Note that the second parameter given to the callback of nb (val) is + * "old_state", not the current state. The current state can be retrieved + * by looking at the third pameter (edev pointer)'s state value. + * + * Returns 0 if success or negaive error number if failure. + */ +int devm_extcon_register_notifier(struct device *dev, struct extcon_dev *edev, + unsigned int id, struct notifier_block *nb) +{ + struct extcon_dev_notifier_devres *ptr; + int ret; + + ptr = devres_alloc(devm_extcon_dev_notifier_unreg, sizeof(*ptr), + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + ret = extcon_register_notifier(edev, id, nb); + if (ret) { + devres_free(ptr); + return ret; + } + + ptr->edev = edev; + ptr->id = id; + ptr->nb = nb; + devres_add(dev, ptr); + + return 0; +} +EXPORT_SYMBOL(devm_extcon_register_notifier); + +/** + * devm_extcon_unregister_notifier() + - Resource-managed extcon_unregister_notifier() + * @dev: device to allocate extcon device + * @edev: the extcon device that has the external connecotr. + * @id: the unique id of each external connector in extcon enumeration. + * @nb: a notifier block to be registered. + */ +void devm_extcon_unregister_notifier(struct device *dev, + struct extcon_dev *edev, unsigned int id, + struct notifier_block *nb) +{ + WARN_ON(devres_release(dev, devm_extcon_dev_notifier_unreg, + devm_extcon_dev_match, edev)); +} +EXPORT_SYMBOL(devm_extcon_unregister_notifier); diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c index 7fc0ae1912f8..44e48aa78a84 100644 --- a/drivers/extcon/extcon-adc-jack.c +++ b/drivers/extcon/extcon-adc-jack.c @@ -38,6 +38,7 @@ * @chan: iio channel being queried. */ struct adc_jack_data { + struct device *dev; struct extcon_dev *edev; const unsigned int **cable_names; @@ -49,6 +50,7 @@ struct adc_jack_data { struct delayed_work handler; struct iio_channel *chan; + bool wakeup_source; }; static void adc_jack_handler(struct work_struct *work) @@ -105,6 +107,7 @@ static int adc_jack_probe(struct platform_device *pdev) return -EINVAL; } + data->dev = &pdev->dev; data->edev = devm_extcon_dev_allocate(&pdev->dev, pdata->cable_names); if (IS_ERR(data->edev)) { dev_err(&pdev->dev, "failed to allocate extcon device\n"); @@ -128,6 +131,7 @@ static int adc_jack_probe(struct platform_device *pdev) return PTR_ERR(data->chan); data->handling_delay = msecs_to_jiffies(pdata->handling_delay_ms); + data->wakeup_source = pdata->wakeup_source; INIT_DEFERRABLE_WORK(&data->handler, adc_jack_handler); @@ -151,6 +155,9 @@ static int adc_jack_probe(struct platform_device *pdev) return err; } + if (data->wakeup_source) + device_init_wakeup(&pdev->dev, 1); + return 0; } @@ -165,11 +172,38 @@ static int adc_jack_remove(struct platform_device *pdev) return 0; } +#ifdef CONFIG_PM_SLEEP +static int adc_jack_suspend(struct device *dev) +{ + struct adc_jack_data *data = dev_get_drvdata(dev); + + cancel_delayed_work_sync(&data->handler); + if (device_may_wakeup(data->dev)) + enable_irq_wake(data->irq); + + return 0; +} + +static int adc_jack_resume(struct device *dev) +{ + struct adc_jack_data *data = dev_get_drvdata(dev); + + if (device_may_wakeup(data->dev)) + disable_irq_wake(data->irq); + + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +static SIMPLE_DEV_PM_OPS(adc_jack_pm_ops, + adc_jack_suspend, adc_jack_resume); + static struct platform_driver adc_jack_driver = { .probe = adc_jack_probe, .remove = adc_jack_remove, .driver = { .name = "adc-jack", + .pm = &adc_jack_pm_ops, }, }; diff --git a/drivers/extcon/extcon-usb-gpio.c b/drivers/extcon/extcon-usb-gpio.c index 2b2fecffb1ad..2512660dc4b9 100644 --- a/drivers/extcon/extcon-usb-gpio.c +++ b/drivers/extcon/extcon-usb-gpio.c @@ -24,8 +24,10 @@ #include #include #include +#include #include #include +#include #define USB_GPIO_DEBOUNCE_MS 20 /* ms */ @@ -91,7 +93,7 @@ static int usb_extcon_probe(struct platform_device *pdev) struct usb_extcon_info *info; int ret; - if (!np) + if (!np && !ACPI_HANDLE(dev)) return -EINVAL; info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL); @@ -141,7 +143,8 @@ static int usb_extcon_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, info); - device_init_wakeup(dev, 1); + device_init_wakeup(dev, true); + dev_pm_set_wake_irq(dev, info->id_irq); /* Perform initial detection */ usb_extcon_detect_cable(&info->wq_detcable.work); @@ -155,6 +158,9 @@ static int usb_extcon_remove(struct platform_device *pdev) cancel_delayed_work_sync(&info->wq_detcable); + dev_pm_clear_wake_irq(&pdev->dev); + device_init_wakeup(&pdev->dev, false); + return 0; } @@ -164,12 +170,6 @@ static int usb_extcon_suspend(struct device *dev) struct usb_extcon_info *info = dev_get_drvdata(dev); int ret = 0; - if (device_may_wakeup(dev)) { - ret = enable_irq_wake(info->id_irq); - if (ret) - return ret; - } - /* * We don't want to process any IRQs after this point * as GPIOs used behind I2C subsystem might not be @@ -185,13 +185,10 @@ static int usb_extcon_resume(struct device *dev) struct usb_extcon_info *info = dev_get_drvdata(dev); int ret = 0; - if (device_may_wakeup(dev)) { - ret = disable_irq_wake(info->id_irq); - if (ret) - return ret; - } - enable_irq(info->id_irq); + if (!device_may_wakeup(dev)) + queue_delayed_work(system_power_efficient_wq, + &info->wq_detcable, 0); return ret; } @@ -206,6 +203,12 @@ static const struct of_device_id usb_extcon_dt_match[] = { }; MODULE_DEVICE_TABLE(of, usb_extcon_dt_match); +static const struct platform_device_id usb_extcon_platform_ids[] = { + { .name = "extcon-usb-gpio", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, usb_extcon_platform_ids); + static struct platform_driver usb_extcon_driver = { .probe = usb_extcon_probe, .remove = usb_extcon_remove, @@ -214,6 +217,7 @@ static struct platform_driver usb_extcon_driver = { .pm = &usb_extcon_pm_ops, .of_match_table = usb_extcon_dt_match, }, + .id_table = usb_extcon_platform_ids, }; module_platform_driver(usb_extcon_driver); diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c index 21a123cadf78..8682efc0f57b 100644 --- a/drivers/extcon/extcon.c +++ b/drivers/extcon/extcon.c @@ -77,6 +77,26 @@ static const char *extcon_name[] = { NULL, }; +/** + * struct extcon_cable - An internal data for each cable of extcon device. + * @edev: The extcon device + * @cable_index: Index of this cable in the edev + * @attr_g: Attribute group for the cable + * @attr_name: "name" sysfs entry + * @attr_state: "state" sysfs entry + * @attrs: Array pointing to attr_name and attr_state for attr_g + */ +struct extcon_cable { + struct extcon_dev *edev; + int cable_index; + + struct attribute_group attr_g; + struct device_attribute attr_name; + struct device_attribute attr_state; + + struct attribute *attrs[3]; /* to be fed to attr_g.attrs */ +}; + static struct class *extcon_class; #if defined(CONFIG_ANDROID) static struct class_compat *switch_class; @@ -127,38 +147,6 @@ static int find_cable_index_by_id(struct extcon_dev *edev, const unsigned int id return -EINVAL; } -static int find_cable_id_by_name(struct extcon_dev *edev, const char *name) -{ - int id = -EINVAL; - int i = 0; - - /* Find the id of extcon cable */ - while (extcon_name[i]) { - if (!strncmp(extcon_name[i], name, CABLE_NAME_MAX)) { - id = i; - break; - } - i++; - } - - return id; -} - -static int find_cable_index_by_name(struct extcon_dev *edev, const char *name) -{ - int id; - - if (edev->max_supported == 0) - return -EINVAL; - - /* Find the the number of extcon cable */ - id = find_cable_id_by_name(edev, name); - if (id < 0) - return id; - - return find_cable_index_by_id(edev, id); -} - static bool is_extcon_changed(u32 prev, u32 new, int idx, bool *attached) { if (((prev >> idx) & 0x1) != ((new >> idx) & 0x1)) { @@ -373,25 +361,6 @@ int extcon_get_cable_state_(struct extcon_dev *edev, const unsigned int id) } EXPORT_SYMBOL_GPL(extcon_get_cable_state_); -/** - * extcon_get_cable_state() - Get the status of a specific cable. - * @edev: the extcon device that has the cable. - * @cable_name: cable name. - * - * Note that this is slower than extcon_get_cable_state_. - */ -int extcon_get_cable_state(struct extcon_dev *edev, const char *cable_name) -{ - int id; - - id = find_cable_id_by_name(edev, cable_name); - if (id < 0) - return id; - - return extcon_get_cable_state_(edev, id); -} -EXPORT_SYMBOL_GPL(extcon_get_cable_state); - /** * extcon_set_cable_state_() - Set the status of a specific cable. * @edev: the extcon device that has the cable. @@ -421,28 +390,6 @@ int extcon_set_cable_state_(struct extcon_dev *edev, unsigned int id, } EXPORT_SYMBOL_GPL(extcon_set_cable_state_); -/** - * extcon_set_cable_state() - Set the status of a specific cable. - * @edev: the extcon device that has the cable. - * @cable_name: cable name. - * @cable_state: the new cable status. The default semantics is - * true: attached / false: detached. - * - * Note that this is slower than extcon_set_cable_state_. - */ -int extcon_set_cable_state(struct extcon_dev *edev, - const char *cable_name, bool cable_state) -{ - int id; - - id = find_cable_id_by_name(edev, cable_name); - if (id < 0) - return id; - - return extcon_set_cable_state_(edev, id, cable_state); -} -EXPORT_SYMBOL_GPL(extcon_set_cable_state); - /** * extcon_get_extcon_dev() - Get the extcon device instance from the name * @extcon_name: The extcon name provided with extcon_dev_register() @@ -466,105 +413,6 @@ out: } EXPORT_SYMBOL_GPL(extcon_get_extcon_dev); -/** - * extcon_register_interest() - Register a notifier for a state change of a - * specific cable, not an entier set of cables of a - * extcon device. - * @obj: an empty extcon_specific_cable_nb object to be returned. - * @extcon_name: the name of extcon device. - * if NULL, extcon_register_interest will register - * every cable with the target cable_name given. - * @cable_name: the target cable name. - * @nb: the notifier block to get notified. - * - * Provide an empty extcon_specific_cable_nb. extcon_register_interest() sets - * the struct for you. - * - * extcon_register_interest is a helper function for those who want to get - * notification for a single specific cable's status change. If a user wants - * to get notification for any changes of all cables of a extcon device, - * he/she should use the general extcon_register_notifier(). - * - * Note that the second parameter given to the callback of nb (val) is - * "old_state", not the current state. The current state can be retrieved - * by looking at the third pameter (edev pointer)'s state value. - */ -int extcon_register_interest(struct extcon_specific_cable_nb *obj, - const char *extcon_name, const char *cable_name, - struct notifier_block *nb) -{ - unsigned long flags; - int ret; - - if (!obj || !cable_name || !nb) - return -EINVAL; - - if (extcon_name) { - obj->edev = extcon_get_extcon_dev(extcon_name); - if (!obj->edev) - return -ENODEV; - - obj->cable_index = find_cable_index_by_name(obj->edev, - cable_name); - if (obj->cable_index < 0) - return obj->cable_index; - - obj->user_nb = nb; - - spin_lock_irqsave(&obj->edev->lock, flags); - ret = raw_notifier_chain_register( - &obj->edev->nh[obj->cable_index], - obj->user_nb); - spin_unlock_irqrestore(&obj->edev->lock, flags); - } else { - struct class_dev_iter iter; - struct extcon_dev *extd; - struct device *dev; - - if (!extcon_class) - return -ENODEV; - class_dev_iter_init(&iter, extcon_class, NULL, NULL); - while ((dev = class_dev_iter_next(&iter))) { - extd = dev_get_drvdata(dev); - - if (find_cable_index_by_name(extd, cable_name) < 0) - continue; - - class_dev_iter_exit(&iter); - return extcon_register_interest(obj, extd->name, - cable_name, nb); - } - - ret = -ENODEV; - } - - return ret; -} -EXPORT_SYMBOL_GPL(extcon_register_interest); - -/** - * extcon_unregister_interest() - Unregister the notifier registered by - * extcon_register_interest(). - * @obj: the extcon_specific_cable_nb object returned by - * extcon_register_interest(). - */ -int extcon_unregister_interest(struct extcon_specific_cable_nb *obj) -{ - unsigned long flags; - int ret; - - if (!obj) - return -EINVAL; - - spin_lock_irqsave(&obj->edev->lock, flags); - ret = raw_notifier_chain_unregister( - &obj->edev->nh[obj->cable_index], obj->user_nb); - spin_unlock_irqrestore(&obj->edev->lock, flags); - - return ret; -} -EXPORT_SYMBOL_GPL(extcon_unregister_interest); - /** * extcon_register_notifier() - Register a notifiee to get notified by * any attach status changes from the extcon. @@ -582,14 +430,35 @@ int extcon_register_notifier(struct extcon_dev *edev, unsigned int id, unsigned long flags; int ret, idx; - if (!edev || !nb) + if (!nb) return -EINVAL; - idx = find_cable_index_by_id(edev, id); + if (edev) { + idx = find_cable_index_by_id(edev, id); + if (idx < 0) + return idx; - spin_lock_irqsave(&edev->lock, flags); - ret = raw_notifier_chain_register(&edev->nh[idx], nb); - spin_unlock_irqrestore(&edev->lock, flags); + spin_lock_irqsave(&edev->lock, flags); + ret = raw_notifier_chain_register(&edev->nh[idx], nb); + spin_unlock_irqrestore(&edev->lock, flags); + } else { + struct extcon_dev *extd; + + mutex_lock(&extcon_dev_list_lock); + list_for_each_entry(extd, &extcon_dev_list, entry) { + idx = find_cable_index_by_id(extd, id); + if (idx >= 0) + break; + } + mutex_unlock(&extcon_dev_list_lock); + + if (idx >= 0) { + edev = extd; + return extcon_register_notifier(extd, id, nb); + } else { + ret = -ENODEV; + } + } return ret; } @@ -611,6 +480,8 @@ int extcon_unregister_notifier(struct extcon_dev *edev, unsigned int id, return -EINVAL; idx = find_cable_index_by_id(edev, id); + if (idx < 0) + return idx; spin_lock_irqsave(&edev->lock, flags); ret = raw_notifier_chain_unregister(&edev->nh[idx], nb); @@ -693,66 +564,6 @@ void extcon_dev_free(struct extcon_dev *edev) } EXPORT_SYMBOL_GPL(extcon_dev_free); -static int devm_extcon_dev_match(struct device *dev, void *res, void *data) -{ - struct extcon_dev **r = res; - - if (WARN_ON(!r || !*r)) - return 0; - - return *r == data; -} - -static void devm_extcon_dev_release(struct device *dev, void *res) -{ - extcon_dev_free(*(struct extcon_dev **)res); -} - -/** - * devm_extcon_dev_allocate - Allocate managed extcon device - * @dev: device owning the extcon device being created - * @supported_cable: Array of supported extcon ending with EXTCON_NONE. - * If supported_cable is NULL, cable name related APIs - * are disabled. - * - * This function manages automatically the memory of extcon device using device - * resource management and simplify the control of freeing the memory of extcon - * device. - * - * Returns the pointer memory of allocated extcon_dev if success - * or ERR_PTR(err) if fail - */ -struct extcon_dev *devm_extcon_dev_allocate(struct device *dev, - const unsigned int *supported_cable) -{ - struct extcon_dev **ptr, *edev; - - ptr = devres_alloc(devm_extcon_dev_release, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return ERR_PTR(-ENOMEM); - - edev = extcon_dev_allocate(supported_cable); - if (IS_ERR(edev)) { - devres_free(ptr); - return edev; - } - - edev->dev.parent = dev; - - *ptr = edev; - devres_add(dev, ptr); - - return edev; -} -EXPORT_SYMBOL_GPL(devm_extcon_dev_allocate); - -void devm_extcon_dev_free(struct device *dev, struct extcon_dev *edev) -{ - WARN_ON(devres_release(dev, devm_extcon_dev_release, - devm_extcon_dev_match, edev)); -} -EXPORT_SYMBOL_GPL(devm_extcon_dev_free); - /** * extcon_dev_register() - Register a new extcon device * @edev : the new extcon device (should be allocated before calling) @@ -1018,63 +829,6 @@ void extcon_dev_unregister(struct extcon_dev *edev) } EXPORT_SYMBOL_GPL(extcon_dev_unregister); -static void devm_extcon_dev_unreg(struct device *dev, void *res) -{ - extcon_dev_unregister(*(struct extcon_dev **)res); -} - -/** - * devm_extcon_dev_register() - Resource-managed extcon_dev_register() - * @dev: device to allocate extcon device - * @edev: the new extcon device to register - * - * Managed extcon_dev_register() function. If extcon device is attached with - * this function, that extcon device is automatically unregistered on driver - * detach. Internally this function calls extcon_dev_register() function. - * To get more information, refer that function. - * - * If extcon device is registered with this function and the device needs to be - * unregistered separately, devm_extcon_dev_unregister() should be used. - * - * Returns 0 if success or negaive error number if failure. - */ -int devm_extcon_dev_register(struct device *dev, struct extcon_dev *edev) -{ - struct extcon_dev **ptr; - int ret; - - ptr = devres_alloc(devm_extcon_dev_unreg, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return -ENOMEM; - - ret = extcon_dev_register(edev); - if (ret) { - devres_free(ptr); - return ret; - } - - *ptr = edev; - devres_add(dev, ptr); - - return 0; -} -EXPORT_SYMBOL_GPL(devm_extcon_dev_register); - -/** - * devm_extcon_dev_unregister() - Resource-managed extcon_dev_unregister() - * @dev: device the extcon belongs to - * @edev: the extcon device to unregister - * - * Unregister extcon device that is registered with devm_extcon_dev_register() - * function. - */ -void devm_extcon_dev_unregister(struct device *dev, struct extcon_dev *edev) -{ - WARN_ON(devres_release(dev, devm_extcon_dev_unreg, - devm_extcon_dev_match, edev)); -} -EXPORT_SYMBOL_GPL(devm_extcon_dev_unregister); - #ifdef CONFIG_OF /* * extcon_get_edev_by_phandle - Get the extcon device from devicetree @@ -1107,10 +861,12 @@ struct extcon_dev *extcon_get_edev_by_phandle(struct device *dev, int index) list_for_each_entry(edev, &extcon_dev_list, entry) { if (edev->dev.parent && edev->dev.parent->of_node == node) { mutex_unlock(&extcon_dev_list_lock); + of_node_put(node); return edev; } } mutex_unlock(&extcon_dev_list_lock); + of_node_put(node); return ERR_PTR(-EPROBE_DEFER); } diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c index 17ccf0a8787a..c394b81fe452 100644 --- a/drivers/firmware/efi/arm-runtime.c +++ b/drivers/firmware/efi/arm-runtime.c @@ -107,6 +107,11 @@ static int __init arm_enable_runtime_services(void) return 0; } + if (efi_enabled(EFI_RUNTIME_SERVICES)) { + pr_info("EFI runtime services access via paravirt.\n"); + return 0; + } + pr_info("Remapping and enabling EFI services.\n"); mapsize = efi.memmap.map_end - efi.memmap.map; diff --git a/drivers/firmware/efi/efi-pstore.c b/drivers/firmware/efi/efi-pstore.c index eac76a79a880..30a24d09ea6c 100644 --- a/drivers/firmware/efi/efi-pstore.c +++ b/drivers/firmware/efi/efi-pstore.c @@ -34,6 +34,7 @@ struct pstore_read_data { int *count; struct timespec *timespec; bool *compressed; + ssize_t *ecc_notice_size; char **buf; }; @@ -69,6 +70,7 @@ static int efi_pstore_read_func(struct efivar_entry *entry, void *data) *cb_data->compressed = true; else *cb_data->compressed = false; + *cb_data->ecc_notice_size = 0; } else if (sscanf(name, "dump-type%u-%u-%d-%lu", cb_data->type, &part, &cnt, &time) == 4) { *cb_data->id = generic_id(time, part, cnt); @@ -76,6 +78,7 @@ static int efi_pstore_read_func(struct efivar_entry *entry, void *data) cb_data->timespec->tv_sec = time; cb_data->timespec->tv_nsec = 0; *cb_data->compressed = false; + *cb_data->ecc_notice_size = 0; } else if (sscanf(name, "dump-type%u-%u-%lu", cb_data->type, &part, &time) == 3) { /* @@ -88,6 +91,7 @@ static int efi_pstore_read_func(struct efivar_entry *entry, void *data) cb_data->timespec->tv_sec = time; cb_data->timespec->tv_nsec = 0; *cb_data->compressed = false; + *cb_data->ecc_notice_size = 0; } else return 0; @@ -210,6 +214,7 @@ static int efi_pstore_sysfs_entry_iter(void *data, struct efivar_entry **pos) static ssize_t efi_pstore_read(u64 *id, enum pstore_type_id *type, int *count, struct timespec *timespec, char **buf, bool *compressed, + ssize_t *ecc_notice_size, struct pstore_info *psi) { struct pstore_read_data data; @@ -220,6 +225,7 @@ static ssize_t efi_pstore_read(u64 *id, enum pstore_type_id *type, data.count = count; data.timespec = timespec; data.compressed = compressed; + data.ecc_notice_size = ecc_notice_size; data.buf = buf; *data.buf = kzalloc(EFIVARS_DATA_SIZE_MAX, GFP_KERNEL); @@ -393,6 +399,13 @@ static __init int efivars_pstore_init(void) static __exit void efivars_pstore_exit(void) { + if (!efi_pstore_info.bufsize) + return; + + pstore_unregister(&efi_pstore_info); + kfree(efi_pstore_info.buf); + efi_pstore_info.buf = NULL; + efi_pstore_info.bufsize = 0; } module_init(efivars_pstore_init); diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 05509f3aaee8..5a2631af7410 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include #include @@ -195,6 +198,96 @@ static void generic_ops_unregister(void) efivars_unregister(&generic_efivars); } +#if IS_ENABLED(CONFIG_ACPI) +#define EFIVAR_SSDT_NAME_MAX 16 +static char efivar_ssdt[EFIVAR_SSDT_NAME_MAX] __initdata; +static int __init efivar_ssdt_setup(char *str) +{ + if (strlen(str) < sizeof(efivar_ssdt)) + memcpy(efivar_ssdt, str, strlen(str)); + else + pr_warn("efivar_ssdt: name too long: %s\n", str); + return 0; +} +__setup("efivar_ssdt=", efivar_ssdt_setup); + +static __init int efivar_ssdt_iter(efi_char16_t *name, efi_guid_t vendor, + unsigned long name_size, void *data) +{ + struct efivar_entry *entry; + struct list_head *list = data; + char utf8_name[EFIVAR_SSDT_NAME_MAX]; + int limit = min_t(unsigned long, EFIVAR_SSDT_NAME_MAX, name_size); + + ucs2_as_utf8(utf8_name, name, limit - 1); + if (strncmp(utf8_name, efivar_ssdt, limit) != 0) + return 0; + + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return 0; + + memcpy(entry->var.VariableName, name, name_size); + memcpy(&entry->var.VendorGuid, &vendor, sizeof(efi_guid_t)); + + efivar_entry_add(entry, list); + + return 0; +} + +static __init int efivar_ssdt_load(void) +{ + LIST_HEAD(entries); + struct efivar_entry *entry, *aux; + unsigned long size; + void *data; + int ret; + + ret = efivar_init(efivar_ssdt_iter, &entries, true, &entries); + + list_for_each_entry_safe(entry, aux, &entries, list) { + pr_info("loading SSDT from variable %s-%pUl\n", efivar_ssdt, + &entry->var.VendorGuid); + + list_del(&entry->list); + + ret = efivar_entry_size(entry, &size); + if (ret) { + pr_err("failed to get var size\n"); + goto free_entry; + } + + data = kmalloc(size, GFP_KERNEL); + if (!data) + goto free_entry; + + ret = efivar_entry_get(entry, NULL, &size, data); + if (ret) { + pr_err("failed to get var data\n"); + goto free_data; + } + + ret = acpi_load_table(data); + if (ret) { + pr_err("failed to load table: %d\n", ret); + goto free_data; + } + + goto free_entry; + +free_data: + kfree(data); + +free_entry: + kfree(entry); + } + + return ret; +} +#else +static inline int efivar_ssdt_load(void) { return 0; } +#endif + /* * We register the efi subsystem with the firmware subsystem and the * efivars subsystem with the efi subsystem, if the system was booted with @@ -218,6 +311,9 @@ static int __init efisubsys_init(void) if (error) goto err_put; + if (efi_enabled(EFI_RUNTIME_SERVICES)) + efivar_ssdt_load(); + error = sysfs_create_group(efi_kobj, &efi_subsys_attr_group); if (error) { pr_err("efi: Sysfs attribute export failed with error %d.\n", @@ -472,12 +568,14 @@ device_initcall(efi_load_efivars); FIELD_SIZEOF(struct efi_fdt_params, field) \ } -static __initdata struct { +struct params { const char name[32]; const char propname[32]; int offset; int size; -} dt_params[] = { +}; + +static __initdata struct params fdt_params[] = { UEFI_PARAM("System Table", "linux,uefi-system-table", system_table), UEFI_PARAM("MemMap Address", "linux,uefi-mmap-start", mmap), UEFI_PARAM("MemMap Size", "linux,uefi-mmap-size", mmap_size), @@ -485,44 +583,91 @@ static __initdata struct { UEFI_PARAM("MemMap Desc. Version", "linux,uefi-mmap-desc-ver", desc_ver) }; +static __initdata struct params xen_fdt_params[] = { + UEFI_PARAM("System Table", "xen,uefi-system-table", system_table), + UEFI_PARAM("MemMap Address", "xen,uefi-mmap-start", mmap), + UEFI_PARAM("MemMap Size", "xen,uefi-mmap-size", mmap_size), + UEFI_PARAM("MemMap Desc. Size", "xen,uefi-mmap-desc-size", desc_size), + UEFI_PARAM("MemMap Desc. Version", "xen,uefi-mmap-desc-ver", desc_ver) +}; + +#define EFI_FDT_PARAMS_SIZE ARRAY_SIZE(fdt_params) + +static __initdata struct { + const char *uname; + const char *subnode; + struct params *params; +} dt_params[] = { + { "hypervisor", "uefi", xen_fdt_params }, + { "chosen", NULL, fdt_params }, +}; + struct param_info { int found; void *params; + const char *missing; }; -static int __init fdt_find_uefi_params(unsigned long node, const char *uname, - int depth, void *data) +static int __init __find_uefi_params(unsigned long node, + struct param_info *info, + struct params *params) { - struct param_info *info = data; const void *prop; void *dest; u64 val; int i, len; - if (depth != 1 || strcmp(uname, "chosen") != 0) - return 0; - - for (i = 0; i < ARRAY_SIZE(dt_params); i++) { - prop = of_get_flat_dt_prop(node, dt_params[i].propname, &len); - if (!prop) + for (i = 0; i < EFI_FDT_PARAMS_SIZE; i++) { + prop = of_get_flat_dt_prop(node, params[i].propname, &len); + if (!prop) { + info->missing = params[i].name; return 0; - dest = info->params + dt_params[i].offset; + } + + dest = info->params + params[i].offset; info->found++; val = of_read_number(prop, len / sizeof(u32)); - if (dt_params[i].size == sizeof(u32)) + if (params[i].size == sizeof(u32)) *(u32 *)dest = val; else *(u64 *)dest = val; if (efi_enabled(EFI_DBG)) - pr_info(" %s: 0x%0*llx\n", dt_params[i].name, - dt_params[i].size * 2, val); + pr_info(" %s: 0x%0*llx\n", params[i].name, + params[i].size * 2, val); } + return 1; } +static int __init fdt_find_uefi_params(unsigned long node, const char *uname, + int depth, void *data) +{ + struct param_info *info = data; + int i; + + for (i = 0; i < ARRAY_SIZE(dt_params); i++) { + const char *subnode = dt_params[i].subnode; + + if (depth != 1 || strcmp(uname, dt_params[i].uname) != 0) { + info->missing = dt_params[i].params[0].name; + continue; + } + + if (subnode) { + node = of_get_flat_dt_subnode_by_name(node, subnode); + if (node < 0) + return 0; + } + + return __find_uefi_params(node, info, dt_params[i].params); + } + + return 0; +} + int __init efi_get_fdt_params(struct efi_fdt_params *params) { struct param_info info; @@ -538,7 +683,7 @@ int __init efi_get_fdt_params(struct efi_fdt_params *params) pr_info("UEFI not found.\n"); else if (!ret) pr_err("Can't find '%s' in device tree!\n", - dt_params[info.found].name); + info.missing); return ret; } diff --git a/drivers/firmware/efi/efibc.c b/drivers/firmware/efi/efibc.c index 8dd0c7085e59..503bbe2a9d49 100644 --- a/drivers/firmware/efi/efibc.c +++ b/drivers/firmware/efi/efibc.c @@ -37,13 +37,13 @@ static int efibc_set_variable(const char *name, const char *value) size_t size = (strlen(value) + 1) * sizeof(efi_char16_t); if (size > sizeof(entry->var.Data)) { - pr_err("value is too large"); + pr_err("value is too large (%zu bytes) for '%s' EFI variable\n", size, name); return -EINVAL; } entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) { - pr_err("failed to allocate efivar entry"); + pr_err("failed to allocate efivar entry for '%s' EFI variable\n", name); return -ENOMEM; } diff --git a/drivers/firmware/efi/runtime-wrappers.c b/drivers/firmware/efi/runtime-wrappers.c index 23bef6bb73ee..41958774cde3 100644 --- a/drivers/firmware/efi/runtime-wrappers.c +++ b/drivers/firmware/efi/runtime-wrappers.c @@ -22,7 +22,16 @@ #include #include -static void efi_call_virt_check_flags(unsigned long flags, const char *call) +/* + * Wrap around the new efi_call_virt_generic() macros so that the + * code doesn't get too cluttered: + */ +#define efi_call_virt(f, args...) \ + efi_call_virt_pointer(efi.systab->runtime, f, args) +#define __efi_call_virt(f, args...) \ + __efi_call_virt_pointer(efi.systab->runtime, f, args) + +void efi_call_virt_check_flags(unsigned long flags, const char *call) { unsigned long cur_flags, mismatch; @@ -38,48 +47,6 @@ static void efi_call_virt_check_flags(unsigned long flags, const char *call) local_irq_restore(flags); } -/* - * Arch code can implement the following three template macros, avoiding - * reptition for the void/non-void return cases of {__,}efi_call_virt: - * - * * arch_efi_call_virt_setup - * - * Sets up the environment for the call (e.g. switching page tables, - * allowing kernel-mode use of floating point, if required). - * - * * arch_efi_call_virt - * - * Performs the call. The last expression in the macro must be the call - * itself, allowing the logic to be shared by the void and non-void - * cases. - * - * * arch_efi_call_virt_teardown - * - * Restores the usual kernel environment once the call has returned. - */ - -#define efi_call_virt(f, args...) \ -({ \ - efi_status_t __s; \ - unsigned long flags; \ - arch_efi_call_virt_setup(); \ - local_save_flags(flags); \ - __s = arch_efi_call_virt(f, args); \ - efi_call_virt_check_flags(flags, __stringify(f)); \ - arch_efi_call_virt_teardown(); \ - __s; \ -}) - -#define __efi_call_virt(f, args...) \ -({ \ - unsigned long flags; \ - arch_efi_call_virt_setup(); \ - local_save_flags(flags); \ - arch_efi_call_virt(f, args); \ - efi_call_virt_check_flags(flags, __stringify(f)); \ - arch_efi_call_virt_teardown(); \ -}) - /* * According to section 7.1 of the UEFI spec, Runtime Services are not fully * reentrant, and there are particular combinations of calls that need to be diff --git a/drivers/firmware/psci.c b/drivers/firmware/psci.c index 03e04582791c..8263429e21b8 100644 --- a/drivers/firmware/psci.c +++ b/drivers/firmware/psci.c @@ -13,6 +13,7 @@ #define pr_fmt(fmt) "psci: " fmt +#include #include #include #include @@ -256,13 +257,6 @@ static int psci_dt_cpu_init_idle(struct device_node *cpu_node, int cpu) u32 *psci_states; struct device_node *state_node; - /* - * If the PSCI cpu_suspend function hook has not been initialized - * idle states must not be enabled, so bail out - */ - if (!psci_ops.cpu_suspend) - return -EOPNOTSUPP; - /* Count idle states */ while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states", count))) { @@ -310,11 +304,69 @@ free_mem: return ret; } +#ifdef CONFIG_ACPI +#include + +static int __maybe_unused psci_acpi_cpu_init_idle(unsigned int cpu) +{ + int i, count; + u32 *psci_states; + struct acpi_lpi_state *lpi; + struct acpi_processor *pr = per_cpu(processors, cpu); + + if (unlikely(!pr || !pr->flags.has_lpi)) + return -EINVAL; + + count = pr->power.count - 1; + if (count <= 0) + return -ENODEV; + + psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL); + if (!psci_states) + return -ENOMEM; + + for (i = 0; i < count; i++) { + u32 state; + + lpi = &pr->power.lpi_states[i + 1]; + /* + * Only bits[31:0] represent a PSCI power_state while + * bits[63:32] must be 0x0 as per ARM ACPI FFH Specification + */ + state = lpi->address; + if (!psci_power_state_is_valid(state)) { + pr_warn("Invalid PSCI power state %#x\n", state); + kfree(psci_states); + return -EINVAL; + } + psci_states[i] = state; + } + /* Idle states parsed correctly, initialize per-cpu pointer */ + per_cpu(psci_power_state, cpu) = psci_states; + return 0; +} +#else +static int __maybe_unused psci_acpi_cpu_init_idle(unsigned int cpu) +{ + return -EINVAL; +} +#endif + int psci_cpu_init_idle(unsigned int cpu) { struct device_node *cpu_node; int ret; + /* + * If the PSCI cpu_suspend function hook has not been initialized + * idle states must not be enabled, so bail out + */ + if (!psci_ops.cpu_suspend) + return -EOPNOTSUPP; + + if (!acpi_disabled) + return psci_acpi_cpu_init_idle(cpu); + cpu_node = of_get_cpu_node(cpu, NULL); if (!cpu_node) return -ENODEV; diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index cebcb405812e..98dd47a30fc7 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -49,7 +49,7 @@ config GPIO_DEVRES config OF_GPIO def_bool y - depends on OF || COMPILE_TEST + depends on OF config GPIO_ACPI def_bool y @@ -250,7 +250,7 @@ config GPIO_LOONGSON driver for GPIO functionality on Loongson-2F/3A/3B processors. config GPIO_LPC18XX - bool "NXP LPC18XX/43XX GPIO support" + tristate "NXP LPC18XX/43XX GPIO support" default y if ARCH_LPC18XX depends on OF_GPIO && (ARCH_LPC18XX || COMPILE_TEST) help @@ -402,9 +402,12 @@ config GPIO_TB10X select OF_GPIO config GPIO_TEGRA - bool - default y + bool "NVIDIA Tegra GPIO support" + default ARCH_TEGRA depends on ARCH_TEGRA || COMPILE_TEST + depends on OF + help + Say yes here to support GPIO pins on NVIDIA Tegra SoCs. config GPIO_TS4800 tristate "TS-4800 DIO blocks and compatibles" @@ -871,6 +874,15 @@ config GPIO_LP3943 LP3943 can be used as a GPIO expander which provides up to 16 GPIOs. Open drain outputs are required for this usage. +config GPIO_MAX77620 + tristate "GPIO support for PMIC MAX77620 and MAX20024" + depends on MFD_MAX77620 + help + GPIO driver for MAX77620 and MAX20024 PMIC from Maxim Semiconductor. + MAX77620 PMIC has 8 pins that can be configured as GPIOs. The + driver also provides interrupt support for each of the gpios. + Say yes here to enable the max77620 to be used as gpio controller. + config GPIO_MSIC bool "Intel MSIC mixed signal gpio support" depends on MFD_INTEL_MSIC @@ -1026,11 +1038,18 @@ config GPIO_BT8XX If unsure, say N. config GPIO_INTEL_MID - bool "Intel Mid GPIO support" - depends on X86 + bool "Intel MID GPIO support" + depends on X86_INTEL_MID + select GPIOLIB_IRQCHIP + help + Say Y here to support Intel MID GPIO. + +config GPIO_MERRIFIELD + tristate "Intel Merrifield GPIO support" + depends on X86_INTEL_MID select GPIOLIB_IRQCHIP help - Say Y here to support Intel Mid GPIO. + Say Y here to support Intel Merrifield GPIO. config GPIO_ML_IOH tristate "OKI SEMICONDUCTOR ML7213 IOH GPIO support" diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 991598ea3fba..2a035ed8f168 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -61,8 +61,10 @@ obj-$(CONFIG_GPIO_MAX730X) += gpio-max730x.o obj-$(CONFIG_GPIO_MAX7300) += gpio-max7300.o obj-$(CONFIG_GPIO_MAX7301) += gpio-max7301.o obj-$(CONFIG_GPIO_MAX732X) += gpio-max732x.o +obj-$(CONFIG_GPIO_MAX77620) += gpio-max77620.o obj-$(CONFIG_GPIO_MB86S7X) += gpio-mb86s7x.o obj-$(CONFIG_GPIO_MENZ127) += gpio-menz127.o +obj-$(CONFIG_GPIO_MERRIFIELD) += gpio-merrifield.o obj-$(CONFIG_GPIO_MC33880) += gpio-mc33880.o obj-$(CONFIG_GPIO_MC9S08DZ60) += gpio-mc9s08dz60.o obj-$(CONFIG_GPIO_MCP23S08) += gpio-mcp23s08.o diff --git a/drivers/gpio/gpio-104-idi-48.c b/drivers/gpio/gpio-104-idi-48.c index 6c75c83baf5a..2d2763ea1a68 100644 --- a/drivers/gpio/gpio-104-idi-48.c +++ b/drivers/gpio/gpio-104-idi-48.c @@ -247,6 +247,7 @@ static int idi_48_probe(struct device *dev, unsigned int id) idi48gpio->irq = irq[id]; spin_lock_init(&idi48gpio->lock); + spin_lock_init(&idi48gpio->ack_lock); dev_set_drvdata(dev, idi48gpio); diff --git a/drivers/gpio/gpio-74x164.c b/drivers/gpio/gpio-74x164.c index 80f9ddf13343..a6607faf2fdf 100644 --- a/drivers/gpio/gpio-74x164.c +++ b/drivers/gpio/gpio-74x164.c @@ -35,13 +35,8 @@ struct gen_74x164_chip { static int __gen_74x164_write_config(struct gen_74x164_chip *chip) { - struct spi_transfer xfer = { - .tx_buf = chip->buffer, - .len = chip->registers, - }; - - return spi_sync_transfer(to_spi_device(chip->gpio_chip.parent), - &xfer, 1); + return spi_write(to_spi_device(chip->gpio_chip.parent), chip->buffer, + chip->registers); } static int gen_74x164_get_value(struct gpio_chip *gc, unsigned offset) diff --git a/drivers/gpio/gpio-clps711x.c b/drivers/gpio/gpio-clps711x.c index 5a690256af9b..52fd63f02134 100644 --- a/drivers/gpio/gpio-clps711x.c +++ b/drivers/gpio/gpio-clps711x.c @@ -20,8 +20,12 @@ static int clps711x_gpio_probe(struct platform_device *pdev) void __iomem *dat, *dir; struct gpio_chip *gc; struct resource *res; - int err, id = np ? of_alias_get_id(np, "gpio") : pdev->id; + int err, id; + if (!np) + return -ENODEV; + + id = of_alias_get_id(np, "gpio"); if ((id < 0) || (id > 4)) return -ENODEV; @@ -63,7 +67,7 @@ static int clps711x_gpio_probe(struct platform_device *pdev) break; } - gc->base = id * 8; + gc->base = -1; gc->owner = THIS_MODULE; platform_set_drvdata(pdev, gc); @@ -71,7 +75,7 @@ static int clps711x_gpio_probe(struct platform_device *pdev) } static const struct of_device_id __maybe_unused clps711x_gpio_ids[] = { - { .compatible = "cirrus,clps711x-gpio" }, + { .compatible = "cirrus,ep7209-gpio" }, { } }; MODULE_DEVICE_TABLE(of, clps711x_gpio_ids); diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c index 34779bb375de..6193f62c0df4 100644 --- a/drivers/gpio/gpio-dwapb.c +++ b/drivers/gpio/gpio-dwapb.c @@ -486,6 +486,7 @@ dwapb_gpio_get_pdata(struct device *dev) pp->idx >= DWAPB_MAX_PORTS) { dev_err(dev, "missing/invalid port index for port%d\n", i); + fwnode_handle_put(fwnode); return ERR_PTR(-EINVAL); } diff --git a/drivers/gpio/gpio-f7188x.c b/drivers/gpio/gpio-f7188x.c index 05aa538c3767..600be8418707 100644 --- a/drivers/gpio/gpio-f7188x.c +++ b/drivers/gpio/gpio-f7188x.c @@ -125,6 +125,7 @@ static inline void superio_exit(int base) * GPIO chip. */ +static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset); static int f7188x_gpio_direction_in(struct gpio_chip *chip, unsigned offset); static int f7188x_gpio_get(struct gpio_chip *chip, unsigned offset); static int f7188x_gpio_direction_out(struct gpio_chip *chip, @@ -139,6 +140,7 @@ static int f7188x_gpio_set_single_ended(struct gpio_chip *gc, .chip = { \ .label = DRVNAME, \ .owner = THIS_MODULE, \ + .get_direction = f7188x_gpio_get_direction, \ .direction_input = f7188x_gpio_direction_in, \ .get = f7188x_gpio_get, \ .direction_output = f7188x_gpio_direction_out, \ @@ -209,6 +211,26 @@ static struct f7188x_gpio_bank f81866_gpio_bank[] = { F7188X_GPIO_BANK(80, 8, 0x88), }; +static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset) +{ + int err; + struct f7188x_gpio_bank *bank = + container_of(chip, struct f7188x_gpio_bank, chip); + struct f7188x_sio *sio = bank->data->sio; + u8 dir; + + err = superio_enter(sio->addr); + if (err) + return err; + superio_select(sio->addr, SIO_LD_GPIO); + + dir = superio_inb(sio->addr, gpio_dir(bank->regbase)); + + superio_exit(sio->addr); + + return !(dir & 1 << offset); +} + static int f7188x_gpio_direction_in(struct gpio_chip *chip, unsigned offset) { int err; diff --git a/drivers/gpio/gpio-intel-mid.c b/drivers/gpio/gpio-intel-mid.c index cdaba13cb8e8..164de64b11fc 100644 --- a/drivers/gpio/gpio-intel-mid.c +++ b/drivers/gpio/gpio-intel-mid.c @@ -1,7 +1,7 @@ /* * Intel MID GPIO driver * - * Copyright (c) 2008-2014 Intel Corporation. + * Copyright (c) 2008-2014,2016 Intel Corporation. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -17,21 +17,20 @@ * Moorestown platform Langwell chip. * Medfield platform Penwell chip. * Clovertrail platform Cloverview chip. - * Merrifield platform Tangier chip. */ -#include -#include -#include -#include #include -#include -#include #include +#include #include #include -#include +#include +#include +#include +#include #include +#include +#include #define INTEL_MID_IRQ_TYPE_EDGE (1 << 0) #define INTEL_MID_IRQ_TYPE_LEVEL (1 << 1) @@ -64,10 +63,6 @@ enum GPIO_REG { /* intel_mid gpio driver data */ struct intel_mid_gpio_ddata { u16 ngpio; /* number of gpio pins */ - u32 gplr_offset; /* offset of first GPLR register from base */ - u32 flis_base; /* base address of FLIS registers */ - u32 flis_len; /* length of FLIS registers */ - u32 (*get_flis_offset)(int gpio); u32 chip_irq_type; /* chip interrupt type */ }; @@ -252,15 +247,6 @@ static const struct intel_mid_gpio_ddata gpio_cloverview_core = { .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, }; -static const struct intel_mid_gpio_ddata gpio_tangier = { - .ngpio = 192, - .gplr_offset = 4, - .flis_base = 0xff0c0000, - .flis_len = 0x8000, - .get_flis_offset = NULL, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, -}; - static const struct pci_device_id intel_gpio_ids[] = { { /* Lincroft */ @@ -287,11 +273,6 @@ static const struct pci_device_id intel_gpio_ids[] = { PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x08f7), .driver_data = (kernel_ulong_t)&gpio_cloverview_core, }, - { - /* Tangier */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1199), - .driver_data = (kernel_ulong_t)&gpio_tangier, - }, { 0 } }; MODULE_DEVICE_TABLE(pci, intel_gpio_ids); @@ -401,7 +382,7 @@ static int intel_gpio_probe(struct pci_dev *pdev, spin_lock_init(&priv->lock); pci_set_drvdata(pdev, priv); - retval = gpiochip_add_data(&priv->chip, priv); + retval = devm_gpiochip_add_data(&pdev->dev, &priv->chip, priv); if (retval) { dev_err(&pdev->dev, "gpiochip_add error %d\n", retval); return retval; diff --git a/drivers/gpio/gpio-lynxpoint.c b/drivers/gpio/gpio-lynxpoint.c index 9df015e85ad9..fbd393b46ce0 100644 --- a/drivers/gpio/gpio-lynxpoint.c +++ b/drivers/gpio/gpio-lynxpoint.c @@ -383,7 +383,6 @@ static int lp_gpio_probe(struct platform_device *pdev) handle_simple_irq, IRQ_TYPE_NONE); if (ret) { dev_err(dev, "failed to add irqchip\n"); - gpiochip_remove(gc); return ret; } diff --git a/drivers/gpio/gpio-max77620.c b/drivers/gpio/gpio-max77620.c new file mode 100644 index 000000000000..b46b436cb97f --- /dev/null +++ b/drivers/gpio/gpio-max77620.c @@ -0,0 +1,315 @@ +/* + * MAXIM MAX77620 GPIO driver + * + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include + +#define GPIO_REG_ADDR(offset) (MAX77620_REG_GPIO0 + offset) + +struct max77620_gpio { + struct gpio_chip gpio_chip; + struct regmap *rmap; + struct device *dev; + int gpio_irq; + int irq_base; + int gpio_base; +}; + +static const struct regmap_irq max77620_gpio_irqs[] = { + [0] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE0, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 0, + }, + [1] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE1, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 1, + }, + [2] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE2, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 2, + }, + [3] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE3, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 3, + }, + [4] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE4, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 4, + }, + [5] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE5, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 5, + }, + [6] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE6, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 6, + }, + [7] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE7, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 7, + }, +}; + +static struct regmap_irq_chip max77620_gpio_irq_chip = { + .name = "max77620-gpio", + .irqs = max77620_gpio_irqs, + .num_irqs = ARRAY_SIZE(max77620_gpio_irqs), + .num_regs = 1, + .num_type_reg = 8, + .irq_reg_stride = 1, + .type_reg_stride = 1, + .status_base = MAX77620_REG_IRQ_LVL2_GPIO, + .type_base = MAX77620_REG_GPIO0, +}; + +static int max77620_gpio_dir_input(struct gpio_chip *gc, unsigned int offset) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + int ret; + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DIR_MASK, + MAX77620_CNFG_GPIO_DIR_INPUT); + if (ret < 0) + dev_err(mgpio->dev, "CNFG_GPIOx dir update failed: %d\n", ret); + + return ret; +} + +static int max77620_gpio_get(struct gpio_chip *gc, unsigned int offset) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + unsigned int val; + int ret; + + ret = regmap_read(mgpio->rmap, GPIO_REG_ADDR(offset), &val); + if (ret < 0) { + dev_err(mgpio->dev, "CNFG_GPIOx read failed: %d\n", ret); + return ret; + } + + if (val & MAX77620_CNFG_GPIO_DIR_MASK) + return !!(val & MAX77620_CNFG_GPIO_INPUT_VAL_MASK); + else + return !!(val & MAX77620_CNFG_GPIO_OUTPUT_VAL_MASK); +} + +static int max77620_gpio_dir_output(struct gpio_chip *gc, unsigned int offset, + int value) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + u8 val; + int ret; + + val = (value) ? MAX77620_CNFG_GPIO_OUTPUT_VAL_HIGH : + MAX77620_CNFG_GPIO_OUTPUT_VAL_LOW; + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_OUTPUT_VAL_MASK, val); + if (ret < 0) { + dev_err(mgpio->dev, "CNFG_GPIOx val update failed: %d\n", ret); + return ret; + } + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DIR_MASK, + MAX77620_CNFG_GPIO_DIR_OUTPUT); + if (ret < 0) + dev_err(mgpio->dev, "CNFG_GPIOx dir update failed: %d\n", ret); + + return ret; +} + +static int max77620_gpio_set_debounce(struct gpio_chip *gc, + unsigned int offset, + unsigned int debounce) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + u8 val; + int ret; + + switch (debounce) { + case 0: + val = MAX77620_CNFG_GPIO_DBNC_None; + break; + case 1 ... 8: + val = MAX77620_CNFG_GPIO_DBNC_8ms; + break; + case 9 ... 16: + val = MAX77620_CNFG_GPIO_DBNC_16ms; + break; + case 17 ... 32: + val = MAX77620_CNFG_GPIO_DBNC_32ms; + break; + default: + dev_err(mgpio->dev, "Illegal value %u\n", debounce); + return -EINVAL; + } + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DBNC_MASK, val); + if (ret < 0) + dev_err(mgpio->dev, "CNFG_GPIOx_DBNC update failed: %d\n", ret); + + return ret; +} + +static void max77620_gpio_set(struct gpio_chip *gc, unsigned int offset, + int value) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + u8 val; + int ret; + + val = (value) ? MAX77620_CNFG_GPIO_OUTPUT_VAL_HIGH : + MAX77620_CNFG_GPIO_OUTPUT_VAL_LOW; + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_OUTPUT_VAL_MASK, val); + if (ret < 0) + dev_err(mgpio->dev, "CNFG_GPIO_OUT update failed: %d\n", ret); +} + +static int max77620_gpio_set_single_ended(struct gpio_chip *gc, + unsigned int offset, + enum single_ended_mode mode) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + + switch (mode) { + case LINE_MODE_OPEN_DRAIN: + return regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DRV_MASK, + MAX77620_CNFG_GPIO_DRV_OPENDRAIN); + case LINE_MODE_PUSH_PULL: + return regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DRV_MASK, + MAX77620_CNFG_GPIO_DRV_PUSHPULL); + default: + break; + } + + return -ENOTSUPP; +} + +static int max77620_gpio_to_irq(struct gpio_chip *gc, unsigned int offset) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + struct max77620_chip *chip = dev_get_drvdata(mgpio->dev->parent); + + return regmap_irq_get_virq(chip->gpio_irq_data, offset); +} + +static int max77620_gpio_probe(struct platform_device *pdev) +{ + struct max77620_chip *chip = dev_get_drvdata(pdev->dev.parent); + struct max77620_gpio *mgpio; + int gpio_irq; + int ret; + + gpio_irq = platform_get_irq(pdev, 0); + if (gpio_irq <= 0) { + dev_err(&pdev->dev, "GPIO irq not available %d\n", gpio_irq); + return -ENODEV; + } + + mgpio = devm_kzalloc(&pdev->dev, sizeof(*mgpio), GFP_KERNEL); + if (!mgpio) + return -ENOMEM; + + mgpio->rmap = chip->rmap; + mgpio->dev = &pdev->dev; + mgpio->gpio_irq = gpio_irq; + + mgpio->gpio_chip.label = pdev->name; + mgpio->gpio_chip.parent = &pdev->dev; + mgpio->gpio_chip.direction_input = max77620_gpio_dir_input; + mgpio->gpio_chip.get = max77620_gpio_get; + mgpio->gpio_chip.direction_output = max77620_gpio_dir_output; + mgpio->gpio_chip.set_debounce = max77620_gpio_set_debounce; + mgpio->gpio_chip.set = max77620_gpio_set; + mgpio->gpio_chip.set_single_ended = max77620_gpio_set_single_ended; + mgpio->gpio_chip.to_irq = max77620_gpio_to_irq; + mgpio->gpio_chip.ngpio = MAX77620_GPIO_NR; + mgpio->gpio_chip.can_sleep = 1; + mgpio->gpio_chip.base = -1; + mgpio->irq_base = -1; +#ifdef CONFIG_OF_GPIO + mgpio->gpio_chip.of_node = pdev->dev.parent->of_node; +#endif + + platform_set_drvdata(pdev, mgpio); + + ret = devm_gpiochip_add_data(&pdev->dev, &mgpio->gpio_chip, mgpio); + if (ret < 0) { + dev_err(&pdev->dev, "gpio_init: Failed to add max77620_gpio\n"); + return ret; + } + + mgpio->gpio_base = mgpio->gpio_chip.base; + ret = devm_regmap_add_irq_chip(&pdev->dev, chip->rmap, mgpio->gpio_irq, + IRQF_ONESHOT, mgpio->irq_base, + &max77620_gpio_irq_chip, + &chip->gpio_irq_data); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to add gpio irq_chip %d\n", ret); + return ret; + } + + return 0; +} + +static const struct platform_device_id max77620_gpio_devtype[] = { + { .name = "max77620-gpio", }, + {}, +}; +MODULE_DEVICE_TABLE(platform, max77620_gpio_devtype); + +static struct platform_driver max77620_gpio_driver = { + .driver.name = "max77620-gpio", + .probe = max77620_gpio_probe, + .id_table = max77620_gpio_devtype, +}; + +module_platform_driver(max77620_gpio_driver); + +MODULE_DESCRIPTION("GPIO interface for MAX77620 and MAX20024 PMIC"); +MODULE_AUTHOR("Laxman Dewangan "); +MODULE_AUTHOR("Chaitanya Bandi "); +MODULE_ALIAS("platform:max77620-gpio"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpio/gpio-menz127.c b/drivers/gpio/gpio-menz127.c index cc103aff45e4..a1210e330571 100644 --- a/drivers/gpio/gpio-menz127.c +++ b/drivers/gpio/gpio-menz127.c @@ -187,7 +187,6 @@ MODULE_DEVICE_TABLE(mcb, men_z127_ids); static struct mcb_driver men_z127_driver = { .driver = { .name = "z127-gpio", - .owner = THIS_MODULE, }, .probe = men_z127_probe, .remove = men_z127_remove, diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c new file mode 100644 index 000000000000..45b51278b8ee --- /dev/null +++ b/drivers/gpio/gpio-merrifield.c @@ -0,0 +1,444 @@ +/* + * Intel Merrifield SoC GPIO driver + * + * Copyright (c) 2016 Intel Corporation. + * Author: Andy Shevchenko + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define GCCR 0x000 /* controller configuration */ +#define GPLR 0x004 /* pin level r/o */ +#define GPDR 0x01c /* pin direction */ +#define GPSR 0x034 /* pin set w/o */ +#define GPCR 0x04c /* pin clear w/o */ +#define GRER 0x064 /* rising edge detect */ +#define GFER 0x07c /* falling edge detect */ +#define GFBR 0x094 /* glitch filter bypass */ +#define GIMR 0x0ac /* interrupt mask */ +#define GISR 0x0c4 /* interrupt source */ +#define GITR 0x300 /* input type */ +#define GLPR 0x318 /* level input polarity */ +#define GWMR 0x400 /* wake mask */ +#define GWSR 0x418 /* wake source */ +#define GSIR 0xc00 /* secure input */ + +/* Intel Merrifield has 192 GPIO pins */ +#define MRFLD_NGPIO 192 + +struct mrfld_gpio_pinrange { + unsigned int gpio_base; + unsigned int pin_base; + unsigned int npins; +}; + +#define GPIO_PINRANGE(gstart, gend, pstart) \ + { \ + .gpio_base = (gstart), \ + .pin_base = (pstart), \ + .npins = (gend) - (gstart) + 1, \ + } + +struct mrfld_gpio { + struct gpio_chip chip; + void __iomem *reg_base; + raw_spinlock_t lock; + struct device *dev; +}; + +static const struct mrfld_gpio_pinrange mrfld_gpio_ranges[] = { + GPIO_PINRANGE(0, 11, 146), + GPIO_PINRANGE(12, 13, 144), + GPIO_PINRANGE(14, 15, 35), + GPIO_PINRANGE(16, 16, 164), + GPIO_PINRANGE(17, 18, 105), + GPIO_PINRANGE(19, 22, 101), + GPIO_PINRANGE(23, 30, 107), + GPIO_PINRANGE(32, 43, 67), + GPIO_PINRANGE(44, 63, 195), + GPIO_PINRANGE(64, 67, 140), + GPIO_PINRANGE(68, 69, 165), + GPIO_PINRANGE(70, 71, 65), + GPIO_PINRANGE(72, 76, 228), + GPIO_PINRANGE(77, 86, 37), + GPIO_PINRANGE(87, 87, 48), + GPIO_PINRANGE(88, 88, 47), + GPIO_PINRANGE(89, 96, 49), + GPIO_PINRANGE(97, 97, 34), + GPIO_PINRANGE(102, 119, 83), + GPIO_PINRANGE(120, 123, 79), + GPIO_PINRANGE(124, 135, 115), + GPIO_PINRANGE(137, 142, 158), + GPIO_PINRANGE(154, 163, 24), + GPIO_PINRANGE(164, 176, 215), + GPIO_PINRANGE(177, 189, 127), + GPIO_PINRANGE(190, 191, 178), +}; + +static void __iomem *gpio_reg(struct gpio_chip *chip, unsigned int offset, + unsigned int reg_type_offset) +{ + struct mrfld_gpio *priv = gpiochip_get_data(chip); + u8 reg = offset / 32; + + return priv->reg_base + reg_type_offset + reg * 4; +} + +static int mrfld_gpio_get(struct gpio_chip *chip, unsigned int offset) +{ + void __iomem *gplr = gpio_reg(chip, offset, GPLR); + + return !!(readl(gplr) & BIT(offset % 32)); +} + +static void mrfld_gpio_set(struct gpio_chip *chip, unsigned int offset, + int value) +{ + struct mrfld_gpio *priv = gpiochip_get_data(chip); + void __iomem *gpsr, *gpcr; + unsigned long flags; + + raw_spin_lock_irqsave(&priv->lock, flags); + + if (value) { + gpsr = gpio_reg(chip, offset, GPSR); + writel(BIT(offset % 32), gpsr); + } else { + gpcr = gpio_reg(chip, offset, GPCR); + writel(BIT(offset % 32), gpcr); + } + + raw_spin_unlock_irqrestore(&priv->lock, flags); +} + +static int mrfld_gpio_direction_input(struct gpio_chip *chip, + unsigned int offset) +{ + struct mrfld_gpio *priv = gpiochip_get_data(chip); + void __iomem *gpdr = gpio_reg(chip, offset, GPDR); + unsigned long flags; + u32 value; + + raw_spin_lock_irqsave(&priv->lock, flags); + + value = readl(gpdr); + value &= ~BIT(offset % 32); + writel(value, gpdr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static int mrfld_gpio_direction_output(struct gpio_chip *chip, + unsigned int offset, int value) +{ + struct mrfld_gpio *priv = gpiochip_get_data(chip); + void __iomem *gpdr = gpio_reg(chip, offset, GPDR); + unsigned long flags; + + mrfld_gpio_set(chip, offset, value); + + raw_spin_lock_irqsave(&priv->lock, flags); + + value = readl(gpdr); + value |= BIT(offset % 32); + writel(value, gpdr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static void mrfld_irq_ack(struct irq_data *d) +{ + struct mrfld_gpio *priv = irq_data_get_irq_chip_data(d); + u32 gpio = irqd_to_hwirq(d); + void __iomem *gisr = gpio_reg(&priv->chip, gpio, GISR); + unsigned long flags; + + raw_spin_lock_irqsave(&priv->lock, flags); + + writel(BIT(gpio % 32), gisr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); +} + +static void mrfld_irq_unmask_mask(struct irq_data *d, bool unmask) +{ + struct mrfld_gpio *priv = irq_data_get_irq_chip_data(d); + u32 gpio = irqd_to_hwirq(d); + void __iomem *gimr = gpio_reg(&priv->chip, gpio, GIMR); + unsigned long flags; + u32 value; + + raw_spin_lock_irqsave(&priv->lock, flags); + + if (unmask) + value = readl(gimr) | BIT(gpio % 32); + else + value = readl(gimr) & ~BIT(gpio % 32); + writel(value, gimr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); +} + +static void mrfld_irq_mask(struct irq_data *d) +{ + mrfld_irq_unmask_mask(d, false); +} + +static void mrfld_irq_unmask(struct irq_data *d) +{ + mrfld_irq_unmask_mask(d, true); +} + +static int mrfld_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct mrfld_gpio *priv = gpiochip_get_data(gc); + u32 gpio = irqd_to_hwirq(d); + void __iomem *grer = gpio_reg(&priv->chip, gpio, GRER); + void __iomem *gfer = gpio_reg(&priv->chip, gpio, GFER); + void __iomem *gitr = gpio_reg(&priv->chip, gpio, GITR); + void __iomem *glpr = gpio_reg(&priv->chip, gpio, GLPR); + unsigned long flags; + u32 value; + + raw_spin_lock_irqsave(&priv->lock, flags); + + if (type & IRQ_TYPE_EDGE_RISING) + value = readl(grer) | BIT(gpio % 32); + else + value = readl(grer) & ~BIT(gpio % 32); + writel(value, grer); + + if (type & IRQ_TYPE_EDGE_FALLING) + value = readl(gfer) | BIT(gpio % 32); + else + value = readl(gfer) & ~BIT(gpio % 32); + writel(value, gfer); + + /* + * To prevent glitches from triggering an unintended level interrupt, + * configure GLPR register first and then configure GITR. + */ + if (type & IRQ_TYPE_LEVEL_LOW) + value = readl(glpr) | BIT(gpio % 32); + else + value = readl(glpr) & ~BIT(gpio % 32); + writel(value, glpr); + + if (type & IRQ_TYPE_LEVEL_MASK) { + value = readl(gitr) | BIT(gpio % 32); + writel(value, gitr); + + irq_set_handler_locked(d, handle_level_irq); + } else if (type & IRQ_TYPE_EDGE_BOTH) { + value = readl(gitr) & ~BIT(gpio % 32); + writel(value, gitr); + + irq_set_handler_locked(d, handle_edge_irq); + } + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static int mrfld_irq_set_wake(struct irq_data *d, unsigned int on) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct mrfld_gpio *priv = gpiochip_get_data(gc); + u32 gpio = irqd_to_hwirq(d); + void __iomem *gwmr = gpio_reg(&priv->chip, gpio, GWMR); + void __iomem *gwsr = gpio_reg(&priv->chip, gpio, GWSR); + unsigned long flags; + u32 value; + + raw_spin_lock_irqsave(&priv->lock, flags); + + /* Clear the existing wake status */ + writel(BIT(gpio % 32), gwsr); + + if (on) + value = readl(gwmr) | BIT(gpio % 32); + else + value = readl(gwmr) & ~BIT(gpio % 32); + writel(value, gwmr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + dev_dbg(priv->dev, "%sable wake for gpio %u\n", on ? "en" : "dis", gpio); + return 0; +} + +static struct irq_chip mrfld_irqchip = { + .name = "gpio-merrifield", + .irq_ack = mrfld_irq_ack, + .irq_mask = mrfld_irq_mask, + .irq_unmask = mrfld_irq_unmask, + .irq_set_type = mrfld_irq_set_type, + .irq_set_wake = mrfld_irq_set_wake, +}; + +static void mrfld_irq_handler(struct irq_desc *desc) +{ + struct gpio_chip *gc = irq_desc_get_handler_data(desc); + struct mrfld_gpio *priv = gpiochip_get_data(gc); + struct irq_chip *irqchip = irq_desc_get_chip(desc); + unsigned long base, gpio; + + chained_irq_enter(irqchip, desc); + + /* Check GPIO controller to check which pin triggered the interrupt */ + for (base = 0; base < priv->chip.ngpio; base += 32) { + void __iomem *gisr = gpio_reg(&priv->chip, base, GISR); + void __iomem *gimr = gpio_reg(&priv->chip, base, GIMR); + unsigned long pending, enabled; + + pending = readl(gisr); + enabled = readl(gimr); + + /* Only interrupts that are enabled */ + pending &= enabled; + + for_each_set_bit(gpio, &pending, 32) { + unsigned int irq; + + irq = irq_find_mapping(gc->irqdomain, base + gpio); + generic_handle_irq(irq); + } + } + + chained_irq_exit(irqchip, desc); +} + +static void mrfld_irq_init_hw(struct mrfld_gpio *priv) +{ + void __iomem *reg; + unsigned int base; + + for (base = 0; base < priv->chip.ngpio; base += 32) { + /* Clear the rising-edge detect register */ + reg = gpio_reg(&priv->chip, base, GRER); + writel(0, reg); + /* Clear the falling-edge detect register */ + reg = gpio_reg(&priv->chip, base, GFER); + writel(0, reg); + } +} + +static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + const struct mrfld_gpio_pinrange *range; + struct mrfld_gpio *priv; + u32 gpio_base, irq_base; + void __iomem *base; + unsigned int i; + int retval; + + retval = pcim_enable_device(pdev); + if (retval) + return retval; + + retval = pcim_iomap_regions(pdev, BIT(1) | BIT(0), pci_name(pdev)); + if (retval) { + dev_err(&pdev->dev, "I/O memory mapping error\n"); + return retval; + } + + base = pcim_iomap_table(pdev)[1]; + + irq_base = readl(base); + gpio_base = readl(sizeof(u32) + base); + + /* Release the IO mapping, since we already get the info from BAR1 */ + pcim_iounmap_regions(pdev, BIT(1)); + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) { + dev_err(&pdev->dev, "can't allocate chip data\n"); + return -ENOMEM; + } + + priv->dev = &pdev->dev; + priv->reg_base = pcim_iomap_table(pdev)[0]; + + priv->chip.label = dev_name(&pdev->dev); + priv->chip.parent = &pdev->dev; + priv->chip.request = gpiochip_generic_request; + priv->chip.free = gpiochip_generic_free; + priv->chip.direction_input = mrfld_gpio_direction_input; + priv->chip.direction_output = mrfld_gpio_direction_output; + priv->chip.get = mrfld_gpio_get; + priv->chip.set = mrfld_gpio_set; + priv->chip.base = gpio_base; + priv->chip.ngpio = MRFLD_NGPIO; + priv->chip.can_sleep = false; + + raw_spin_lock_init(&priv->lock); + + pci_set_drvdata(pdev, priv); + retval = devm_gpiochip_add_data(&pdev->dev, &priv->chip, priv); + if (retval) { + dev_err(&pdev->dev, "gpiochip_add error %d\n", retval); + return retval; + } + + for (i = 0; i < ARRAY_SIZE(mrfld_gpio_ranges); i++) { + range = &mrfld_gpio_ranges[i]; + retval = gpiochip_add_pin_range(&priv->chip, + "pinctrl-merrifield", + range->gpio_base, + range->pin_base, + range->npins); + if (retval) { + dev_err(&pdev->dev, "failed to add GPIO pin range\n"); + return retval; + } + } + + retval = gpiochip_irqchip_add(&priv->chip, &mrfld_irqchip, irq_base, + handle_simple_irq, IRQ_TYPE_NONE); + if (retval) { + dev_err(&pdev->dev, "could not connect irqchip to gpiochip\n"); + return retval; + } + + mrfld_irq_init_hw(priv); + + gpiochip_set_chained_irqchip(&priv->chip, &mrfld_irqchip, pdev->irq, + mrfld_irq_handler); + + return 0; +} + +static const struct pci_device_id mrfld_gpio_ids[] = { + { PCI_VDEVICE(INTEL, 0x1199) }, + { } +}; +MODULE_DEVICE_TABLE(pci, mrfld_gpio_ids); + +static struct pci_driver mrfld_gpio_driver = { + .name = "gpio-merrifield", + .id_table = mrfld_gpio_ids, + .probe = mrfld_gpio_probe, +}; + +module_pci_driver(mrfld_gpio_driver); + +MODULE_AUTHOR("Andy Shevchenko "); +MODULE_DESCRIPTION("Intel Merrifield SoC GPIO driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpio/gpio-mmio.c b/drivers/gpio/gpio-mmio.c index 6c1cb3b8c02c..6ec144baeb11 100644 --- a/drivers/gpio/gpio-mmio.c +++ b/drivers/gpio/gpio-mmio.c @@ -61,6 +61,8 @@ o ` ~~~~\___/~~~~ ` controller in FPGA is ,.` #include #include #include +#include +#include static void bgpio_write8(void __iomem *reg, unsigned long data) { @@ -569,6 +571,41 @@ static void __iomem *bgpio_map(struct platform_device *pdev, return devm_ioremap_resource(&pdev->dev, r); } +#ifdef CONFIG_OF +static const struct of_device_id bgpio_of_match[] = { + { .compatible = "wd,mbl-gpio" }, + { } +}; +MODULE_DEVICE_TABLE(of, bgpio_of_match); + +static struct bgpio_pdata *bgpio_parse_dt(struct platform_device *pdev, + unsigned long *flags) +{ + struct bgpio_pdata *pdata; + + if (!of_match_device(bgpio_of_match, &pdev->dev)) + return NULL; + + pdata = devm_kzalloc(&pdev->dev, sizeof(struct bgpio_pdata), + GFP_KERNEL); + if (!pdata) + return ERR_PTR(-ENOMEM); + + pdata->base = -1; + + if (of_property_read_bool(pdev->dev.of_node, "no-output")) + *flags |= BGPIOF_NO_OUTPUT; + + return pdata; +} +#else +static struct bgpio_pdata *bgpio_parse_dt(struct platform_device *pdev, + unsigned long *flags) +{ + return NULL; +} +#endif /* CONFIG_OF */ + static int bgpio_pdev_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -579,10 +616,19 @@ static int bgpio_pdev_probe(struct platform_device *pdev) void __iomem *dirout; void __iomem *dirin; unsigned long sz; - unsigned long flags = pdev->id_entry->driver_data; + unsigned long flags = 0; int err; struct gpio_chip *gc; - struct bgpio_pdata *pdata = dev_get_platdata(dev); + struct bgpio_pdata *pdata; + + pdata = bgpio_parse_dt(pdev, &flags); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + + if (!pdata) { + pdata = dev_get_platdata(dev); + flags = pdev->id_entry->driver_data; + } r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dat"); if (!r) @@ -646,6 +692,7 @@ MODULE_DEVICE_TABLE(platform, bgpio_id_table); static struct platform_driver bgpio_driver = { .driver = { .name = "basic-mmio-gpio", + .of_match_table = of_match_ptr(bgpio_of_match), }, .id_table = bgpio_id_table, .probe = bgpio_pdev_probe, diff --git a/drivers/gpio/gpio-palmas.c b/drivers/gpio/gpio-palmas.c index e248707ca39e..839474430229 100644 --- a/drivers/gpio/gpio-palmas.c +++ b/drivers/gpio/gpio-palmas.c @@ -208,7 +208,6 @@ static int palmas_gpio_probe(struct platform_device *pdev) static struct platform_driver palmas_gpio_driver = { .driver.name = "palmas-gpio", - .driver.owner = THIS_MODULE, .driver.of_match_table = of_palmas_gpio_match, .probe = palmas_gpio_probe, }; diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index 5e3be32ebb8d..02f2a5621bb0 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -44,7 +44,7 @@ #define PCA_GPIO_MASK 0x00FF #define PCA_INT 0x0100 -#define PCA_PCAL 0x0200 +#define PCA_PCAL 0x0200 #define PCA953X_TYPE 0x1000 #define PCA957X_TYPE 0x2000 #define PCA_TYPE_MASK 0xF000 @@ -67,6 +67,8 @@ static const struct i2c_device_id pca953x_id[] = { { "pca9575", 16 | PCA957X_TYPE | PCA_INT, }, { "pca9698", 40 | PCA953X_TYPE, }, + { "pcal9555a", 16 | PCA953X_TYPE | PCA_INT | PCA_PCAL, }, + { "max7310", 8 | PCA953X_TYPE, }, { "max7312", 16 | PCA953X_TYPE | PCA_INT, }, { "max7313", 16 | PCA953X_TYPE | PCA_INT, }, @@ -90,7 +92,7 @@ MODULE_DEVICE_TABLE(acpi, pca953x_acpi_ids); #define MAX_BANK 5 #define BANK_SZ 8 -#define NBANK(chip) (chip->gpio_chip.ngpio / BANK_SZ) +#define NBANK(chip) DIV_ROUND_UP(chip->gpio_chip.ngpio, BANK_SZ) struct pca953x_chip { unsigned gpio_start; @@ -135,7 +137,7 @@ static int pca953x_read_single(struct pca953x_chip *chip, int reg, u32 *val, static int pca953x_write_single(struct pca953x_chip *chip, int reg, u32 val, int off) { - int ret = 0; + int ret; int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ); int offset = off / BANK_SZ; @@ -163,10 +165,13 @@ static int pca953x_write_regs(struct pca953x_chip *chip, int reg, u8 *val) NBANK(chip), val); } else { switch (chip->chip_type) { - case PCA953X_TYPE: - ret = i2c_smbus_write_word_data(chip->client, - reg << 1, cpu_to_le16(get_unaligned((u16 *)val))); + case PCA953X_TYPE: { + __le16 word = cpu_to_le16(get_unaligned((u16 *)val)); + + ret = i2c_smbus_write_word_data(chip->client, reg << 1, + (__force u16)word); break; + } case PCA957X_TYPE: ret = i2c_smbus_write_byte_data(chip->client, reg << 1, val[0]); @@ -235,7 +240,6 @@ static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned off) goto exit; chip->reg_direction[off / BANK_SZ] = reg_val; - ret = 0; exit: mutex_unlock(&chip->i2c_lock); return ret; @@ -286,7 +290,6 @@ static int pca953x_gpio_direction_output(struct gpio_chip *gc, goto exit; chip->reg_direction[off / BANK_SZ] = reg_val; - ret = 0; exit: mutex_unlock(&chip->i2c_lock); return ret; @@ -351,7 +354,6 @@ exit: mutex_unlock(&chip->i2c_lock); } - static void pca953x_gpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, unsigned long *bits) { @@ -820,7 +822,7 @@ static int pca953x_remove(struct i2c_client *client) { struct pca953x_platform_data *pdata = dev_get_platdata(&client->dev); struct pca953x_chip *chip = i2c_get_clientdata(client); - int ret = 0; + int ret; if (pdata && pdata->teardown) { ret = pdata->teardown(client, chip->gpio_chip.base, @@ -861,6 +863,7 @@ static const struct of_device_id pca953x_dt_ids[] = { { .compatible = "maxim,max7315", .data = OF_953X( 8, PCA_INT), }, { .compatible = "ti,pca6107", .data = OF_953X( 8, PCA_INT), }, + { .compatible = "ti,pca9536", .data = OF_953X( 4, 0), }, { .compatible = "ti,tca6408", .data = OF_953X( 8, PCA_INT), }, { .compatible = "ti,tca6416", .data = OF_953X(16, PCA_INT), }, { .compatible = "ti,tca6424", .data = OF_953X(24, PCA_INT), }, diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c index 169c09aa33c8..d168410e2338 100644 --- a/drivers/gpio/gpio-pcf857x.c +++ b/drivers/gpio/gpio-pcf857x.c @@ -440,6 +440,14 @@ static int pcf857x_remove(struct i2c_client *client) return status; } +static void pcf857x_shutdown(struct i2c_client *client) +{ + struct pcf857x *gpio = i2c_get_clientdata(client); + + /* Drive all the I/O lines high */ + gpio->write(gpio->client, BIT(gpio->chip.ngpio) - 1); +} + static struct i2c_driver pcf857x_driver = { .driver = { .name = "pcf857x", @@ -447,6 +455,7 @@ static struct i2c_driver pcf857x_driver = { }, .probe = pcf857x_probe, .remove = pcf857x_remove, + .shutdown = pcf857x_shutdown, .id_table = pcf857x_id, }; diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c index 681c93fb9e70..b96e0b466f74 100644 --- a/drivers/gpio/gpio-rcar.c +++ b/drivers/gpio/gpio-rcar.c @@ -334,6 +334,9 @@ static const struct of_device_id gpio_rcar_of_table[] = { }, { .compatible = "renesas,gpio-r8a7791", .data = &gpio_rcar_info_gen2, + }, { + .compatible = "renesas,gpio-r8a7792", + .data = &gpio_rcar_info_gen2, }, { .compatible = "renesas,gpio-r8a7793", .data = &gpio_rcar_info_gen2, diff --git a/drivers/gpio/gpio-rdc321x.c b/drivers/gpio/gpio-rdc321x.c index ec945b90f54d..cbf0f9e6465b 100644 --- a/drivers/gpio/gpio-rdc321x.c +++ b/drivers/gpio/gpio-rdc321x.c @@ -200,7 +200,6 @@ static int rdc321x_gpio_probe(struct platform_device *pdev) static struct platform_driver rdc321x_gpio_driver = { .driver.name = "rdc321x-gpio", - .driver.owner = THIS_MODULE, .probe = rdc321x_gpio_probe, }; diff --git a/drivers/gpio/gpio-sch.c b/drivers/gpio/gpio-sch.c index e85e7539cf5d..eb43ae4835c1 100644 --- a/drivers/gpio/gpio-sch.c +++ b/drivers/gpio/gpio-sch.c @@ -61,9 +61,8 @@ static unsigned sch_gpio_bit(struct sch_gpio *sch, unsigned gpio) return gpio % 8; } -static int sch_gpio_reg_get(struct gpio_chip *gc, unsigned gpio, unsigned reg) +static int sch_gpio_reg_get(struct sch_gpio *sch, unsigned gpio, unsigned reg) { - struct sch_gpio *sch = gpiochip_get_data(gc); unsigned short offset, bit; u8 reg_val; @@ -75,10 +74,9 @@ static int sch_gpio_reg_get(struct gpio_chip *gc, unsigned gpio, unsigned reg) return reg_val; } -static void sch_gpio_reg_set(struct gpio_chip *gc, unsigned gpio, unsigned reg, +static void sch_gpio_reg_set(struct sch_gpio *sch, unsigned gpio, unsigned reg, int val) { - struct sch_gpio *sch = gpiochip_get_data(gc); unsigned short offset, bit; u8 reg_val; @@ -98,14 +96,15 @@ static int sch_gpio_direction_in(struct gpio_chip *gc, unsigned gpio_num) struct sch_gpio *sch = gpiochip_get_data(gc); spin_lock(&sch->lock); - sch_gpio_reg_set(gc, gpio_num, GIO, 1); + sch_gpio_reg_set(sch, gpio_num, GIO, 1); spin_unlock(&sch->lock); return 0; } static int sch_gpio_get(struct gpio_chip *gc, unsigned gpio_num) { - return sch_gpio_reg_get(gc, gpio_num, GLV); + struct sch_gpio *sch = gpiochip_get_data(gc); + return sch_gpio_reg_get(sch, gpio_num, GLV); } static void sch_gpio_set(struct gpio_chip *gc, unsigned gpio_num, int val) @@ -113,7 +112,7 @@ static void sch_gpio_set(struct gpio_chip *gc, unsigned gpio_num, int val) struct sch_gpio *sch = gpiochip_get_data(gc); spin_lock(&sch->lock); - sch_gpio_reg_set(gc, gpio_num, GLV, val); + sch_gpio_reg_set(sch, gpio_num, GLV, val); spin_unlock(&sch->lock); } @@ -123,7 +122,7 @@ static int sch_gpio_direction_out(struct gpio_chip *gc, unsigned gpio_num, struct sch_gpio *sch = gpiochip_get_data(gc); spin_lock(&sch->lock); - sch_gpio_reg_set(gc, gpio_num, GIO, 0); + sch_gpio_reg_set(sch, gpio_num, GIO, 0); spin_unlock(&sch->lock); /* @@ -182,13 +181,13 @@ static int sch_gpio_probe(struct platform_device *pdev) * GPIO7 is configured by the CMC as SLPIOVR * Enable GPIO[9:8] core powered gpios explicitly */ - sch_gpio_reg_set(&sch->chip, 8, GEN, 1); - sch_gpio_reg_set(&sch->chip, 9, GEN, 1); + sch_gpio_reg_set(sch, 8, GEN, 1); + sch_gpio_reg_set(sch, 9, GEN, 1); /* * SUS_GPIO[2:0] enabled by default * Enable SUS_GPIO3 resume powered gpio explicitly */ - sch_gpio_reg_set(&sch->chip, 13, GEN, 1); + sch_gpio_reg_set(sch, 13, GEN, 1); break; case PCI_DEVICE_ID_INTEL_ITC_LPC: diff --git a/drivers/gpio/gpio-sch311x.c b/drivers/gpio/gpio-sch311x.c index a03b38ee2e02..b96990c262a1 100644 --- a/drivers/gpio/gpio-sch311x.c +++ b/drivers/gpio/gpio-sch311x.c @@ -296,7 +296,6 @@ static int sch311x_gpio_remove(struct platform_device *pdev) static struct platform_driver sch311x_gpio_driver = { .driver.name = DRV_NAME, - .driver.owner = THIS_MODULE, .probe = sch311x_gpio_probe, .remove = sch311x_gpio_remove, }; diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c index 6f7af28b8966..f675132de10e 100644 --- a/drivers/gpio/gpio-stmpe.c +++ b/drivers/gpio/gpio-stmpe.c @@ -68,6 +68,22 @@ static void stmpe_gpio_set(struct gpio_chip *chip, unsigned offset, int val) stmpe_reg_write(stmpe, reg, mask); } +static int stmpe_gpio_get_direction(struct gpio_chip *chip, + unsigned offset) +{ + struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip); + struct stmpe *stmpe = stmpe_gpio->stmpe; + u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB] - (offset / 8); + u8 mask = 1 << (offset % 8); + int ret; + + ret = stmpe_reg_read(stmpe, reg); + if (ret < 0) + return ret; + + return !(ret & mask); +} + static int stmpe_gpio_direction_output(struct gpio_chip *chip, unsigned offset, int val) { @@ -106,6 +122,7 @@ static int stmpe_gpio_request(struct gpio_chip *chip, unsigned offset) static struct gpio_chip template_chip = { .label = "stmpe", .owner = THIS_MODULE, + .get_direction = stmpe_gpio_get_direction, .direction_input = stmpe_gpio_direction_input, .get = stmpe_gpio_get, .direction_output = stmpe_gpio_direction_output, @@ -416,7 +433,6 @@ static struct platform_driver stmpe_gpio_driver = { .driver = { .suppress_bind_attrs = true, .name = "stmpe-gpio", - .owner = THIS_MODULE, }, .probe = stmpe_gpio_probe, }; diff --git a/drivers/gpio/gpio-syscon.c b/drivers/gpio/gpio-syscon.c index 24b6d643ecdb..537cec7583fc 100644 --- a/drivers/gpio/gpio-syscon.c +++ b/drivers/gpio/gpio-syscon.c @@ -129,7 +129,7 @@ static int syscon_gpio_dir_out(struct gpio_chip *chip, unsigned offset, int val) static const struct syscon_gpio_data clps711x_mctrl_gpio = { /* ARM CLPS711X SYSFLG1 Bits 8-10 */ - .compatible = "cirrus,clps711x-syscon1", + .compatible = "cirrus,ep7209-syscon1", .flags = GPIO_SYSCON_FEAT_IN, .bit_count = 3, .dat_bit_offset = 0x40 * 8 + 8, @@ -168,7 +168,7 @@ static const struct syscon_gpio_data keystone_dsp_gpio = { static const struct of_device_id syscon_gpio_ids[] = { { - .compatible = "cirrus,clps711x-mctrl-gpio", + .compatible = "cirrus,ep7209-mctrl-gpio", .data = &clps711x_mctrl_gpio, }, { diff --git a/drivers/gpio/gpio-tc3589x.c b/drivers/gpio/gpio-tc3589x.c index 2e35ed3abbcf..8b3659352e49 100644 --- a/drivers/gpio/gpio-tc3589x.c +++ b/drivers/gpio/gpio-tc3589x.c @@ -343,7 +343,6 @@ static int tc3589x_gpio_probe(struct platform_device *pdev) static struct platform_driver tc3589x_gpio_driver = { .driver.name = "tc3589x-gpio", - .driver.owner = THIS_MODULE, .probe = tc3589x_gpio_probe, }; diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c index ec891a27952f..661b0e34e067 100644 --- a/drivers/gpio/gpio-tegra.c +++ b/drivers/gpio/gpio-tegra.c @@ -98,7 +98,6 @@ struct tegra_gpio_info { const struct tegra_gpio_soc_config *soc; struct gpio_chip gc; struct irq_chip ic; - struct lock_class_key lock_class; u32 bank_count; }; @@ -547,6 +546,12 @@ static const struct dev_pm_ops tegra_gpio_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(tegra_gpio_suspend, tegra_gpio_resume) }; +/* + * This lock class tells lockdep that GPIO irqs are in a different category + * than their parents, so it won't report false recursion. + */ +static struct lock_class_key gpio_lock_class; + static int tegra_gpio_probe(struct platform_device *pdev) { const struct tegra_gpio_soc_config *config; @@ -660,7 +665,7 @@ static int tegra_gpio_probe(struct platform_device *pdev) bank = &tgi->bank_info[GPIO_BANK(gpio)]; - irq_set_lockdep_class(irq, &tgi->lock_class); + irq_set_lockdep_class(irq, &gpio_lock_class); irq_set_chip_data(irq, bank); irq_set_chip_and_handler(irq, &tgi->ic, handle_simple_irq); } diff --git a/drivers/gpio/gpio-tps65218.c b/drivers/gpio/gpio-tps65218.c index 0eaeac8de9de..1c09a19ae10c 100644 --- a/drivers/gpio/gpio-tps65218.c +++ b/drivers/gpio/gpio-tps65218.c @@ -230,6 +230,12 @@ static const struct of_device_id tps65218_dt_match[] = { }; MODULE_DEVICE_TABLE(of, tps65218_dt_match); +static const struct platform_device_id tps65218_gpio_id_table[] = { + { "tps65218-gpio", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, tps65218_gpio_id_table); + static struct platform_driver tps65218_gpio_driver = { .driver = { .name = "tps65218-gpio", @@ -237,6 +243,7 @@ static struct platform_driver tps65218_gpio_driver = { }, .probe = tps65218_gpio_probe, .remove = tps65218_gpio_remove, + .id_table = tps65218_gpio_id_table, }; module_platform_driver(tps65218_gpio_driver); diff --git a/drivers/gpio/gpio-tps6586x.c b/drivers/gpio/gpio-tps6586x.c index 6b15e68a314f..042b9a20781a 100644 --- a/drivers/gpio/gpio-tps6586x.c +++ b/drivers/gpio/gpio-tps6586x.c @@ -131,7 +131,6 @@ static int tps6586x_gpio_probe(struct platform_device *pdev) static struct platform_driver tps6586x_gpio_driver = { .driver.name = "tps6586x-gpio", - .driver.owner = THIS_MODULE, .probe = tps6586x_gpio_probe, }; diff --git a/drivers/gpio/gpio-tps65910.c b/drivers/gpio/gpio-tps65910.c index 0ae6a5a54ea8..e63d7dabf78b 100644 --- a/drivers/gpio/gpio-tps65910.c +++ b/drivers/gpio/gpio-tps65910.c @@ -184,7 +184,6 @@ skip_init: static struct platform_driver tps65910_gpio_driver = { .driver.name = "tps65910-gpio", - .driver.owner = THIS_MODULE, .probe = tps65910_gpio_probe, }; diff --git a/drivers/gpio/gpio-viperboard.c b/drivers/gpio/gpio-viperboard.c index dec47aafd5cd..e6d1328dddfa 100644 --- a/drivers/gpio/gpio-viperboard.c +++ b/drivers/gpio/gpio-viperboard.c @@ -440,7 +440,6 @@ static int vprbrd_gpio_probe(struct platform_device *pdev) static struct platform_driver vprbrd_gpio_driver = { .driver.name = "viperboard-gpio", - .driver.owner = THIS_MODULE, .probe = vprbrd_gpio_probe, }; diff --git a/drivers/gpio/gpio-wm831x.c b/drivers/gpio/gpio-wm831x.c index 41ec7834059a..21f97bcd0062 100644 --- a/drivers/gpio/gpio-wm831x.c +++ b/drivers/gpio/gpio-wm831x.c @@ -296,7 +296,6 @@ static int wm831x_gpio_probe(struct platform_device *pdev) static struct platform_driver wm831x_gpio_driver = { .driver.name = "wm831x-gpio", - .driver.owner = THIS_MODULE, .probe = wm831x_gpio_probe, }; diff --git a/drivers/gpio/gpio-wm8350.c b/drivers/gpio/gpio-wm8350.c index 07d45a3b205a..e9765707d5c1 100644 --- a/drivers/gpio/gpio-wm8350.c +++ b/drivers/gpio/gpio-wm8350.c @@ -139,7 +139,6 @@ static int wm8350_gpio_probe(struct platform_device *pdev) static struct platform_driver wm8350_gpio_driver = { .driver.name = "wm8350-gpio", - .driver.owner = THIS_MODULE, .probe = wm8350_gpio_probe, }; diff --git a/drivers/gpio/gpio-wm8994.c b/drivers/gpio/gpio-wm8994.c index 744af388c949..2457aac8592e 100644 --- a/drivers/gpio/gpio-wm8994.c +++ b/drivers/gpio/gpio-wm8994.c @@ -299,7 +299,6 @@ static int wm8994_gpio_probe(struct platform_device *pdev) static struct platform_driver wm8994_gpio_driver = { .driver.name = "wm8994-gpio", - .driver.owner = THIS_MODULE, .probe = wm8994_gpio_probe, }; diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c index d0fbb7f99523..14b2a62338ea 100644 --- a/drivers/gpio/gpio-xilinx.c +++ b/drivers/gpio/gpio-xilinx.c @@ -132,6 +132,53 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) spin_unlock_irqrestore(&chip->gpio_lock[index], flags); } +/** + * xgpio_set_multiple - Write the specified signals of the GPIO device. + * @gc: Pointer to gpio_chip device structure. + * @mask: Mask of the GPIOS to modify. + * @bits: Value to be wrote on each GPIO + * + * This function writes the specified values into the specified signals of the + * GPIO devices. + */ +static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, + unsigned long *bits) +{ + unsigned long flags; + struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); + struct xgpio_instance *chip = gpiochip_get_data(gc); + int index = xgpio_index(chip, 0); + int offset, i; + + spin_lock_irqsave(&chip->gpio_lock[index], flags); + + /* Write to GPIO signals */ + for (i = 0; i < gc->ngpio; i++) { + if (*mask == 0) + break; + if (index != xgpio_index(chip, i)) { + xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET + + xgpio_regoffset(chip, i), + chip->gpio_state[index]); + spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + index = xgpio_index(chip, i); + spin_lock_irqsave(&chip->gpio_lock[index], flags); + } + if (__test_and_clear_bit(i, mask)) { + offset = xgpio_offset(chip, i); + if (test_bit(i, bits)) + chip->gpio_state[index] |= BIT(offset); + else + chip->gpio_state[index] &= ~BIT(offset); + } + } + + xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET + + xgpio_regoffset(chip, i), chip->gpio_state[index]); + + spin_unlock_irqrestore(&chip->gpio_lock[index], flags); +} + /** * xgpio_dir_in - Set the direction of the specified GPIO signal as input. * @gc: Pointer to gpio_chip device structure. @@ -306,6 +353,7 @@ static int xgpio_probe(struct platform_device *pdev) chip->mmchip.gc.direction_output = xgpio_dir_out; chip->mmchip.gc.get = xgpio_get; chip->mmchip.gc.set = xgpio_set; + chip->mmchip.gc.set_multiple = xgpio_set_multiple; chip->mmchip.save_regs = xgpio_save_regs; diff --git a/drivers/gpio/gpio-xlp.c b/drivers/gpio/gpio-xlp.c index 1a33a19d95b9..4620d050e5a8 100644 --- a/drivers/gpio/gpio-xlp.c +++ b/drivers/gpio/gpio-xlp.c @@ -19,6 +19,7 @@ #include #include #include +#include /* * XLP GPIO has multiple 32 bit registers for each feature where each register @@ -299,7 +300,6 @@ static int xlp_gpio_probe(struct platform_device *pdev) struct gpio_chip *gc; struct resource *iores; struct xlp_gpio_priv *priv; - const struct of_device_id *of_id; void __iomem *gpio_base; int irq_base, irq, err; int ngpio; @@ -321,13 +321,26 @@ static int xlp_gpio_probe(struct platform_device *pdev) if (irq < 0) return irq; - of_id = of_match_device(xlp_gpio_of_ids, &pdev->dev); - if (!of_id) { - dev_err(&pdev->dev, "Failed to get soc type!\n"); - return -ENODEV; - } + if (pdev->dev.of_node) { + const struct of_device_id *of_id; - soc_type = (uintptr_t) of_id->data; + of_id = of_match_device(xlp_gpio_of_ids, &pdev->dev); + if (!of_id) { + dev_err(&pdev->dev, "Unable to match OF ID\n"); + return -ENODEV; + } + soc_type = (uintptr_t) of_id->data; + } else { + const struct acpi_device_id *acpi_id; + + acpi_id = acpi_match_device(pdev->dev.driver->acpi_match_table, + &pdev->dev); + if (!acpi_id || !acpi_id->driver_data) { + dev_err(&pdev->dev, "Unable to match ACPI ID\n"); + return -ENODEV; + } + soc_type = (uintptr_t) acpi_id->driver_data; + } switch (soc_type) { case XLP_GPIO_VARIANT_XLP832: @@ -388,14 +401,16 @@ static int xlp_gpio_probe(struct platform_device *pdev) gc->get = xlp_gpio_get; spin_lock_init(&priv->lock); - /* XLP has fixed IRQ range for GPIO interrupts */ - if (soc_type == GPIO_VARIANT_VULCAN) - irq_base = irq_alloc_descs(-1, 0, gc->ngpio, 0); - else + + /* XLP(MIPS) has fixed range for GPIO IRQs, Vulcan(ARM64) does not */ + if (soc_type != GPIO_VARIANT_VULCAN) { irq_base = irq_alloc_descs(-1, XLP_GPIO_IRQ_BASE, gc->ngpio, 0); - if (irq_base < 0) { - dev_err(&pdev->dev, "Failed to allocate IRQ numbers\n"); - return irq_base; + if (irq_base < 0) { + dev_err(&pdev->dev, "Failed to allocate IRQ numbers\n"); + return irq_base; + } + } else { + irq_base = 0; } err = gpiochip_add_data(gc, priv); @@ -423,10 +438,19 @@ out_free_desc: return err; } +#ifdef CONFIG_ACPI +static const struct acpi_device_id xlp_gpio_acpi_match[] = { + { "BRCM9006", GPIO_VARIANT_VULCAN }, + {}, +}; +MODULE_DEVICE_TABLE(acpi, xlp_gpio_acpi_match); +#endif + static struct platform_driver xlp_gpio_driver = { .driver = { .name = "xlp-gpio", .of_match_table = xlp_gpio_of_ids, + .acpi_match_table = ACPI_PTR(xlp_gpio_acpi_match), }, .probe = xlp_gpio_probe, }; diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 2dc52585e3f2..af514618d7fb 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -836,6 +836,7 @@ void acpi_gpiochip_add(struct gpio_chip *chip) } acpi_gpiochip_request_regions(acpi_gpio); + acpi_walk_dep_device_list(handle); } void acpi_gpiochip_remove(struct gpio_chip *chip) diff --git a/drivers/gpio/gpiolib-legacy.c b/drivers/gpio/gpiolib-legacy.c index 3a5c7011ad3b..8b830996fe02 100644 --- a/drivers/gpio/gpiolib-legacy.c +++ b/drivers/gpio/gpiolib-legacy.c @@ -28,6 +28,10 @@ int gpio_request_one(unsigned gpio, unsigned long flags, const char *label) if (!desc && gpio_is_valid(gpio)) return -EPROBE_DEFER; + err = gpiod_request(desc, label); + if (err) + return err; + if (flags & GPIOF_OPEN_DRAIN) set_bit(FLAG_OPEN_DRAIN, &desc->flags); @@ -37,10 +41,6 @@ int gpio_request_one(unsigned gpio, unsigned long flags, const char *label) if (flags & GPIOF_ACTIVE_LOW) set_bit(FLAG_ACTIVE_LOW, &desc->flags); - err = gpiod_request(desc, label); - if (err) - return err; - if (flags & GPIOF_DIR_IN) err = gpiod_direction_input(desc); else diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 4aabddb38b59..75e7b3919ea7 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -27,38 +27,30 @@ #include "gpiolib.h" -/* Private data structure for of_gpiochip_find_and_xlate */ -struct gg_data { - enum of_gpio_flags *flags; - struct of_phandle_args gpiospec; +static int of_gpiochip_match_node(struct gpio_chip *chip, void *data) +{ + return chip->gpiodev->dev.of_node == data; +} - struct gpio_desc *out_gpio; -}; +static struct gpio_chip *of_find_gpiochip_by_node(struct device_node *np) +{ + return gpiochip_find(np, of_gpiochip_match_node); +} -/* Private function for resolving node pointer to gpio_chip */ -static int of_gpiochip_find_and_xlate(struct gpio_chip *gc, void *data) +static struct gpio_desc *of_xlate_and_get_gpiod_flags(struct gpio_chip *chip, + struct of_phandle_args *gpiospec, + enum of_gpio_flags *flags) { - struct gg_data *gg_data = data; int ret; - if ((gc->of_node != gg_data->gpiospec.np) || - (gc->of_gpio_n_cells != gg_data->gpiospec.args_count) || - (!gc->of_xlate)) - return false; - - ret = gc->of_xlate(gc, &gg_data->gpiospec, gg_data->flags); - if (ret < 0) { - /* We've found a gpio chip, but the translation failed. - * Store translation error in out_gpio. - * Return false to keep looking, as more than one gpio chip - * could be registered per of-node. - */ - gg_data->out_gpio = ERR_PTR(ret); - return false; - } - - gg_data->out_gpio = gpiochip_get_desc(gc, ret); - return true; + if (chip->of_gpio_n_cells != gpiospec->args_count) + return ERR_PTR(-EINVAL); + + ret = chip->of_xlate(chip, gpiospec, flags); + if (ret < 0) + return ERR_PTR(ret); + + return gpiochip_get_desc(chip, ret); } /** @@ -75,34 +67,37 @@ static int of_gpiochip_find_and_xlate(struct gpio_chip *gc, void *data) struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np, const char *propname, int index, enum of_gpio_flags *flags) { - /* Return -EPROBE_DEFER to support probe() functions to be called - * later when the GPIO actually becomes available - */ - struct gg_data gg_data = { - .flags = flags, - .out_gpio = ERR_PTR(-EPROBE_DEFER) - }; + struct of_phandle_args gpiospec; + struct gpio_chip *chip; + struct gpio_desc *desc; int ret; - /* .of_xlate might decide to not fill in the flags, so clear it. */ - if (flags) - *flags = 0; - ret = of_parse_phandle_with_args(np, propname, "#gpio-cells", index, - &gg_data.gpiospec); + &gpiospec); if (ret) { pr_debug("%s: can't parse '%s' property of node '%s[%d]'\n", __func__, propname, np->full_name, index); return ERR_PTR(ret); } - gpiochip_find(&gg_data, of_gpiochip_find_and_xlate); + chip = of_find_gpiochip_by_node(gpiospec.np); + if (!chip) { + desc = ERR_PTR(-EPROBE_DEFER); + goto out; + } + + desc = of_xlate_and_get_gpiod_flags(chip, &gpiospec, flags); + if (IS_ERR(desc)) + goto out; - of_node_put(gg_data.gpiospec.np); pr_debug("%s: parsed '%s' property of node '%s[%d]' - status (%d)\n", __func__, propname, np->full_name, index, - PTR_ERR_OR_ZERO(gg_data.out_gpio)); - return gg_data.out_gpio; + PTR_ERR_OR_ZERO(desc)); + +out: + of_node_put(gpiospec.np); + + return desc; } int of_get_named_gpio_flags(struct device_node *np, const char *list_name, @@ -122,6 +117,7 @@ EXPORT_SYMBOL(of_get_named_gpio_flags); /** * of_parse_own_gpio() - Get a GPIO hog descriptor, names and flags for GPIO API * @np: device node to get GPIO from + * @chip: GPIO chip whose hog is parsed * @name: GPIO line name * @lflags: gpio_lookup_flags - returned from of_find_gpio() or * of_parse_own_gpio() @@ -131,19 +127,19 @@ EXPORT_SYMBOL(of_get_named_gpio_flags); * value on the error condition. */ static struct gpio_desc *of_parse_own_gpio(struct device_node *np, + struct gpio_chip *chip, const char **name, enum gpio_lookup_flags *lflags, enum gpiod_flags *dflags) { struct device_node *chip_np; enum of_gpio_flags xlate_flags; - struct gg_data gg_data = { - .flags = &xlate_flags, - }; + struct of_phandle_args gpiospec; + struct gpio_desc *desc; u32 tmp; - int i, ret; + int ret; - chip_np = np->parent; + chip_np = chip->of_node; if (!chip_np) return ERR_PTR(-EINVAL); @@ -155,25 +151,16 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np, if (ret) return ERR_PTR(ret); - if (tmp > MAX_PHANDLE_ARGS) - return ERR_PTR(-EINVAL); + gpiospec.np = chip_np; + gpiospec.args_count = tmp; - gg_data.gpiospec.args_count = tmp; - gg_data.gpiospec.np = chip_np; - for (i = 0; i < tmp; i++) { - ret = of_property_read_u32_index(np, "gpios", i, - &gg_data.gpiospec.args[i]); - if (ret) - return ERR_PTR(ret); - } + ret = of_property_read_u32_array(np, "gpios", gpiospec.args, tmp); + if (ret) + return ERR_PTR(ret); - gpiochip_find(&gg_data, of_gpiochip_find_and_xlate); - if (!gg_data.out_gpio) { - if (np->parent == np) - return ERR_PTR(-ENXIO); - else - return ERR_PTR(-EINVAL); - } + desc = of_xlate_and_get_gpiod_flags(chip, &gpiospec, &xlate_flags); + if (IS_ERR(desc)) + return desc; if (xlate_flags & OF_GPIO_ACTIVE_LOW) *lflags |= GPIO_ACTIVE_LOW; @@ -186,14 +173,14 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np, *dflags |= GPIOD_OUT_HIGH; else { pr_warn("GPIO line %d (%s): no hogging state specified, bailing out\n", - desc_to_gpio(gg_data.out_gpio), np->name); + desc_to_gpio(desc), np->name); return ERR_PTR(-EINVAL); } if (name && of_property_read_string(np, "line-name", name)) *name = np->name; - return gg_data.out_gpio; + return desc; } /** @@ -262,7 +249,7 @@ static int of_gpiochip_scan_gpios(struct gpio_chip *chip) if (!of_property_read_bool(np, "gpio-hog")) continue; - desc = of_parse_own_gpio(np, &name, &lflags, &dflags); + desc = of_parse_own_gpio(np, chip, &name, &lflags, &dflags); if (IS_ERR(desc)) continue; @@ -410,6 +397,7 @@ static int of_gpiochip_add_pin_range(struct gpio_chip *chip) break; pctldev = of_pinctrl_get(pinspec.np); + of_node_put(pinspec.np); if (!pctldev) return -EPROBE_DEFER; @@ -487,6 +475,9 @@ int of_gpiochip_add(struct gpio_chip *chip) chip->of_xlate = of_gpio_simple_xlate; } + if (chip->of_gpio_n_cells > MAX_PHANDLE_ARGS) + return -EINVAL; + status = of_gpiochip_add_pin_range(chip); if (status) return status; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 58d822d7e8da..53ff25ac66d8 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -16,11 +16,14 @@ #include #include #include -#include #include #include #include #include +#include +#include +#include +#include #include #include "gpiolib.h" @@ -310,6 +313,497 @@ static int gpiochip_set_desc_names(struct gpio_chip *gc) return 0; } +/* + * GPIO line handle management + */ + +/** + * struct linehandle_state - contains the state of a userspace handle + * @gdev: the GPIO device the handle pertains to + * @label: consumer label used to tag descriptors + * @descs: the GPIO descriptors held by this handle + * @numdescs: the number of descriptors held in the descs array + */ +struct linehandle_state { + struct gpio_device *gdev; + const char *label; + struct gpio_desc *descs[GPIOHANDLES_MAX]; + u32 numdescs; +}; + +static long linehandle_ioctl(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + struct linehandle_state *lh = filep->private_data; + void __user *ip = (void __user *)arg; + struct gpiohandle_data ghd; + int i; + + if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) { + int val; + + /* TODO: check if descriptors are really input */ + for (i = 0; i < lh->numdescs; i++) { + val = gpiod_get_value_cansleep(lh->descs[i]); + if (val < 0) + return val; + ghd.values[i] = val; + } + + if (copy_to_user(ip, &ghd, sizeof(ghd))) + return -EFAULT; + + return 0; + } else if (cmd == GPIOHANDLE_SET_LINE_VALUES_IOCTL) { + int vals[GPIOHANDLES_MAX]; + + /* TODO: check if descriptors are really output */ + if (copy_from_user(&ghd, ip, sizeof(ghd))) + return -EFAULT; + + /* Clamp all values to [0,1] */ + for (i = 0; i < lh->numdescs; i++) + vals[i] = !!ghd.values[i]; + + /* Reuse the array setting function */ + gpiod_set_array_value_complex(false, + true, + lh->numdescs, + lh->descs, + vals); + return 0; + } + return -EINVAL; +} + +#ifdef CONFIG_COMPAT +static long linehandle_ioctl_compat(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + return linehandle_ioctl(filep, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static int linehandle_release(struct inode *inode, struct file *filep) +{ + struct linehandle_state *lh = filep->private_data; + struct gpio_device *gdev = lh->gdev; + int i; + + for (i = 0; i < lh->numdescs; i++) + gpiod_free(lh->descs[i]); + kfree(lh->label); + kfree(lh); + put_device(&gdev->dev); + return 0; +} + +static const struct file_operations linehandle_fileops = { + .release = linehandle_release, + .owner = THIS_MODULE, + .llseek = noop_llseek, + .unlocked_ioctl = linehandle_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = linehandle_ioctl_compat, +#endif +}; + +static int linehandle_create(struct gpio_device *gdev, void __user *ip) +{ + struct gpiohandle_request handlereq; + struct linehandle_state *lh; + int fd, i, ret; + + if (copy_from_user(&handlereq, ip, sizeof(handlereq))) + return -EFAULT; + if ((handlereq.lines == 0) || (handlereq.lines > GPIOHANDLES_MAX)) + return -EINVAL; + + lh = kzalloc(sizeof(*lh), GFP_KERNEL); + if (!lh) + return -ENOMEM; + lh->gdev = gdev; + get_device(&gdev->dev); + + /* Make sure this is terminated */ + handlereq.consumer_label[sizeof(handlereq.consumer_label)-1] = '\0'; + if (strlen(handlereq.consumer_label)) { + lh->label = kstrdup(handlereq.consumer_label, + GFP_KERNEL); + if (!lh->label) { + ret = -ENOMEM; + goto out_free_lh; + } + } + + /* Request each GPIO */ + for (i = 0; i < handlereq.lines; i++) { + u32 offset = handlereq.lineoffsets[i]; + u32 lflags = handlereq.flags; + struct gpio_desc *desc; + + desc = &gdev->descs[offset]; + ret = gpiod_request(desc, lh->label); + if (ret) + goto out_free_descs; + lh->descs[i] = desc; + + if (lflags & GPIOHANDLE_REQUEST_ACTIVE_LOW) + set_bit(FLAG_ACTIVE_LOW, &desc->flags); + if (lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) + set_bit(FLAG_OPEN_DRAIN, &desc->flags); + if (lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE) + set_bit(FLAG_OPEN_SOURCE, &desc->flags); + + /* + * Lines have to be requested explicitly for input + * or output, else the line will be treated "as is". + */ + if (lflags & GPIOHANDLE_REQUEST_OUTPUT) { + int val = !!handlereq.default_values[i]; + + ret = gpiod_direction_output(desc, val); + if (ret) + goto out_free_descs; + } else if (lflags & GPIOHANDLE_REQUEST_INPUT) { + ret = gpiod_direction_input(desc); + if (ret) + goto out_free_descs; + } + dev_dbg(&gdev->dev, "registered chardev handle for line %d\n", + offset); + } + /* Let i point at the last handle */ + i--; + lh->numdescs = handlereq.lines; + + fd = anon_inode_getfd("gpio-linehandle", + &linehandle_fileops, + lh, + O_RDONLY | O_CLOEXEC); + if (fd < 0) { + ret = fd; + goto out_free_descs; + } + + handlereq.fd = fd; + if (copy_to_user(ip, &handlereq, sizeof(handlereq))) { + ret = -EFAULT; + goto out_free_descs; + } + + dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n", + lh->numdescs); + + return 0; + +out_free_descs: + for (; i >= 0; i--) + gpiod_free(lh->descs[i]); + kfree(lh->label); +out_free_lh: + kfree(lh); + put_device(&gdev->dev); + return ret; +} + +/* + * GPIO line event management + */ + +/** + * struct lineevent_state - contains the state of a userspace event + * @gdev: the GPIO device the event pertains to + * @label: consumer label used to tag descriptors + * @desc: the GPIO descriptor held by this event + * @eflags: the event flags this line was requested with + * @irq: the interrupt that trigger in response to events on this GPIO + * @wait: wait queue that handles blocking reads of events + * @events: KFIFO for the GPIO events + * @read_lock: mutex lock to protect reads from colliding with adding + * new events to the FIFO + */ +struct lineevent_state { + struct gpio_device *gdev; + const char *label; + struct gpio_desc *desc; + u32 eflags; + int irq; + wait_queue_head_t wait; + DECLARE_KFIFO(events, struct gpioevent_data, 16); + struct mutex read_lock; +}; + +static unsigned int lineevent_poll(struct file *filep, + struct poll_table_struct *wait) +{ + struct lineevent_state *le = filep->private_data; + unsigned int events = 0; + + poll_wait(filep, &le->wait, wait); + + if (!kfifo_is_empty(&le->events)) + events = POLLIN | POLLRDNORM; + + return events; +} + + +static ssize_t lineevent_read(struct file *filep, + char __user *buf, + size_t count, + loff_t *f_ps) +{ + struct lineevent_state *le = filep->private_data; + unsigned int copied; + int ret; + + if (count < sizeof(struct gpioevent_data)) + return -EINVAL; + + do { + if (kfifo_is_empty(&le->events)) { + if (filep->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible(le->wait, + !kfifo_is_empty(&le->events)); + if (ret) + return ret; + } + + if (mutex_lock_interruptible(&le->read_lock)) + return -ERESTARTSYS; + ret = kfifo_to_user(&le->events, buf, count, &copied); + mutex_unlock(&le->read_lock); + + if (ret) + return ret; + + /* + * If we couldn't read anything from the fifo (a different + * thread might have been faster) we either return -EAGAIN if + * the file descriptor is non-blocking, otherwise we go back to + * sleep and wait for more data to arrive. + */ + if (copied == 0 && (filep->f_flags & O_NONBLOCK)) + return -EAGAIN; + + } while (copied == 0); + + return copied; +} + +static int lineevent_release(struct inode *inode, struct file *filep) +{ + struct lineevent_state *le = filep->private_data; + struct gpio_device *gdev = le->gdev; + + free_irq(le->irq, le); + gpiod_free(le->desc); + kfree(le->label); + kfree(le); + put_device(&gdev->dev); + return 0; +} + +static long lineevent_ioctl(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + struct lineevent_state *le = filep->private_data; + void __user *ip = (void __user *)arg; + struct gpiohandle_data ghd; + + /* + * We can get the value for an event line but not set it, + * because it is input by definition. + */ + if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) { + int val; + + val = gpiod_get_value_cansleep(le->desc); + if (val < 0) + return val; + ghd.values[0] = val; + + if (copy_to_user(ip, &ghd, sizeof(ghd))) + return -EFAULT; + + return 0; + } + return -EINVAL; +} + +#ifdef CONFIG_COMPAT +static long lineevent_ioctl_compat(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + return lineevent_ioctl(filep, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static const struct file_operations lineevent_fileops = { + .release = lineevent_release, + .read = lineevent_read, + .poll = lineevent_poll, + .owner = THIS_MODULE, + .llseek = noop_llseek, + .unlocked_ioctl = lineevent_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = lineevent_ioctl_compat, +#endif +}; + +static irqreturn_t lineevent_irq_thread(int irq, void *p) +{ + struct lineevent_state *le = p; + struct gpioevent_data ge; + int ret; + + ge.timestamp = ktime_get_real_ns(); + + if (le->eflags & GPIOEVENT_REQUEST_BOTH_EDGES) { + int level = gpiod_get_value_cansleep(le->desc); + + if (level) + /* Emit low-to-high event */ + ge.id = GPIOEVENT_EVENT_RISING_EDGE; + else + /* Emit high-to-low event */ + ge.id = GPIOEVENT_EVENT_FALLING_EDGE; + } else if (le->eflags & GPIOEVENT_REQUEST_RISING_EDGE) { + /* Emit low-to-high event */ + ge.id = GPIOEVENT_EVENT_RISING_EDGE; + } else if (le->eflags & GPIOEVENT_REQUEST_FALLING_EDGE) { + /* Emit high-to-low event */ + ge.id = GPIOEVENT_EVENT_FALLING_EDGE; + } else { + return IRQ_NONE; + } + + ret = kfifo_put(&le->events, ge); + if (ret != 0) + wake_up_poll(&le->wait, POLLIN); + + return IRQ_HANDLED; +} + +static int lineevent_create(struct gpio_device *gdev, void __user *ip) +{ + struct gpioevent_request eventreq; + struct lineevent_state *le; + struct gpio_desc *desc; + u32 offset; + u32 lflags; + u32 eflags; + int fd; + int ret; + int irqflags = 0; + + if (copy_from_user(&eventreq, ip, sizeof(eventreq))) + return -EFAULT; + + le = kzalloc(sizeof(*le), GFP_KERNEL); + if (!le) + return -ENOMEM; + le->gdev = gdev; + get_device(&gdev->dev); + + /* Make sure this is terminated */ + eventreq.consumer_label[sizeof(eventreq.consumer_label)-1] = '\0'; + if (strlen(eventreq.consumer_label)) { + le->label = kstrdup(eventreq.consumer_label, + GFP_KERNEL); + if (!le->label) { + ret = -ENOMEM; + goto out_free_le; + } + } + + offset = eventreq.lineoffset; + lflags = eventreq.handleflags; + eflags = eventreq.eventflags; + + /* This is just wrong: we don't look for events on output lines */ + if (lflags & GPIOHANDLE_REQUEST_OUTPUT) { + ret = -EINVAL; + goto out_free_label; + } + + desc = &gdev->descs[offset]; + ret = gpiod_request(desc, le->label); + if (ret) + goto out_free_desc; + le->desc = desc; + le->eflags = eflags; + + if (lflags & GPIOHANDLE_REQUEST_ACTIVE_LOW) + set_bit(FLAG_ACTIVE_LOW, &desc->flags); + if (lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) + set_bit(FLAG_OPEN_DRAIN, &desc->flags); + if (lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE) + set_bit(FLAG_OPEN_SOURCE, &desc->flags); + + ret = gpiod_direction_input(desc); + if (ret) + goto out_free_desc; + + le->irq = gpiod_to_irq(desc); + if (le->irq <= 0) { + ret = -ENODEV; + goto out_free_desc; + } + + if (eflags & GPIOEVENT_REQUEST_RISING_EDGE) + irqflags |= IRQF_TRIGGER_RISING; + if (eflags & GPIOEVENT_REQUEST_FALLING_EDGE) + irqflags |= IRQF_TRIGGER_FALLING; + irqflags |= IRQF_ONESHOT; + irqflags |= IRQF_SHARED; + + INIT_KFIFO(le->events); + init_waitqueue_head(&le->wait); + mutex_init(&le->read_lock); + + /* Request a thread to read the events */ + ret = request_threaded_irq(le->irq, + NULL, + lineevent_irq_thread, + irqflags, + le->label, + le); + if (ret) + goto out_free_desc; + + fd = anon_inode_getfd("gpio-event", + &lineevent_fileops, + le, + O_RDONLY | O_CLOEXEC); + if (fd < 0) { + ret = fd; + goto out_free_irq; + } + + eventreq.fd = fd; + if (copy_to_user(ip, &eventreq, sizeof(eventreq))) { + ret = -EFAULT; + goto out_free_irq; + } + + return 0; + +out_free_irq: + free_irq(le->irq, le); +out_free_desc: + gpiod_free(le->desc); +out_free_label: + kfree(le->label); +out_free_le: + kfree(le); + put_device(&gdev->dev); + return ret; +} + /** * gpio_ioctl() - ioctl handler for the GPIO chardev */ @@ -385,6 +879,10 @@ static long gpio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) return -EFAULT; return 0; + } else if (cmd == GPIO_GET_LINEHANDLE_IOCTL) { + return linehandle_create(gdev, ip); + } else if (cmd == GPIO_GET_LINEEVENT_IOCTL) { + return lineevent_create(gdev, ip); } return -EINVAL; } @@ -548,13 +1046,14 @@ int gpiochip_add_data(struct gpio_chip *chip, void *data) if (chip->parent) { gdev->dev.parent = chip->parent; gdev->dev.of_node = chip->parent->of_node; - } else { + } + #ifdef CONFIG_OF_GPIO /* If the gpiochip has an assigned OF node this takes precedence */ - if (chip->of_node) - gdev->dev.of_node = chip->of_node; + if (chip->of_node) + gdev->dev.of_node = chip->of_node; #endif - } + gdev->id = ida_simple_get(&gpio_ida, 0, 0, GFP_KERNEL); if (gdev->id < 0) { status = gdev->id; @@ -1352,14 +1851,6 @@ static int __gpiod_request(struct gpio_desc *desc, const char *label) spin_lock_irqsave(&gpio_lock, flags); } done: - if (status < 0) { - /* Clear flags that might have been set by the caller before - * requesting the GPIO. - */ - clear_bit(FLAG_ACTIVE_LOW, &desc->flags); - clear_bit(FLAG_OPEN_DRAIN, &desc->flags); - clear_bit(FLAG_OPEN_SOURCE, &desc->flags); - } spin_unlock_irqrestore(&gpio_lock, flags); return status; } @@ -1373,8 +1864,12 @@ done: #define VALIDATE_DESC(desc) do { \ if (!desc) \ return 0; \ + if (IS_ERR(desc)) { \ + pr_warn("%s: invalid GPIO (errorpointer)\n", __func__); \ + return PTR_ERR(desc); \ + } \ if (!desc->gdev) { \ - pr_warn("%s: invalid GPIO\n", __func__); \ + pr_warn("%s: invalid GPIO (no device)\n", __func__); \ return -EINVAL; \ } \ if ( !desc->gdev->chip ) { \ @@ -1386,8 +1881,12 @@ done: #define VALIDATE_DESC_VOID(desc) do { \ if (!desc) \ return; \ + if (IS_ERR(desc)) { \ + pr_warn("%s: invalid GPIO (errorpointer)\n", __func__); \ + return; \ + } \ if (!desc->gdev) { \ - pr_warn("%s: invalid GPIO\n", __func__); \ + pr_warn("%s: invalid GPIO (no device)\n", __func__); \ return; \ } \ if (!desc->gdev->chip) { \ @@ -2056,7 +2555,14 @@ int gpiod_to_irq(const struct gpio_desc *desc) struct gpio_chip *chip; int offset; - VALIDATE_DESC(desc); + /* + * Cannot VALIDATE_DESC() here as gpiod_to_irq() consumer semantics + * requires this function to not return zero on an invalid descriptor + * but rather a negative error number. + */ + if (!desc || IS_ERR(desc) || !desc->gdev || !desc->gdev->chip) + return -EINVAL; + chip = desc->gdev->chip; offset = gpio_chip_hwgpio(desc); if (chip->to_irq) { @@ -2326,7 +2832,7 @@ static struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, desc = of_get_named_gpiod_flags(dev->of_node, prop_name, idx, &of_flags); - if (!IS_ERR(desc) || (PTR_ERR(desc) == -EPROBE_DEFER)) + if (!IS_ERR(desc) || (PTR_ERR(desc) != -ENOENT)) break; } @@ -2572,28 +3078,13 @@ struct gpio_desc *__must_check gpiod_get_optional(struct device *dev, } EXPORT_SYMBOL_GPL(gpiod_get_optional); -/** - * gpiod_parse_flags - helper function to parse GPIO lookup flags - * @desc: gpio to be setup - * @lflags: gpio_lookup_flags - returned from of_find_gpio() or - * of_get_gpio_hog() - * - * Set the GPIO descriptor flags based on the given GPIO lookup flags. - */ -static void gpiod_parse_flags(struct gpio_desc *desc, unsigned long lflags) -{ - if (lflags & GPIO_ACTIVE_LOW) - set_bit(FLAG_ACTIVE_LOW, &desc->flags); - if (lflags & GPIO_OPEN_DRAIN) - set_bit(FLAG_OPEN_DRAIN, &desc->flags); - if (lflags & GPIO_OPEN_SOURCE) - set_bit(FLAG_OPEN_SOURCE, &desc->flags); -} /** * gpiod_configure_flags - helper function to configure a given GPIO * @desc: gpio whose value will be assigned * @con_id: function within the GPIO consumer + * @lflags: gpio_lookup_flags - returned from of_find_gpio() or + * of_get_gpio_hog() * @dflags: gpiod_flags - optional GPIO initialization flags * * Return 0 on success, -ENOENT if no GPIO has been assigned to the @@ -2601,10 +3092,17 @@ static void gpiod_parse_flags(struct gpio_desc *desc, unsigned long lflags) * occurred while trying to acquire the GPIO. */ static int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id, - enum gpiod_flags dflags) + unsigned long lflags, enum gpiod_flags dflags) { int status; + if (lflags & GPIO_ACTIVE_LOW) + set_bit(FLAG_ACTIVE_LOW, &desc->flags); + if (lflags & GPIO_OPEN_DRAIN) + set_bit(FLAG_OPEN_DRAIN, &desc->flags); + if (lflags & GPIO_OPEN_SOURCE) + set_bit(FLAG_OPEN_SOURCE, &desc->flags); + /* No particular flag request, return here... */ if (!(dflags & GPIOD_FLAGS_BIT_DIR_SET)) { pr_debug("no flags found for %s\n", con_id); @@ -2671,13 +3169,11 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev, return desc; } - gpiod_parse_flags(desc, lookupflags); - status = gpiod_request(desc, con_id); if (status < 0) return ERR_PTR(status); - status = gpiod_configure_flags(desc, con_id, flags); + status = gpiod_configure_flags(desc, con_id, lookupflags, flags); if (status < 0) { dev_dbg(dev, "setup of GPIO %s failed\n", con_id); gpiod_put(desc); @@ -2733,6 +3229,10 @@ struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode, if (IS_ERR(desc)) return desc; + ret = gpiod_request(desc, NULL); + if (ret) + return ERR_PTR(ret); + if (active_low) set_bit(FLAG_ACTIVE_LOW, &desc->flags); @@ -2743,10 +3243,6 @@ struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode, set_bit(FLAG_OPEN_SOURCE, &desc->flags); } - ret = gpiod_request(desc, NULL); - if (ret) - return ERR_PTR(ret); - return desc; } EXPORT_SYMBOL_GPL(fwnode_get_named_gpiod); @@ -2799,8 +3295,6 @@ int gpiod_hog(struct gpio_desc *desc, const char *name, chip = gpiod_to_chip(desc); hwnum = gpio_chip_hwgpio(desc); - gpiod_parse_flags(desc, lflags); - local_desc = gpiochip_request_own_desc(chip, hwnum, name); if (IS_ERR(local_desc)) { status = PTR_ERR(local_desc); @@ -2809,7 +3303,7 @@ int gpiod_hog(struct gpio_desc *desc, const char *name, return status; } - status = gpiod_configure_flags(desc, name, dflags); + status = gpiod_configure_flags(desc, name, lflags, dflags); if (status < 0) { pr_err("setup of hog GPIO %s (chip %s, offset %d) failed, %d\n", name, chip->label, hwnum, status); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index 252edba16e36..892d60fb225b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -421,29 +421,6 @@ static int acp_suspend(void *handle) static int acp_resume(void *handle) { - int i, ret; - struct acp_pm_domain *apd; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - /* return early if no ACP */ - if (!adev->acp.acp_genpd) - return 0; - - /* SMU block will power on ACP irrespective of ACP runtime status. - * Power off explicitly based on genpd ACP runtime status so that ACP - * hw and ACP-genpd status are in sync. - * 'suspend_power_off' represents "Power status before system suspend" - */ - if (adev->acp.acp_genpd->gpd.suspend_power_off == true) { - apd = container_of(&adev->acp.acp_genpd->gpd, - struct acp_pm_domain, gpd); - - for (i = 4; i >= 0 ; i--) { - ret = acp_suspend_tile(apd->cgs_dev, ACP_TILE_P1 + i); - if (ret) - pr_err("ACP tile %d tile suspend failed\n", i); - } - } return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c index 8943099eb135..cf6f49fc1c75 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c @@ -909,7 +909,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, struct cgs_acpi_method_argument *argument = NULL; uint32_t i, count; acpi_status status; - int result; + int result = 0; uint32_t func_no = 0xFFFFFFFF; handle = ACPI_HANDLE(&adev->pdev->dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 66482b429458..6e920086af46 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -1535,7 +1535,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, /* Post card if necessary */ if (!amdgpu_card_posted(adev) || (adev->virtualization.is_virtual && - !adev->virtualization.caps & AMDGPU_VIRT_CAPS_SRIOV_EN)) { + !(adev->virtualization.caps & AMDGPU_VIRT_CAPS_SRIOV_EN))) { if (!adev->bios) { dev_err(adev->dev, "Card not posted and no BIOS - ignoring\n"); return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 40a23704a981..d851ea15059f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -447,7 +447,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file dev_info.max_memory_clock = adev->pm.default_mclk * 10; } dev_info.enabled_rb_pipes_mask = adev->gfx.config.backend_enable_mask; - dev_info.num_rb_pipes = adev->gfx.config.num_rbs; + dev_info.num_rb_pipes = adev->gfx.config.max_backends_per_se * + adev->gfx.config.max_shader_engines; dev_info.num_hw_gfx_contexts = adev->gfx.config.max_hw_contexts; dev_info._pad = 0; dev_info.ids_flags = 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c index 589b36e8c5cf..0e13d80d2a95 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c @@ -270,30 +270,28 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev, struct drm_device *ddev = dev_get_drvdata(dev); struct amdgpu_device *adev = ddev->dev_private; enum amd_pm_state_type state = 0; - long idx; + unsigned long idx; int ret; if (strlen(buf) == 1) adev->pp_force_state_enabled = false; - else { - ret = kstrtol(buf, 0, &idx); + else if (adev->pp_enabled) { + struct pp_states_info data; - if (ret) { + ret = kstrtoul(buf, 0, &idx); + if (ret || idx >= ARRAY_SIZE(data.states)) { count = -EINVAL; goto fail; } - if (adev->pp_enabled) { - struct pp_states_info data; - amdgpu_dpm_get_pp_num_states(adev, &data); - state = data.states[idx]; - /* only set user selected power states */ - if (state != POWER_STATE_TYPE_INTERNAL_BOOT && - state != POWER_STATE_TYPE_DEFAULT) { - amdgpu_dpm_dispatch_task(adev, - AMD_PP_EVENT_ENABLE_USER_STATE, &state, NULL); - adev->pp_force_state_enabled = true; - } + amdgpu_dpm_get_pp_num_states(adev, &data); + state = data.states[idx]; + /* only set user selected power states */ + if (state != POWER_STATE_TYPE_INTERNAL_BOOT && + state != POWER_STATE_TYPE_DEFAULT) { + amdgpu_dpm_dispatch_task(adev, + AMD_PP_EVENT_ENABLE_USER_STATE, &state, NULL); + adev->pp_force_state_enabled = true; } } fail: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index e19520c4b4b6..d9c88d13f8db 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -1106,6 +1106,10 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work) if (fences == 0 && handles == 0) { if (adev->pm.dpm_enabled) { amdgpu_dpm_enable_uvd(adev, false); + /* just work around for uvd clock remain high even + * when uvd dpm disabled on Polaris10 */ + if (adev->asic_type == CHIP_POLARIS10) + amdgpu_asic_set_uvd_clocks(adev, 0, 0); } else { amdgpu_asic_set_uvd_clocks(adev, 0, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c index 13cdb01e9b45..bc56c8a181e6 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c @@ -156,3 +156,18 @@ u32 amdgpu_atombios_i2c_func(struct i2c_adapter *adap) return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; } +void amdgpu_atombios_i2c_channel_trans(struct amdgpu_device* adev, u8 slave_addr, u8 line_number, u8 offset, u8 data) +{ + PROCESS_I2C_CHANNEL_TRANSACTION_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, ProcessI2cChannelTransaction); + + args.ucRegIndex = offset; + args.lpI2CDataOut = data; + args.ucFlag = 1; + args.ucI2CSpeed = TARGET_HW_I2C_CLOCK; + args.ucTransBytes = 1; + args.ucSlaveAddr = slave_addr; + args.ucLineNumber = line_number; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.h b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.h index d6128d9de56e..251aaf41f65d 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.h +++ b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.h @@ -27,5 +27,7 @@ int amdgpu_atombios_i2c_xfer(struct i2c_adapter *i2c_adap, struct i2c_msg *msgs, int num); u32 amdgpu_atombios_i2c_func(struct i2c_adapter *adap); +void amdgpu_atombios_i2c_channel_trans(struct amdgpu_device* adev, + u8 slave_addr, u8 line_number, u8 offset, u8 data); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 9f6f8669edc3..c2ef94511f70 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -28,6 +28,7 @@ #include "vid.h" #include "amdgpu_ucode.h" #include "amdgpu_atombios.h" +#include "atombios_i2c.h" #include "clearstate_vi.h" #include "gmc/gmc_8_2_d.h" @@ -47,6 +48,8 @@ #include "dce/dce_10_0_d.h" #include "dce/dce_10_0_sh_mask.h" +#include "smu/smu_7_1_3_d.h" + #define GFX8_NUM_GFX_RINGS 1 #define GFX8_NUM_COMPUTE_RINGS 8 @@ -282,6 +285,7 @@ static const u32 golden_settings_polaris11_a11[] = mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210, + mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, }; static const u32 polaris11_golden_common_all[] = @@ -297,7 +301,8 @@ static const u32 polaris11_golden_common_all[] = static const u32 golden_settings_polaris10_a11[] = { mmATC_MISC_CG, 0x000c0fc0, 0x000c0200, - mmCB_HW_CONTROL, 0xfffdf3cf, 0x00006208, + mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208, + mmCB_HW_CONTROL_2, 0, 0x0f000000, mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, mmDB_DEBUG2, 0xf00fffff, 0x00000400, mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, @@ -311,6 +316,7 @@ static const u32 golden_settings_polaris10_a11[] = mmTCC_CTRL, 0x00100000, 0xf31fff7f, mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, + mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, }; static const u32 polaris10_golden_common_all[] = @@ -692,6 +698,11 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev) amdgpu_program_register_sequence(adev, polaris10_golden_common_all, (const u32)ARRAY_SIZE(polaris10_golden_common_all)); + WREG32_SMC(ixCG_ACLK_CNTL, 0x0000001C); + if (adev->pdev->revision == 0xc7) { + amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD); + amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0); + } break; case CHIP_CARRIZO: amdgpu_program_register_sequence(adev, diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h index 32f3e345de08..3493da5c8f0e 100644 --- a/drivers/gpu/drm/amd/include/atombios.h +++ b/drivers/gpu/drm/amd/include/atombios.h @@ -5538,6 +5538,78 @@ typedef struct _ATOM_ASIC_PROFILING_INFO_V3_5 ULONG ulReserved[12]; }ATOM_ASIC_PROFILING_INFO_V3_5; +/* for Polars10/11 AVFS parameters */ +typedef struct _ATOM_ASIC_PROFILING_INFO_V3_6 +{ + ATOM_COMMON_TABLE_HEADER asHeader; + ULONG ulMaxVddc; + ULONG ulMinVddc; + USHORT usLkgEuseIndex; + UCHAR ucLkgEfuseBitLSB; + UCHAR ucLkgEfuseLength; + ULONG ulLkgEncodeLn_MaxDivMin; + ULONG ulLkgEncodeMax; + ULONG ulLkgEncodeMin; + EFUSE_LINEAR_FUNC_PARAM sRoFuse; + ULONG ulEvvDefaultVddc; + ULONG ulEvvNoCalcVddc; + ULONG ulSpeed_Model; + ULONG ulSM_A0; + ULONG ulSM_A1; + ULONG ulSM_A2; + ULONG ulSM_A3; + ULONG ulSM_A4; + ULONG ulSM_A5; + ULONG ulSM_A6; + ULONG ulSM_A7; + UCHAR ucSM_A0_sign; + UCHAR ucSM_A1_sign; + UCHAR ucSM_A2_sign; + UCHAR ucSM_A3_sign; + UCHAR ucSM_A4_sign; + UCHAR ucSM_A5_sign; + UCHAR ucSM_A6_sign; + UCHAR ucSM_A7_sign; + ULONG ulMargin_RO_a; + ULONG ulMargin_RO_b; + ULONG ulMargin_RO_c; + ULONG ulMargin_fixed; + ULONG ulMargin_Fmax_mean; + ULONG ulMargin_plat_mean; + ULONG ulMargin_Fmax_sigma; + ULONG ulMargin_plat_sigma; + ULONG ulMargin_DC_sigma; + ULONG ulLoadLineSlop; + ULONG ulaTDClimitPerDPM[8]; + ULONG ulaNoCalcVddcPerDPM[8]; + ULONG ulAVFS_meanNsigma_Acontant0; + ULONG ulAVFS_meanNsigma_Acontant1; + ULONG ulAVFS_meanNsigma_Acontant2; + USHORT usAVFS_meanNsigma_DC_tol_sigma; + USHORT usAVFS_meanNsigma_Platform_mean; + USHORT usAVFS_meanNsigma_Platform_sigma; + ULONG ulGB_VDROOP_TABLE_CKSOFF_a0; + ULONG ulGB_VDROOP_TABLE_CKSOFF_a1; + ULONG ulGB_VDROOP_TABLE_CKSOFF_a2; + ULONG ulGB_VDROOP_TABLE_CKSON_a0; + ULONG ulGB_VDROOP_TABLE_CKSON_a1; + ULONG ulGB_VDROOP_TABLE_CKSON_a2; + ULONG ulAVFSGB_FUSE_TABLE_CKSOFF_m1; + USHORT usAVFSGB_FUSE_TABLE_CKSOFF_m2; + ULONG ulAVFSGB_FUSE_TABLE_CKSOFF_b; + ULONG ulAVFSGB_FUSE_TABLE_CKSON_m1; + USHORT usAVFSGB_FUSE_TABLE_CKSON_m2; + ULONG ulAVFSGB_FUSE_TABLE_CKSON_b; + USHORT usMaxVoltage_0_25mv; + UCHAR ucEnableGB_VDROOP_TABLE_CKSOFF; + UCHAR ucEnableGB_VDROOP_TABLE_CKSON; + UCHAR ucEnableGB_FUSE_TABLE_CKSOFF; + UCHAR ucEnableGB_FUSE_TABLE_CKSON; + USHORT usPSM_Age_ComFactor; + UCHAR ucEnableApplyAVFS_CKS_OFF_Voltage; + UCHAR ucReserved; +}ATOM_ASIC_PROFILING_INFO_V3_6; + typedef struct _ATOM_SCLK_FCW_RANGE_ENTRY_V1{ ULONG ulMaxSclkFreq; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c index 586f73276226..92912ab20944 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c @@ -633,6 +633,8 @@ static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr) data->vddci_control = FIJI_VOLTAGE_CONTROL_NONE; data->mvdd_control = FIJI_VOLTAGE_CONTROL_NONE; + data->force_pcie_gen = PP_PCIEGenInvalid; + if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2)) data->voltage_control = FIJI_VOLTAGE_CONTROL_BY_SVID2; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c index fa208ada6892..efb77eda7508 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c @@ -306,10 +306,14 @@ int phm_store_dal_configuration_data(struct pp_hwmgr *hwmgr, { PHM_FUNC_CHECK(hwmgr); - if (hwmgr->hwmgr_func->store_cc6_data == NULL) + if (display_config == NULL) return -EINVAL; hwmgr->display_config = *display_config; + + if (hwmgr->hwmgr_func->store_cc6_data == NULL) + return -EINVAL; + /* to do pass other display configuration in furture */ if (hwmgr->hwmgr_func->store_cc6_data) diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c index 1400bc420881..91e25f942d90 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c @@ -732,7 +732,7 @@ static int polaris10_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr, table->Smio[level] |= data->mvdd_voltage_table.entries[level].smio_low; } - table->SmioMask2 = data->vddci_voltage_table.mask_low; + table->SmioMask2 = data->mvdd_voltage_table.mask_low; table->MvddLevelCount = (uint32_t) PP_HOST_TO_SMC_UL(count); } @@ -1296,7 +1296,6 @@ static int polaris10_populate_single_memory_level(struct pp_hwmgr *hwmgr, } mem_level->MclkFrequency = clock; - mem_level->StutterEnable = 0; mem_level->EnabledForThrottle = 1; mem_level->EnabledForActivity = 0; mem_level->UpHyst = 0; @@ -1304,7 +1303,6 @@ static int polaris10_populate_single_memory_level(struct pp_hwmgr *hwmgr, mem_level->VoltageDownHyst = 0; mem_level->ActivityLevel = (uint16_t)data->mclk_activity_target; mem_level->StutterEnable = false; - mem_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW; data->display_timing.num_existing_displays = info.display_count; @@ -1363,7 +1361,7 @@ static int polaris10_populate_all_memory_levels(struct pp_hwmgr *hwmgr) * a higher state by default such that we are not effected by * up threshold or and MCLK DPM latency. */ - levels[0].ActivityLevel = (uint16_t)data->mclk_dpm0_activity_target; + levels[0].ActivityLevel = 0x1f; CONVERT_FROM_HOST_TO_SMC_US(levels[0].ActivityLevel); data->smc_state_table.MemoryDpmLevelCount = @@ -1424,22 +1422,19 @@ static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr, table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC; - if (!data->sclk_dpm_key_disabled) { - /* Get MinVoltage and Frequency from DPM0, - * already converted to SMC_UL */ - sclk_frequency = data->dpm_table.sclk_table.dpm_levels[0].value; - result = polaris10_get_dependency_volt_by_clk(hwmgr, - table_info->vdd_dep_on_sclk, - table->ACPILevel.SclkFrequency, - &table->ACPILevel.MinVoltage, &mvdd); - PP_ASSERT_WITH_CODE((0 == result), - "Cannot find ACPI VDDC voltage value " - "in Clock Dependency Table", ); - } else { - sclk_frequency = data->vbios_boot_state.sclk_bootup_value; - table->ACPILevel.MinVoltage = - data->vbios_boot_state.vddc_bootup_value * VOLTAGE_SCALE; - } + + /* Get MinVoltage and Frequency from DPM0, + * already converted to SMC_UL */ + sclk_frequency = data->dpm_table.sclk_table.dpm_levels[0].value; + result = polaris10_get_dependency_volt_by_clk(hwmgr, + table_info->vdd_dep_on_sclk, + sclk_frequency, + &table->ACPILevel.MinVoltage, &mvdd); + PP_ASSERT_WITH_CODE((0 == result), + "Cannot find ACPI VDDC voltage value " + "in Clock Dependency Table", + ); + result = polaris10_calculate_sclk_params(hwmgr, sclk_frequency, &(table->ACPILevel.SclkSetting)); PP_ASSERT_WITH_CODE(result == 0, "Error retrieving Engine Clock dividers from VBIOS.", return result); @@ -1464,24 +1459,18 @@ static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr, CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw1_frac); CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Sclk_ss_slew_rate); - if (!data->mclk_dpm_key_disabled) { - /* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */ - table->MemoryACPILevel.MclkFrequency = - data->dpm_table.mclk_table.dpm_levels[0].value; - result = polaris10_get_dependency_volt_by_clk(hwmgr, - table_info->vdd_dep_on_mclk, - table->MemoryACPILevel.MclkFrequency, - &table->MemoryACPILevel.MinVoltage, &mvdd); - PP_ASSERT_WITH_CODE((0 == result), - "Cannot find ACPI VDDCI voltage value " - "in Clock Dependency Table", - ); - } else { - table->MemoryACPILevel.MclkFrequency = - data->vbios_boot_state.mclk_bootup_value; - table->MemoryACPILevel.MinVoltage = - data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE; - } + + /* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */ + table->MemoryACPILevel.MclkFrequency = + data->dpm_table.mclk_table.dpm_levels[0].value; + result = polaris10_get_dependency_volt_by_clk(hwmgr, + table_info->vdd_dep_on_mclk, + table->MemoryACPILevel.MclkFrequency, + &table->MemoryACPILevel.MinVoltage, &mvdd); + PP_ASSERT_WITH_CODE((0 == result), + "Cannot find ACPI VDDCI voltage value " + "in Clock Dependency Table", + ); us_mvdd = 0; if ((POLARIS10_VOLTAGE_CONTROL_NONE == data->mvdd_control) || @@ -1526,6 +1515,7 @@ static int polaris10_populate_smc_vce_level(struct pp_hwmgr *hwmgr, struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = table_info->mm_dep_table; struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + uint32_t vddci; table->VceLevelCount = (uint8_t)(mm_table->count); table->VceBootLevel = 0; @@ -1535,9 +1525,18 @@ static int polaris10_populate_smc_vce_level(struct pp_hwmgr *hwmgr, table->VceLevel[count].MinVoltage = 0; table->VceLevel[count].MinVoltage |= (mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT; + + if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) + vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table), + mm_table->entries[count].vddc - VDDC_VDDCI_DELTA); + else if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) + vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA; + else + vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT; + + table->VceLevel[count].MinVoltage |= - ((mm_table->entries[count].vddc - data->vddc_vddci_delta) * - VOLTAGE_SCALE) << VDDCI_SHIFT; + (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT; table->VceLevel[count].MinVoltage |= 1 << PHASES_SHIFT; /*retrieve divider value for VBIOS */ @@ -1566,6 +1565,7 @@ static int polaris10_populate_smc_samu_level(struct pp_hwmgr *hwmgr, struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = table_info->mm_dep_table; struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + uint32_t vddci; table->SamuBootLevel = 0; table->SamuLevelCount = (uint8_t)(mm_table->count); @@ -1576,8 +1576,16 @@ static int polaris10_populate_smc_samu_level(struct pp_hwmgr *hwmgr, table->SamuLevel[count].Frequency = mm_table->entries[count].samclock; table->SamuLevel[count].MinVoltage |= (mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT; - table->SamuLevel[count].MinVoltage |= ((mm_table->entries[count].vddc - - data->vddc_vddci_delta) * VOLTAGE_SCALE) << VDDCI_SHIFT; + + if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) + vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table), + mm_table->entries[count].vddc - VDDC_VDDCI_DELTA); + else if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) + vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA; + else + vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT; + + table->SamuLevel[count].MinVoltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT; table->SamuLevel[count].MinVoltage |= 1 << PHASES_SHIFT; /* retrieve divider value for VBIOS */ @@ -1660,6 +1668,7 @@ static int polaris10_populate_smc_uvd_level(struct pp_hwmgr *hwmgr, struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = table_info->mm_dep_table; struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + uint32_t vddci; table->UvdLevelCount = (uint8_t)(mm_table->count); table->UvdBootLevel = 0; @@ -1670,8 +1679,16 @@ static int polaris10_populate_smc_uvd_level(struct pp_hwmgr *hwmgr, table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk; table->UvdLevel[count].MinVoltage |= (mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT; - table->UvdLevel[count].MinVoltage |= ((mm_table->entries[count].vddc - - data->vddc_vddci_delta) * VOLTAGE_SCALE) << VDDCI_SHIFT; + + if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) + vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table), + mm_table->entries[count].vddc - VDDC_VDDCI_DELTA); + else if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) + vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA; + else + vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT; + + table->UvdLevel[count].MinVoltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT; table->UvdLevel[count].MinVoltage |= 1 << PHASES_SHIFT; /* retrieve divider value for VBIOS */ @@ -1692,8 +1709,8 @@ static int polaris10_populate_smc_uvd_level(struct pp_hwmgr *hwmgr, CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency); CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency); CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].MinVoltage); - } + return result; } @@ -1761,12 +1778,9 @@ static int polaris10_populate_smc_initailial_state(struct pp_hwmgr *hwmgr) static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr) { - uint32_t ro, efuse, efuse2, clock_freq, volt_without_cks, - volt_with_cks, value; - uint16_t clock_freq_u16; + uint32_t ro, efuse, volt_without_cks, volt_with_cks, value, max, min; struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); - uint8_t type, i, j, cks_setting, stretch_amount, stretch_amount2, - volt_offset = 0; + uint8_t i, stretch_amount, stretch_amount2, volt_offset = 0; struct phm_ppt_v1_information *table_info = (struct phm_ppt_v1_information *)(hwmgr->pptable); struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table = @@ -1778,50 +1792,44 @@ static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr) * if the part is SS or FF. if RO >= 1660MHz, part is FF. */ efuse = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, - ixSMU_EFUSE_0 + (146 * 4)); - efuse2 = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, - ixSMU_EFUSE_0 + (148 * 4)); + ixSMU_EFUSE_0 + (67 * 4)); efuse &= 0xFF000000; efuse = efuse >> 24; - efuse2 &= 0xF; - if (efuse2 == 1) - ro = (2300 - 1350) * efuse / 255 + 1350; - else - ro = (2500 - 1000) * efuse / 255 + 1000; - - if (ro >= 1660) - type = 0; - else - type = 1; + if (hwmgr->chip_id == CHIP_POLARIS10) { + min = 1000; + max = 2300; + } else { + min = 1100; + max = 2100; + } - /* Populate Stretch amount */ - data->smc_state_table.ClockStretcherAmount = stretch_amount; + ro = efuse * (max -min)/255 + min; /* Populate Sclk_CKS_masterEn0_7 and Sclk_voltageOffset */ for (i = 0; i < sclk_table->count; i++) { data->smc_state_table.Sclk_CKS_masterEn0_7 |= sclk_table->entries[i].cks_enable << i; - volt_without_cks = (uint32_t)((14041 * - (sclk_table->entries[i].clk/100) / 10000 + 3571 + 75 - ro) * 1000 / - (4026 - (13924 * (sclk_table->entries[i].clk/100) / 10000))); - volt_with_cks = (uint32_t)((13946 * - (sclk_table->entries[i].clk/100) / 10000 + 3320 + 45 - ro) * 1000 / - (3664 - (11454 * (sclk_table->entries[i].clk/100) / 10000))); + if (hwmgr->chip_id == CHIP_POLARIS10) { + volt_without_cks = (uint32_t)((2753594000U + (sclk_table->entries[i].clk/100) * 136418 -(ro - 70) * 1000000) / \ + (2424180 - (sclk_table->entries[i].clk/100) * 1132925/1000)); + volt_with_cks = (uint32_t)((2797202000U + sclk_table->entries[i].clk/100 * 3232 - (ro - 65) * 1000000) / \ + (2522480 - sclk_table->entries[i].clk/100 * 115764/100)); + } else { + volt_without_cks = (uint32_t)((2416794800U + (sclk_table->entries[i].clk/100) * 1476925/10 -(ro - 50) * 1000000) / \ + (2625416 - (sclk_table->entries[i].clk/100) * (12586807/10000))); + volt_with_cks = (uint32_t)((2999656000U - sclk_table->entries[i].clk/100 * 392803 - (ro - 44) * 1000000) / \ + (3422454 - sclk_table->entries[i].clk/100 * (18886376/10000))); + } + if (volt_without_cks >= volt_with_cks) volt_offset = (uint8_t)(((volt_without_cks - volt_with_cks + - sclk_table->entries[i].cks_voffset) * 100 / 625) + 1); + sclk_table->entries[i].cks_voffset) * 100 + 624) / 625); + data->smc_state_table.Sclk_voltageOffset[i] = volt_offset; } - PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE, - STRETCH_ENABLE, 0x0); - PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE, - masterReset, 0x1); - /* PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE, staticEnable, 0x1); */ - PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE, - masterReset, 0x0); - + data->smc_state_table.LdoRefSel = (table_info->cac_dtp_table->ucCKS_LDO_REFSEL != 0) ? table_info->cac_dtp_table->ucCKS_LDO_REFSEL : 6; /* Populate CKS Lookup Table */ if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5) stretch_amount2 = 0; @@ -1835,69 +1843,6 @@ static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr) return -EINVAL); } - value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, - ixPWR_CKS_CNTL); - value &= 0xFFC2FF87; - data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].minFreq = - polaris10_clock_stretcher_lookup_table[stretch_amount2][0]; - data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].maxFreq = - polaris10_clock_stretcher_lookup_table[stretch_amount2][1]; - clock_freq_u16 = (uint16_t)(PP_SMC_TO_HOST_UL(data->smc_state_table. - GraphicsLevel[data->smc_state_table.GraphicsDpmLevelCount - 1].SclkSetting.SclkFrequency) / 100); - if (polaris10_clock_stretcher_lookup_table[stretch_amount2][0] < clock_freq_u16 - && polaris10_clock_stretcher_lookup_table[stretch_amount2][1] > clock_freq_u16) { - /* Program PWR_CKS_CNTL. CKS_USE_FOR_LOW_FREQ */ - value |= (polaris10_clock_stretcher_lookup_table[stretch_amount2][3]) << 16; - /* Program PWR_CKS_CNTL. CKS_LDO_REFSEL */ - value |= (polaris10_clock_stretcher_lookup_table[stretch_amount2][2]) << 18; - /* Program PWR_CKS_CNTL. CKS_STRETCH_AMOUNT */ - value |= (polaris10_clock_stretch_amount_conversion - [polaris10_clock_stretcher_lookup_table[stretch_amount2][3]] - [stretch_amount]) << 3; - } - CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].minFreq); - CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].maxFreq); - data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting = - polaris10_clock_stretcher_lookup_table[stretch_amount2][2] & 0x7F; - data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting |= - (polaris10_clock_stretcher_lookup_table[stretch_amount2][3]) << 7; - - cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, - ixPWR_CKS_CNTL, value); - - /* Populate DDT Lookup Table */ - for (i = 0; i < 4; i++) { - /* Assign the minimum and maximum VID stored - * in the last row of Clock Stretcher Voltage Table. - */ - data->smc_state_table.ClockStretcherDataTable.ClockStretcherDataTableEntry[i].minVID = - (uint8_t) polaris10_clock_stretcher_ddt_table[type][i][2]; - data->smc_state_table.ClockStretcherDataTable.ClockStretcherDataTableEntry[i].maxVID = - (uint8_t) polaris10_clock_stretcher_ddt_table[type][i][3]; - /* Loop through each SCLK and check the frequency - * to see if it lies within the frequency for clock stretcher. - */ - for (j = 0; j < data->smc_state_table.GraphicsDpmLevelCount; j++) { - cks_setting = 0; - clock_freq = PP_SMC_TO_HOST_UL( - data->smc_state_table.GraphicsLevel[j].SclkSetting.SclkFrequency); - /* Check the allowed frequency against the sclk level[j]. - * Sclk's endianness has already been converted, - * and it's in 10Khz unit, - * as opposed to Data table, which is in Mhz unit. - */ - if (clock_freq >= (polaris10_clock_stretcher_ddt_table[type][i][0]) * 100) { - cks_setting |= 0x2; - if (clock_freq < (polaris10_clock_stretcher_ddt_table[type][i][1]) * 100) - cks_setting |= 0x1; - } - data->smc_state_table.ClockStretcherDataTable.ClockStretcherDataTableEntry[i].setting - |= cks_setting << (j * 2); - } - CONVERT_FROM_HOST_TO_SMC_US( - data->smc_state_table.ClockStretcherDataTable.ClockStretcherDataTableEntry[i].setting); - } - value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL); value &= 0xFFFFFFFE; cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL, value); @@ -1956,6 +1901,90 @@ static int polaris10_populate_vr_config(struct pp_hwmgr *hwmgr, return 0; } + +int polaris10_populate_avfs_parameters(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + SMU74_Discrete_DpmTable *table = &(data->smc_state_table); + int result = 0; + struct pp_atom_ctrl__avfs_parameters avfs_params = {0}; + AVFS_meanNsigma_t AVFS_meanNsigma = { {0} }; + AVFS_Sclk_Offset_t AVFS_SclkOffset = { {0} }; + uint32_t tmp, i; + struct pp_smumgr *smumgr = hwmgr->smumgr; + struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend); + + struct phm_ppt_v1_information *table_info = + (struct phm_ppt_v1_information *)hwmgr->pptable; + struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table = + table_info->vdd_dep_on_sclk; + + + if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED) + return result; + + result = atomctrl_get_avfs_information(hwmgr, &avfs_params); + + if (0 == result) { + table->BTCGB_VDROOP_TABLE[0].a0 = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a0); + table->BTCGB_VDROOP_TABLE[0].a1 = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a1); + table->BTCGB_VDROOP_TABLE[0].a2 = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a2); + table->BTCGB_VDROOP_TABLE[1].a0 = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a0); + table->BTCGB_VDROOP_TABLE[1].a1 = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a1); + table->BTCGB_VDROOP_TABLE[1].a2 = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a2); + table->AVFSGB_VDROOP_TABLE[0].m1 = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSON_m1); + table->AVFSGB_VDROOP_TABLE[0].m2 = PP_HOST_TO_SMC_US(avfs_params.usAVFSGB_FUSE_TABLE_CKSON_m2); + table->AVFSGB_VDROOP_TABLE[0].b = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSON_b); + table->AVFSGB_VDROOP_TABLE[0].m1_shift = 24; + table->AVFSGB_VDROOP_TABLE[0].m2_shift = 12; + table->AVFSGB_VDROOP_TABLE[1].m1 = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSOFF_m1); + table->AVFSGB_VDROOP_TABLE[1].m2 = PP_HOST_TO_SMC_US(avfs_params.usAVFSGB_FUSE_TABLE_CKSOFF_m2); + table->AVFSGB_VDROOP_TABLE[1].b = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSOFF_b); + table->AVFSGB_VDROOP_TABLE[1].m1_shift = 24; + table->AVFSGB_VDROOP_TABLE[1].m2_shift = 12; + table->MaxVoltage = PP_HOST_TO_SMC_US(avfs_params.usMaxVoltage_0_25mv); + AVFS_meanNsigma.Aconstant[0] = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant0); + AVFS_meanNsigma.Aconstant[1] = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant1); + AVFS_meanNsigma.Aconstant[2] = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant2); + AVFS_meanNsigma.DC_tol_sigma = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_DC_tol_sigma); + AVFS_meanNsigma.Platform_mean = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_Platform_mean); + AVFS_meanNsigma.PSM_Age_CompFactor = PP_HOST_TO_SMC_US(avfs_params.usPSM_Age_ComFactor); + AVFS_meanNsigma.Platform_sigma = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_Platform_sigma); + + for (i = 0; i < NUM_VFT_COLUMNS; i++) { + AVFS_meanNsigma.Static_Voltage_Offset[i] = (uint8_t)(sclk_table->entries[i].cks_voffset * 100 / 625); + AVFS_SclkOffset.Sclk_Offset[i] = PP_HOST_TO_SMC_US((uint16_t)(sclk_table->entries[i].sclk_offset) / 100); + } + + result = polaris10_read_smc_sram_dword(smumgr, + SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, AvfsMeanNSigma), + &tmp, data->sram_end); + + polaris10_copy_bytes_to_smc(smumgr, + tmp, + (uint8_t *)&AVFS_meanNsigma, + sizeof(AVFS_meanNsigma_t), + data->sram_end); + + result = polaris10_read_smc_sram_dword(smumgr, + SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, AvfsSclkOffsetTable), + &tmp, data->sram_end); + polaris10_copy_bytes_to_smc(smumgr, + tmp, + (uint8_t *)&AVFS_SclkOffset, + sizeof(AVFS_Sclk_Offset_t), + data->sram_end); + + data->avfs_vdroop_override_setting = (avfs_params.ucEnableGB_VDROOP_TABLE_CKSON << BTCGB0_Vdroop_Enable_SHIFT) | + (avfs_params.ucEnableGB_VDROOP_TABLE_CKSOFF << BTCGB1_Vdroop_Enable_SHIFT) | + (avfs_params.ucEnableGB_FUSE_TABLE_CKSON << AVFSGB0_Vdroop_Enable_SHIFT) | + (avfs_params.ucEnableGB_FUSE_TABLE_CKSOFF << AVFSGB1_Vdroop_Enable_SHIFT); + data->apply_avfs_cks_off_voltage = (avfs_params.ucEnableApplyAVFS_CKS_OFF_Voltage == 1) ? true : false; + } + return result; +} + + /** * Initializes the SMC table and uploads it * @@ -2056,6 +2085,10 @@ static int polaris10_init_smc_table(struct pp_hwmgr *hwmgr) "Failed to populate Clock Stretcher Data Table!", return result); } + + result = polaris10_populate_avfs_parameters(hwmgr); + PP_ASSERT_WITH_CODE(0 == result, "Failed to populate AVFS Parameters!", return result;); + table->CurrSclkPllRange = 0xff; table->GraphicsVoltageChangeEnable = 1; table->GraphicsThermThrottleEnable = 1; @@ -2252,6 +2285,9 @@ static int polaris10_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr) static int polaris10_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) { struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + uint32_t soft_register_value = 0; + uint32_t handshake_disables_offset = data->soft_regs_start + + offsetof(SMU74_SoftRegisters, HandshakeDisables); /* enable SCLK dpm */ if (!data->sclk_dpm_key_disabled) @@ -2262,6 +2298,12 @@ static int polaris10_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) /* enable MCLK dpm */ if (0 == data->mclk_dpm_key_disabled) { +/* Disable UVD - SMU handshake for MCLK. */ + soft_register_value = cgs_read_ind_register(hwmgr->device, + CGS_IND_REG__SMC, handshake_disables_offset); + soft_register_value |= SMU7_UVD_MCLK_HANDSHAKE_DISABLE; + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + handshake_disables_offset, soft_register_value); PP_ASSERT_WITH_CODE( (0 == smum_send_msg_to_smc(hwmgr->smumgr, @@ -2269,7 +2311,6 @@ static int polaris10_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) "Failed to enable MCLK DPM during DPM Start Function!", return -1); - PHM_WRITE_FIELD(hwmgr->device, MC_SEQ_CNTL_3, CAC_EN, 0x1); cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC0_CNTL, 0x5); @@ -2471,6 +2512,8 @@ int polaris10_enable_dpm_tasks(struct pp_hwmgr *hwmgr) PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to enable VR hot GPIO interrupt!", result = tmp_result); + smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay); + tmp_result = polaris10_enable_sclk_control(hwmgr); PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to enable SCLK control!", result = tmp_result); @@ -2606,6 +2649,7 @@ int polaris10_set_features_platform_caps(struct pp_hwmgr *hwmgr) phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_FanSpeedInTableIsRPM); + if (hwmgr->chip_id == CHIP_POLARIS11) phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SPLLShutdownSupport); @@ -2638,7 +2682,7 @@ static int polaris10_get_evv_voltages(struct pp_hwmgr *hwmgr) { struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); uint16_t vv_id; - uint16_t vddc = 0; + uint32_t vddc = 0; uint16_t i, j; uint32_t sclk = 0; struct phm_ppt_v1_information *table_info = @@ -2669,8 +2713,9 @@ static int polaris10_get_evv_voltages(struct pp_hwmgr *hwmgr) continue); - /* need to make sure vddc is less than 2v or else, it could burn the ASIC. */ - PP_ASSERT_WITH_CODE((vddc < 2000 && vddc != 0), + /* need to make sure vddc is less than 2v or else, it could burn the ASIC. + * real voltage level in unit of 0.01mv */ + PP_ASSERT_WITH_CODE((vddc < 200000 && vddc != 0), "Invalid VDDC value", result = -EINVAL;); /* the voltage should not be zero nor equal to leakage ID */ @@ -2896,6 +2941,31 @@ static int polaris10_set_private_data_based_on_pptable(struct pp_hwmgr *hwmgr) return 0; } +int polaris10_patch_voltage_workaround(struct pp_hwmgr *hwmgr) +{ + struct phm_ppt_v1_information *table_info = + (struct phm_ppt_v1_information *)(hwmgr->pptable); + struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table = + table_info->vdd_dep_on_mclk; + struct phm_ppt_v1_voltage_lookup_table *lookup_table = + table_info->vddc_lookup_table; + uint32_t i; + + if (hwmgr->chip_id == CHIP_POLARIS10 && hwmgr->hw_revision == 0xC7) { + if (lookup_table->entries[dep_mclk_table->entries[dep_mclk_table->count-1].vddInd].us_vdd >= 1000) + return 0; + + for (i = 0; i < lookup_table->count; i++) { + if (lookup_table->entries[i].us_vdd < 0xff01 && lookup_table->entries[i].us_vdd >= 1000) { + dep_mclk_table->entries[dep_mclk_table->count-1].vddInd = (uint8_t) i; + return 0; + } + } + } + return 0; +} + + int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr) { struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); @@ -2938,6 +3008,11 @@ int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr) data->vddci_control = POLARIS10_VOLTAGE_CONTROL_NONE; data->mvdd_control = POLARIS10_VOLTAGE_CONTROL_NONE; + data->enable_tdc_limit_feature = true; + data->enable_pkg_pwr_tracking_feature = true; + data->force_pcie_gen = PP_PCIEGenInvalid; + data->mclk_stutter_mode_threshold = 40000; + if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2)) data->voltage_control = POLARIS10_VOLTAGE_CONTROL_BY_SVID2; @@ -2962,8 +3037,13 @@ int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr) data->vddci_control = POLARIS10_VOLTAGE_CONTROL_BY_SVID2; } + if (table_info->cac_dtp_table->usClockStretchAmount != 0) + phm_cap_set(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_ClockStretcher); + polaris10_set_features_platform_caps(hwmgr); + polaris10_patch_voltage_workaround(hwmgr); polaris10_init_dpm_defaults(hwmgr); /* Get leakage voltage based on leakage ID. */ @@ -4333,6 +4413,15 @@ static int polaris10_notify_link_speed_change_after_state_change( return 0; } +static int polaris10_notify_smc_display(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + + smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, + (PPSMC_Msg)PPSMC_MSG_SetVBITimeout, data->frame_time_x2); + return (smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay) == 0) ? 0 : -EINVAL; +} + static int polaris10_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *input) { int tmp_result, result = 0; @@ -4381,6 +4470,11 @@ static int polaris10_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *i "Failed to program memory timing parameters!", result = tmp_result); + tmp_result = polaris10_notify_smc_display(hwmgr); + PP_ASSERT_WITH_CODE((0 == tmp_result), + "Failed to notify smc display settings!", + result = tmp_result); + tmp_result = polaris10_unfreeze_sclk_mclk_dpm(hwmgr); PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to unfreeze SCLK MCLK DPM!", @@ -4415,6 +4509,7 @@ static int polaris10_set_max_fan_pwm_output(struct pp_hwmgr *hwmgr, uint16_t us_ PPSMC_MSG_SetFanPwmMax, us_max_fan_pwm); } + int polaris10_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display) { PPSMC_Msg msg = has_display ? (PPSMC_Msg)PPSMC_HasDisplay : (PPSMC_Msg)PPSMC_NoDisplay; @@ -4434,8 +4529,6 @@ int polaris10_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwm if (num_active_displays > 1) /* to do && (pHwMgr->pPECI->displayConfiguration.bMultiMonitorInSync != TRUE)) */ polaris10_notify_smc_display_change(hwmgr, false); - else - polaris10_notify_smc_display_change(hwmgr, true); return 0; } @@ -4476,6 +4569,8 @@ int polaris10_program_display_gap(struct pp_hwmgr *hwmgr) frame_time_in_us = 1000000 / refresh_rate; pre_vbi_time_in_us = frame_time_in_us - 200 - mode_info.vblank_time_us; + data->frame_time_x2 = frame_time_in_us * 2 / 100; + display_gap2 = pre_vbi_time_in_us * (ref_clock / 100); cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL2, display_gap2); @@ -4484,8 +4579,6 @@ int polaris10_program_display_gap(struct pp_hwmgr *hwmgr) cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU74_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us)); - polaris10_notify_smc_display_change(hwmgr, num_active_displays != 0); - return 0; } @@ -4597,7 +4690,7 @@ int polaris10_upload_mc_firmware(struct pp_hwmgr *hwmgr) return 0; } - data->need_long_memory_training = true; + data->need_long_memory_training = false; /* * PPMCME_FirmwareDescriptorEntry *pfd = NULL; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h index beedf35cbfa6..afc3434822d1 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h @@ -312,6 +312,10 @@ struct polaris10_hwmgr { /* soft pptable for re-uploading into smu */ void *soft_pp_table; + + uint32_t avfs_vdroop_override_setting; + bool apply_avfs_cks_off_voltage; + uint32_t frame_time_x2; }; /* To convert to Q8.8 format for firmware */ diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.c index aba167f7d167..b206632d4650 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.c @@ -625,10 +625,14 @@ static int tf_polaris10_thermal_avfs_enable(struct pp_hwmgr *hwmgr, int ret; struct pp_smumgr *smumgr = (struct pp_smumgr *)(hwmgr->smumgr); struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend); + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); - if (smu_data->avfs.avfs_btc_status != AVFS_BTC_ENABLEAVFS) + if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED) return 0; + ret = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, + PPSMC_MSG_SetGBDroopSettings, data->avfs_vdroop_override_setting); + ret = (smum_send_msg_to_smc(smumgr, PPSMC_MSG_EnableAvfs) == 0) ? 0 : -1; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c index 58742e0d1492..a3c38bbd1e94 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c @@ -44,6 +44,20 @@ bool acpi_atcs_functions_supported(void *device, uint32_t index) return result == 0 ? (output_buf.function_bits & (1 << (index - 1))) != 0 : false; } +bool acpi_atcs_notify_pcie_device_ready(void *device) +{ + int32_t temp_buffer = 1; + + return cgs_call_acpi_method(device, CGS_ACPI_METHOD_ATCS, + ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION, + &temp_buffer, + NULL, + 0, + sizeof(temp_buffer), + 0); +} + + int acpi_pcie_perf_request(void *device, uint8_t perf_req, bool advertise) { struct atcs_pref_req_input atcs_input; @@ -52,7 +66,7 @@ int acpi_pcie_perf_request(void *device, uint8_t perf_req, bool advertise) int result; struct cgs_system_info info = {0}; - if (!acpi_atcs_functions_supported(device, ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST)) + if( 0 != acpi_atcs_notify_pcie_device_ready(device)) return -EINVAL; info.size = sizeof(struct cgs_system_info); @@ -77,7 +91,7 @@ int acpi_pcie_perf_request(void *device, uint8_t perf_req, bool advertise) ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST, &atcs_input, &atcs_output, - 0, + 1, sizeof(atcs_input), sizeof(atcs_output)); if (result != 0) diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c index da9f5f1b6dc2..90b35c5c10a4 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c @@ -1256,7 +1256,7 @@ int atomctrl_set_ac_timing_ai(struct pp_hwmgr *hwmgr, uint32_t memory_clock, } int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_type, - uint32_t sclk, uint16_t virtual_voltage_Id, uint16_t *voltage) + uint32_t sclk, uint16_t virtual_voltage_Id, uint32_t *voltage) { int result; @@ -1274,7 +1274,7 @@ int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_ if (0 != result) return result; - *voltage = get_voltage_info_param_space.usVoltageLevel; + *voltage = ((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *)(&get_voltage_info_param_space))->ulVoltageLevel; return result; } @@ -1302,3 +1302,46 @@ int atomctrl_get_smc_sclk_range_table(struct pp_hwmgr *hwmgr, struct pp_atom_ctr return 0; } + +int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr, struct pp_atom_ctrl__avfs_parameters *param) +{ + ATOM_ASIC_PROFILING_INFO_V3_6 *profile = NULL; + + if (param == NULL) + return -EINVAL; + + profile = (ATOM_ASIC_PROFILING_INFO_V3_6 *) + cgs_atom_get_data_table(hwmgr->device, + GetIndexIntoMasterTable(DATA, ASIC_ProfilingInfo), + NULL, NULL, NULL); + if (!profile) + return -1; + + param->ulAVFS_meanNsigma_Acontant0 = profile->ulAVFS_meanNsigma_Acontant0; + param->ulAVFS_meanNsigma_Acontant1 = profile->ulAVFS_meanNsigma_Acontant1; + param->ulAVFS_meanNsigma_Acontant2 = profile->ulAVFS_meanNsigma_Acontant2; + param->usAVFS_meanNsigma_DC_tol_sigma = profile->usAVFS_meanNsigma_DC_tol_sigma; + param->usAVFS_meanNsigma_Platform_mean = profile->usAVFS_meanNsigma_Platform_mean; + param->usAVFS_meanNsigma_Platform_sigma = profile->usAVFS_meanNsigma_Platform_sigma; + param->ulGB_VDROOP_TABLE_CKSOFF_a0 = profile->ulGB_VDROOP_TABLE_CKSOFF_a0; + param->ulGB_VDROOP_TABLE_CKSOFF_a1 = profile->ulGB_VDROOP_TABLE_CKSOFF_a1; + param->ulGB_VDROOP_TABLE_CKSOFF_a2 = profile->ulGB_VDROOP_TABLE_CKSOFF_a2; + param->ulGB_VDROOP_TABLE_CKSON_a0 = profile->ulGB_VDROOP_TABLE_CKSON_a0; + param->ulGB_VDROOP_TABLE_CKSON_a1 = profile->ulGB_VDROOP_TABLE_CKSON_a1; + param->ulGB_VDROOP_TABLE_CKSON_a2 = profile->ulGB_VDROOP_TABLE_CKSON_a2; + param->ulAVFSGB_FUSE_TABLE_CKSOFF_m1 = profile->ulAVFSGB_FUSE_TABLE_CKSOFF_m1; + param->usAVFSGB_FUSE_TABLE_CKSOFF_m2 = profile->usAVFSGB_FUSE_TABLE_CKSOFF_m2; + param->ulAVFSGB_FUSE_TABLE_CKSOFF_b = profile->ulAVFSGB_FUSE_TABLE_CKSOFF_b; + param->ulAVFSGB_FUSE_TABLE_CKSON_m1 = profile->ulAVFSGB_FUSE_TABLE_CKSON_m1; + param->usAVFSGB_FUSE_TABLE_CKSON_m2 = profile->usAVFSGB_FUSE_TABLE_CKSON_m2; + param->ulAVFSGB_FUSE_TABLE_CKSON_b = profile->ulAVFSGB_FUSE_TABLE_CKSON_b; + param->usMaxVoltage_0_25mv = profile->usMaxVoltage_0_25mv; + param->ucEnableGB_VDROOP_TABLE_CKSOFF = profile->ucEnableGB_VDROOP_TABLE_CKSOFF; + param->ucEnableGB_VDROOP_TABLE_CKSON = profile->ucEnableGB_VDROOP_TABLE_CKSON; + param->ucEnableGB_FUSE_TABLE_CKSOFF = profile->ucEnableGB_FUSE_TABLE_CKSOFF; + param->ucEnableGB_FUSE_TABLE_CKSON = profile->ucEnableGB_FUSE_TABLE_CKSON; + param->usPSM_Age_ComFactor = profile->usPSM_Age_ComFactor; + param->ucEnableApplyAVFS_CKS_OFF_Voltage = profile->ucEnableApplyAVFS_CKS_OFF_Voltage; + + return 0; +} diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h index d24ebb566905..1e35a9625baf 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h @@ -250,6 +250,35 @@ struct pp_atomctrl_gpio_pin_assignment { }; typedef struct pp_atomctrl_gpio_pin_assignment pp_atomctrl_gpio_pin_assignment; +struct pp_atom_ctrl__avfs_parameters { + uint32_t ulAVFS_meanNsigma_Acontant0; + uint32_t ulAVFS_meanNsigma_Acontant1; + uint32_t ulAVFS_meanNsigma_Acontant2; + uint16_t usAVFS_meanNsigma_DC_tol_sigma; + uint16_t usAVFS_meanNsigma_Platform_mean; + uint16_t usAVFS_meanNsigma_Platform_sigma; + uint32_t ulGB_VDROOP_TABLE_CKSOFF_a0; + uint32_t ulGB_VDROOP_TABLE_CKSOFF_a1; + uint32_t ulGB_VDROOP_TABLE_CKSOFF_a2; + uint32_t ulGB_VDROOP_TABLE_CKSON_a0; + uint32_t ulGB_VDROOP_TABLE_CKSON_a1; + uint32_t ulGB_VDROOP_TABLE_CKSON_a2; + uint32_t ulAVFSGB_FUSE_TABLE_CKSOFF_m1; + uint16_t usAVFSGB_FUSE_TABLE_CKSOFF_m2; + uint32_t ulAVFSGB_FUSE_TABLE_CKSOFF_b; + uint32_t ulAVFSGB_FUSE_TABLE_CKSON_m1; + uint16_t usAVFSGB_FUSE_TABLE_CKSON_m2; + uint32_t ulAVFSGB_FUSE_TABLE_CKSON_b; + uint16_t usMaxVoltage_0_25mv; + uint8_t ucEnableGB_VDROOP_TABLE_CKSOFF; + uint8_t ucEnableGB_VDROOP_TABLE_CKSON; + uint8_t ucEnableGB_FUSE_TABLE_CKSOFF; + uint8_t ucEnableGB_FUSE_TABLE_CKSON; + uint16_t usPSM_Age_ComFactor; + uint8_t ucEnableApplyAVFS_CKS_OFF_Voltage; + uint8_t ucReserved; +}; + extern bool atomctrl_get_pp_assign_pin(struct pp_hwmgr *hwmgr, const uint32_t pinId, pp_atomctrl_gpio_pin_assignment *gpio_pin_assignment); extern int atomctrl_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type, uint32_t sclk, uint16_t virtual_voltage_Id, uint16_t *voltage); extern uint32_t atomctrl_get_mpll_reference_clock(struct pp_hwmgr *hwmgr); @@ -276,7 +305,10 @@ extern int atomctrl_get_engine_pll_dividers_ai(struct pp_hwmgr *hwmgr, uint32_t extern int atomctrl_set_ac_timing_ai(struct pp_hwmgr *hwmgr, uint32_t memory_clock, uint8_t level); extern int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_type, - uint32_t sclk, uint16_t virtual_voltage_Id, uint16_t *voltage); + uint32_t sclk, uint16_t virtual_voltage_Id, uint32_t *voltage); extern int atomctrl_get_smc_sclk_range_table(struct pp_hwmgr *hwmgr, struct pp_atom_ctrl_sclk_range_table *table); + +extern int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr, struct pp_atom_ctrl__avfs_parameters *param); + #endif diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c index d27e8c40602a..5d0f655bf160 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c @@ -1302,7 +1302,7 @@ static int tonga_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr, table->Smio[count] |= data->mvdd_voltage_table.entries[count].smio_low; } - table->SmioMask2 = data->vddci_voltage_table.mask_low; + table->SmioMask2 = data->mvdd_voltage_table.mask_low; CONVERT_FROM_HOST_TO_SMC_UL(table->MvddLevelCount); } @@ -4489,6 +4489,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) data->vdd_ci_control = TONGA_VOLTAGE_CONTROL_NONE; data->vdd_gfx_control = TONGA_VOLTAGE_CONTROL_NONE; data->mvdd_control = TONGA_VOLTAGE_CONTROL_NONE; + data->force_pcie_gen = PP_PCIEGenInvalid; if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2)) { diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c index 671fdb4d615a..dccc859f638c 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c @@ -302,7 +302,7 @@ static int init_dpm_2_parameters( (((unsigned long)powerplay_table) + le16_to_cpu(powerplay_table->usPPMTableOffset)); if (0 != powerplay_table->usPPMTableOffset) { - if (1 == get_platform_power_management_table(hwmgr, atom_ppm_table)) { + if (get_platform_power_management_table(hwmgr, atom_ppm_table) == 0) { phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_EnablePlatformPowerManagement); } diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h index 28f571449495..77e8e33d5870 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h +++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h @@ -411,6 +411,8 @@ struct phm_cac_tdp_table { uint8_t ucVr_I2C_Line; uint8_t ucPlx_I2C_address; uint8_t ucPlx_I2C_Line; + uint32_t usBoostPowerLimit; + uint8_t ucCKS_LDO_REFSEL; }; struct phm_ppm_table { diff --git a/drivers/gpu/drm/amd/powerplay/inc/polaris10_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/polaris10_ppsmc.h index 0c6a413eaa5b..b8f4b73c322e 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/polaris10_ppsmc.h +++ b/drivers/gpu/drm/amd/powerplay/inc/polaris10_ppsmc.h @@ -27,6 +27,7 @@ #pragma pack(push, 1) +#define PPSMC_MSG_SetGBDroopSettings ((uint16_t) 0x305) #define PPSMC_SWSTATE_FLAG_DC 0x01 #define PPSMC_SWSTATE_FLAG_UVD 0x02 @@ -391,6 +392,8 @@ typedef uint16_t PPSMC_Result; #define PPSMC_MSG_SetGpuPllDfsForSclk ((uint16_t) 0x300) #define PPSMC_MSG_Didt_Block_Function ((uint16_t) 0x301) +#define PPSMC_MSG_SetVBITimeout ((uint16_t) 0x306) + #define PPSMC_MSG_SecureSRBMWrite ((uint16_t) 0x600) #define PPSMC_MSG_SecureSRBMRead ((uint16_t) 0x601) #define PPSMC_MSG_SetAddress ((uint16_t) 0x800) diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h b/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h index 3bd5e69b9045..3df5de2cdab0 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h +++ b/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h @@ -26,3 +26,4 @@ extern bool acpi_atcs_functions_supported(void *device, extern int acpi_pcie_perf_request(void *device, uint8_t perf_req, bool advertise); +extern bool acpi_atcs_notify_pcie_device_ready(void *device); diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu74.h b/drivers/gpu/drm/amd/powerplay/inc/smu74.h index 1a12d85b8e97..fd10a9fa843d 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu74.h +++ b/drivers/gpu/drm/amd/powerplay/inc/smu74.h @@ -34,6 +34,30 @@ #define SMU__NUM_LCLK_DPM_LEVELS 8 #define SMU__NUM_PCIE_DPM_LEVELS 8 +#define EXP_M1 35 +#define EXP_M2 92821 +#define EXP_B 66629747 + +#define EXP_M1_1 365 +#define EXP_M2_1 658700 +#define EXP_B_1 305506134 + +#define EXP_M1_2 189 +#define EXP_M2_2 379692 +#define EXP_B_2 194609469 + +#define EXP_M1_3 99 +#define EXP_M2_3 217915 +#define EXP_B_3 122255994 + +#define EXP_M1_4 51 +#define EXP_M2_4 122643 +#define EXP_B_4 74893384 + +#define EXP_M1_5 423 +#define EXP_M2_5 1103326 +#define EXP_B_5 728122621 + enum SID_OPTION { SID_OPTION_HI, SID_OPTION_LO, @@ -548,20 +572,20 @@ struct SMU74_Firmware_Header { uint32_t CacConfigTable; uint32_t CacStatusTable; - uint32_t mcRegisterTable; - uint32_t mcArbDramTimingTable; - - - uint32_t PmFuseTable; uint32_t Globals; uint32_t ClockStretcherTable; uint32_t VftTable; - uint32_t Reserved[21]; + uint32_t Reserved1; + uint32_t AvfsTable; + uint32_t AvfsCksOffGbvTable; + uint32_t AvfsMeanNSigma; + uint32_t AvfsSclkOffsetTable; + uint32_t Reserved[16]; uint32_t Signature; }; @@ -701,8 +725,6 @@ VR Config info is contained in dpmTable.VRConfig */ struct SMU_ClockStretcherDataTableEntry { uint8_t minVID; uint8_t maxVID; - - uint16_t setting; }; typedef struct SMU_ClockStretcherDataTableEntry SMU_ClockStretcherDataTableEntry; @@ -769,6 +791,43 @@ struct VFT_TABLE_t { typedef struct VFT_TABLE_t VFT_TABLE_t; +/* Total margin, root mean square of Fmax + DC + Platform */ +struct AVFS_Margin_t { + VFT_CELL_t Cell[NUM_VFT_COLUMNS]; +}; +typedef struct AVFS_Margin_t AVFS_Margin_t; + +#define BTCGB_VDROOP_TABLE_MAX_ENTRIES 2 +#define AVFSGB_VDROOP_TABLE_MAX_ENTRIES 2 + +struct GB_VDROOP_TABLE_t { + int32_t a0; + int32_t a1; + int32_t a2; + uint32_t spare; +}; +typedef struct GB_VDROOP_TABLE_t GB_VDROOP_TABLE_t; + +struct AVFS_CksOff_Gbv_t { + VFT_CELL_t Cell[NUM_VFT_COLUMNS]; +}; +typedef struct AVFS_CksOff_Gbv_t AVFS_CksOff_Gbv_t; + +struct AVFS_meanNsigma_t { + uint32_t Aconstant[3]; + uint16_t DC_tol_sigma; + uint16_t Platform_mean; + uint16_t Platform_sigma; + uint16_t PSM_Age_CompFactor; + uint8_t Static_Voltage_Offset[NUM_VFT_COLUMNS]; +}; +typedef struct AVFS_meanNsigma_t AVFS_meanNsigma_t; + +struct AVFS_Sclk_Offset_t { + uint16_t Sclk_Offset[8]; +}; +typedef struct AVFS_Sclk_Offset_t AVFS_Sclk_Offset_t; + #endif diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu74_discrete.h b/drivers/gpu/drm/amd/powerplay/inc/smu74_discrete.h index 0dfe82336dc7..899d6d8108c2 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu74_discrete.h +++ b/drivers/gpu/drm/amd/powerplay/inc/smu74_discrete.h @@ -223,6 +223,16 @@ struct SMU74_Discrete_StateInfo { typedef struct SMU74_Discrete_StateInfo SMU74_Discrete_StateInfo; +struct SMU_QuadraticCoeffs { + int32_t m1; + uint32_t b; + + int16_t m2; + uint8_t m1_shift; + uint8_t m2_shift; +}; +typedef struct SMU_QuadraticCoeffs SMU_QuadraticCoeffs; + struct SMU74_Discrete_DpmTable { SMU74_PIDController GraphicsPIDController; @@ -258,7 +268,15 @@ struct SMU74_Discrete_DpmTable { uint8_t ThermOutPolarity; uint8_t ThermOutMode; uint8_t BootPhases; - uint32_t Reserved[4]; + + uint8_t VRHotLevel; + uint8_t LdoRefSel; + uint8_t Reserved1[2]; + uint16_t FanStartTemperature; + uint16_t FanStopTemperature; + uint16_t MaxVoltage; + uint16_t Reserved2; + uint32_t Reserved[1]; SMU74_Discrete_GraphicsLevel GraphicsLevel[SMU74_MAX_LEVELS_GRAPHICS]; SMU74_Discrete_MemoryLevel MemoryACPILevel; @@ -347,6 +365,8 @@ struct SMU74_Discrete_DpmTable { uint32_t CurrSclkPllRange; sclkFcwRange_t SclkFcwRangeTable[NUM_SCLK_RANGE]; + GB_VDROOP_TABLE_t BTCGB_VDROOP_TABLE[BTCGB_VDROOP_TABLE_MAX_ENTRIES]; + SMU_QuadraticCoeffs AVFSGB_VDROOP_TABLE[AVFSGB_VDROOP_TABLE_MAX_ENTRIES]; }; typedef struct SMU74_Discrete_DpmTable SMU74_Discrete_DpmTable; @@ -550,16 +570,6 @@ struct SMU7_AcpiScoreboard { typedef struct SMU7_AcpiScoreboard SMU7_AcpiScoreboard; -struct SMU_QuadraticCoeffs { - int32_t m1; - uint32_t b; - - int16_t m2; - uint8_t m1_shift; - uint8_t m2_shift; -}; -typedef struct SMU_QuadraticCoeffs SMU_QuadraticCoeffs; - struct SMU74_Discrete_PmFuses { uint8_t BapmVddCVidHiSidd[8]; uint8_t BapmVddCVidLoSidd[8]; @@ -821,6 +831,17 @@ typedef struct SMU7_GfxCuPgScoreboard SMU7_GfxCuPgScoreboard; #define DB_PCC_SHIFT 26 #define DB_EDC_SHIFT 27 +#define BTCGB0_Vdroop_Enable_MASK 0x1 +#define BTCGB1_Vdroop_Enable_MASK 0x2 +#define AVFSGB0_Vdroop_Enable_MASK 0x4 +#define AVFSGB1_Vdroop_Enable_MASK 0x8 + +#define BTCGB0_Vdroop_Enable_SHIFT 0 +#define BTCGB1_Vdroop_Enable_SHIFT 1 +#define AVFSGB0_Vdroop_Enable_SHIFT 2 +#define AVFSGB1_Vdroop_Enable_SHIFT 3 + + #pragma pack(pop) diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c index 043b6ac09d5f..5dba7c509710 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c @@ -52,19 +52,18 @@ static const SMU74_Discrete_GraphicsLevel avfs_graphics_level_polaris10[8] = { /* Min pcie DeepSleep Activity CgSpll CgSpll CcPwr CcPwr Sclk Enabled Enabled Voltage Power */ /* Voltage, DpmLevel, DivId, Level, FuncCntl3, FuncCntl4, DynRm, DynRm1 Did, Padding,ForActivity, ForThrottle, UpHyst, DownHyst, DownHyst, Throttle */ - { 0x3c0fd047, 0x00, 0x03, 0x1e00, 0x00200410, 0x87020000, 0, 0, 0x16, 0, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, { 0x30750000, 0, 0, 0, 0, 0, 0, 0 } }, - { 0xa00fd047, 0x01, 0x04, 0x1e00, 0x00800510, 0x87020000, 0, 0, 0x16, 0, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, { 0x409c0000, 0, 0, 0, 0, 0, 0, 0 } }, - { 0x0410d047, 0x01, 0x00, 0x1e00, 0x00600410, 0x87020000, 0, 0, 0x0e, 0, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, { 0x50c30000, 0, 0, 0, 0, 0, 0, 0 } }, - { 0x6810d047, 0x01, 0x00, 0x1e00, 0x00800410, 0x87020000, 0, 0, 0x0c, 0, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, { 0x60ea0000, 0, 0, 0, 0, 0, 0, 0 } }, - { 0xcc10d047, 0x01, 0x00, 0x1e00, 0x00e00410, 0x87020000, 0, 0, 0x0c, 0, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, { 0xe8fd0000, 0, 0, 0, 0, 0, 0, 0 } }, - { 0x3011d047, 0x01, 0x00, 0x1e00, 0x00400510, 0x87020000, 0, 0, 0x0c, 0, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, { 0x70110100, 0, 0, 0, 0, 0, 0, 0 } }, - { 0x9411d047, 0x01, 0x00, 0x1e00, 0x00a00510, 0x87020000, 0, 0, 0x0c, 0, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, { 0xf8240100, 0, 0, 0, 0, 0, 0, 0 } }, - { 0xf811d047, 0x01, 0x00, 0x1e00, 0x00000610, 0x87020000, 0, 0, 0x0c, 0, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, { 0x80380100, 0, 0, 0, 0, 0, 0, 0 } } + { 0x100ea446, 0x00, 0x03, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0x30750000, 0x3000, 0, 0x2600, 0, 0, 0x0004, 0x8f02, 0xffff, 0x2f00, 0x300e, 0x2700 } }, + { 0x400ea446, 0x01, 0x04, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0x409c0000, 0x2000, 0, 0x1e00, 1, 1, 0x0004, 0x8300, 0xffff, 0x1f00, 0xcb5e, 0x1a00 } }, + { 0x740ea446, 0x01, 0x00, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0x50c30000, 0x2800, 0, 0x2000, 1, 1, 0x0004, 0x0c02, 0xffff, 0x2700, 0x6433, 0x2100 } }, + { 0xa40ea446, 0x01, 0x00, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0x60ea0000, 0x3000, 0, 0x2600, 1, 1, 0x0004, 0x8f02, 0xffff, 0x2f00, 0x300e, 0x2700 } }, + { 0xd80ea446, 0x01, 0x00, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0x70110100, 0x3800, 0, 0x2c00, 1, 1, 0x0004, 0x1203, 0xffff, 0x3600, 0xc9e2, 0x2e00 } }, + { 0x3c0fa446, 0x01, 0x00, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0x80380100, 0x2000, 0, 0x1e00, 2, 1, 0x0004, 0x8300, 0xffff, 0x1f00, 0xcb5e, 0x1a00 } }, + { 0x6c0fa446, 0x01, 0x00, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0x905f0100, 0x2400, 0, 0x1e00, 2, 1, 0x0004, 0x8901, 0xffff, 0x2300, 0x314c, 0x1d00 } }, + { 0xa00fa446, 0x01, 0x00, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0xa0860100, 0x2800, 0, 0x2000, 2, 1, 0x0004, 0x0c02, 0xffff, 0x2700, 0x6433, 0x2100 } } }; static const SMU74_Discrete_MemoryLevel avfs_memory_level_polaris10 = - {0x50140000, 0x50140000, 0x00320000, 0x00, 0x00, - 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x0000, 0x00, 0x00}; + {0x100ea446, 0, 0x30750000, 0x01, 0x01, 0x01, 0x00, 0x00, 0x64, 0x00, 0x00, 0x1f00, 0x00, 0x00}; /** * Set the address for reading/writing the SMC SRAM space. @@ -219,6 +218,18 @@ bool polaris10_is_smc_ram_running(struct pp_smumgr *smumgr) && (0x20100 <= cgs_read_ind_register(smumgr->device, CGS_IND_REG__SMC, ixSMC_PC_C))); } +static bool polaris10_is_hw_avfs_present(struct pp_smumgr *smumgr) +{ + uint32_t efuse; + + efuse = cgs_read_ind_register(smumgr->device, CGS_IND_REG__SMC, ixSMU_EFUSE_0 + (49*4)); + efuse &= 0x00000001; + if (efuse) + return true; + + return false; +} + /** * Send a message to the SMC, and wait for its response. * @@ -228,21 +239,27 @@ bool polaris10_is_smc_ram_running(struct pp_smumgr *smumgr) */ int polaris10_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg) { + int ret; + if (!polaris10_is_smc_ram_running(smumgr)) return -1; + SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0); - if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP)) - printk("Failed to send Previous Message.\n"); + ret = SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP); + if (ret != 1) + printk("\n failed to send pre message %x ret is %d \n", msg, ret); cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg); SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0); - if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP)) - printk("Failed to send Message.\n"); + ret = SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP); + + if (ret != 1) + printk("\n failed to send message %x ret is %d \n", msg, ret); return 0; } @@ -953,6 +970,11 @@ static int polaris10_smu_init(struct pp_smumgr *smumgr) (cgs_handle_t)smu_data->smu_buffer.handle); return -1;); + if (polaris10_is_hw_avfs_present(smumgr)) + smu_data->avfs.avfs_btc_status = AVFS_BTC_BOOT; + else + smu_data->avfs.avfs_btc_status = AVFS_BTC_NOTSUPPORTED; + return 0; } diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c index 39802c0539b6..3d34fc4ca826 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c @@ -266,9 +266,10 @@ int atmel_hlcdc_create_outputs(struct drm_device *dev) if (!ret) ret = atmel_hlcdc_check_endpoint(dev, &ep); - of_node_put(ep_np); - if (ret) + if (ret) { + of_node_put(ep_np); return ret; + } } for_each_endpoint_of_node(dev->dev->of_node, ep_np) { @@ -276,9 +277,10 @@ int atmel_hlcdc_create_outputs(struct drm_device *dev) if (!ret) ret = atmel_hlcdc_attach_endpoint(dev, &ep); - of_node_put(ep_np); - if (ret) + if (ret) { + of_node_put(ep_np); return ret; + } } return 0; diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c index aef3ca8a81fa..016c191221f3 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c @@ -339,6 +339,8 @@ atmel_hlcdc_plane_update_pos_and_size(struct atmel_hlcdc_plane *plane, atmel_hlcdc_layer_update_cfg(&plane->layer, 13, 0xffffffff, factor_reg); + } else { + atmel_hlcdc_layer_update_cfg(&plane->layer, 13, 0xffffffff, 0); } } diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c index c204ef32df16..9bb99e274d23 100644 --- a/drivers/gpu/drm/drm_atomic.c +++ b/drivers/gpu/drm/drm_atomic.c @@ -1296,14 +1296,39 @@ EXPORT_SYMBOL(drm_atomic_add_affected_planes); */ void drm_atomic_legacy_backoff(struct drm_atomic_state *state) { + struct drm_device *dev = state->dev; + unsigned crtc_mask = 0; + struct drm_crtc *crtc; int ret; + bool global = false; + + drm_for_each_crtc(crtc, dev) { + if (crtc->acquire_ctx != state->acquire_ctx) + continue; + + crtc_mask |= drm_crtc_mask(crtc); + crtc->acquire_ctx = NULL; + } + + if (WARN_ON(dev->mode_config.acquire_ctx == state->acquire_ctx)) { + global = true; + + dev->mode_config.acquire_ctx = NULL; + } retry: drm_modeset_backoff(state->acquire_ctx); - ret = drm_modeset_lock_all_ctx(state->dev, state->acquire_ctx); + ret = drm_modeset_lock_all_ctx(dev, state->acquire_ctx); if (ret) goto retry; + + drm_for_each_crtc(crtc, dev) + if (drm_crtc_mask(crtc) & crtc_mask) + crtc->acquire_ctx = state->acquire_ctx; + + if (global) + dev->mode_config.acquire_ctx = state->acquire_ctx; } EXPORT_SYMBOL(drm_atomic_legacy_backoff); diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c index f6223f907c15..7f9901b7777b 100644 --- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c +++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c @@ -31,7 +31,6 @@ #include "exynos_drm_plane.h" #include "exynos_drm_drv.h" #include "exynos_drm_fb.h" -#include "exynos_drm_fbdev.h" #include "exynos_drm_iommu.h" /* diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c index 468498e3fec1..4c1fb3f8b5a6 100644 --- a/drivers/gpu/drm/exynos/exynos_dp.c +++ b/drivers/gpu/drm/exynos/exynos_dp.c @@ -34,7 +34,7 @@ struct exynos_dp_device { struct drm_encoder encoder; - struct drm_connector connector; + struct drm_connector *connector; struct drm_bridge *ptn_bridge; struct drm_device *drm_dev; struct device *dev; @@ -70,7 +70,7 @@ static int exynos_dp_poweroff(struct analogix_dp_plat_data *plat_data) static int exynos_dp_get_modes(struct analogix_dp_plat_data *plat_data) { struct exynos_dp_device *dp = to_dp(plat_data); - struct drm_connector *connector = &dp->connector; + struct drm_connector *connector = dp->connector; struct drm_display_mode *mode; int num_modes = 0; @@ -103,6 +103,7 @@ static int exynos_dp_bridge_attach(struct analogix_dp_plat_data *plat_data, int ret; drm_connector_register(connector); + dp->connector = connector; /* Pre-empt DP connector creation if there's a bridge */ if (dp->ptn_bridge) { diff --git a/drivers/gpu/drm/exynos/exynos_drm_core.c b/drivers/gpu/drm/exynos/exynos_drm_core.c index 011211e4167d..edbd98ff293e 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_core.c +++ b/drivers/gpu/drm/exynos/exynos_drm_core.c @@ -15,7 +15,6 @@ #include #include "exynos_drm_drv.h" #include "exynos_drm_crtc.h" -#include "exynos_drm_fbdev.h" static LIST_HEAD(exynos_drm_subdrv_list); diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c index 3efe1aa89416..d47216488985 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c +++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c @@ -30,7 +30,6 @@ #include "exynos_drm_drv.h" #include "exynos_drm_fb.h" -#include "exynos_drm_fbdev.h" #include "exynos_drm_crtc.h" #include "exynos_drm_plane.h" #include "exynos_drm_iommu.h" @@ -120,7 +119,6 @@ static struct fimd_driver_data s3c64xx_fimd_driver_data = { .timing_base = 0x0, .has_clksel = 1, .has_limited_fmt = 1, - .has_hw_trigger = 1, }; static struct fimd_driver_data exynos3_fimd_driver_data = { @@ -171,14 +169,11 @@ static struct fimd_driver_data exynos5420_fimd_driver_data = { .lcdblk_vt_shift = 24, .lcdblk_bypass_shift = 15, .lcdblk_mic_bypass_shift = 11, - .trg_type = I80_HW_TRG, .has_shadowcon = 1, .has_vidoutcon = 1, .has_vtsel = 1, .has_mic_bypass = 1, .has_dp_clk = 1, - .has_hw_trigger = 1, - .has_trigger_per_te = 1, }; struct fimd_context { diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c index 493552368295..8564c3da0d22 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c +++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c @@ -48,13 +48,13 @@ /* registers for base address */ #define G2D_SRC_BASE_ADDR 0x0304 -#define G2D_SRC_STRIDE_REG 0x0308 +#define G2D_SRC_STRIDE 0x0308 #define G2D_SRC_COLOR_MODE 0x030C #define G2D_SRC_LEFT_TOP 0x0310 #define G2D_SRC_RIGHT_BOTTOM 0x0314 #define G2D_SRC_PLANE2_BASE_ADDR 0x0318 #define G2D_DST_BASE_ADDR 0x0404 -#define G2D_DST_STRIDE_REG 0x0408 +#define G2D_DST_STRIDE 0x0408 #define G2D_DST_COLOR_MODE 0x040C #define G2D_DST_LEFT_TOP 0x0410 #define G2D_DST_RIGHT_BOTTOM 0x0414 @@ -563,7 +563,7 @@ static enum g2d_reg_type g2d_get_reg_type(int reg_offset) switch (reg_offset) { case G2D_SRC_BASE_ADDR: - case G2D_SRC_STRIDE_REG: + case G2D_SRC_STRIDE: case G2D_SRC_COLOR_MODE: case G2D_SRC_LEFT_TOP: case G2D_SRC_RIGHT_BOTTOM: @@ -573,7 +573,7 @@ static enum g2d_reg_type g2d_get_reg_type(int reg_offset) reg_type = REG_TYPE_SRC_PLANE2; break; case G2D_DST_BASE_ADDR: - case G2D_DST_STRIDE_REG: + case G2D_DST_STRIDE: case G2D_DST_COLOR_MODE: case G2D_DST_LEFT_TOP: case G2D_DST_RIGHT_BOTTOM: @@ -968,8 +968,8 @@ static int g2d_check_reg_offset(struct device *dev, } else buf_info->types[reg_type] = BUF_TYPE_GEM; break; - case G2D_SRC_STRIDE_REG: - case G2D_DST_STRIDE_REG: + case G2D_SRC_STRIDE: + case G2D_DST_STRIDE: if (for_addr) goto err; diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c index 55f1d37c666a..77f12c00abf9 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_plane.c +++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c @@ -242,7 +242,7 @@ exynos_drm_plane_check_size(const struct exynos_drm_plane_config *config, state->v_ratio == (1 << 15)) height_ok = true; - if (width_ok & height_ok) + if (width_ok && height_ok) return 0; DRM_DEBUG_KMS("scaling mode is not supported"); diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 32690332d441..103546834b60 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -2365,16 +2365,16 @@ static int i915_ppgtt_info(struct seq_file *m, void *data) task = get_pid_task(file->pid, PIDTYPE_PID); if (!task) { ret = -ESRCH; - goto out_put; + goto out_unlock; } seq_printf(m, "\nproc: %s\n", task->comm); put_task_struct(task); idr_for_each(&file_priv->context_idr, per_file_ctx, (void *)(unsigned long)m); } +out_unlock: mutex_unlock(&dev->filelist_mutex); -out_put: intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index f313b4d8344f..85c4debf47e0 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -512,6 +512,10 @@ void intel_detect_pch(struct drm_device *dev) DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n"); WARN_ON(!IS_SKYLAKE(dev) && !IS_KABYLAKE(dev)); + } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) { + dev_priv->pch_type = PCH_KBP; + DRM_DEBUG_KMS("Found KabyPoint PCH\n"); + WARN_ON(!IS_KABYLAKE(dev)); } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) || (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) || ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) && diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 7c334e902266..bc3f2e6842e7 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -990,6 +990,7 @@ enum intel_pch { PCH_CPT, /* Cougarpoint PCH */ PCH_LPT, /* Lynxpoint PCH */ PCH_SPT, /* Sunrisepoint PCH */ + PCH_KBP, /* Kabypoint PCH */ PCH_NOP, }; @@ -2600,6 +2601,15 @@ struct drm_i915_cmd_table { #define IS_BXT_REVID(p, since, until) (IS_BROXTON(p) && IS_REVID(p, since, until)) +#define KBL_REVID_A0 0x0 +#define KBL_REVID_B0 0x1 +#define KBL_REVID_C0 0x2 +#define KBL_REVID_D0 0x3 +#define KBL_REVID_E0 0x4 + +#define IS_KBL_REVID(p, since, until) \ + (IS_KABYLAKE(p) && IS_REVID(p, since, until)) + /* * The genX designation typically refers to the render engine, so render * capability related checks should use IS_GEN, while display and other checks @@ -2708,11 +2718,13 @@ struct drm_i915_cmd_table { #define INTEL_PCH_LPT_LP_DEVICE_ID_TYPE 0x9c00 #define INTEL_PCH_SPT_DEVICE_ID_TYPE 0xA100 #define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE 0x9D00 +#define INTEL_PCH_KBP_DEVICE_ID_TYPE 0xA200 #define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100 #define INTEL_PCH_P3X_DEVICE_ID_TYPE 0x7000 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE 0x2900 /* qemu q35 has 2918 */ #define INTEL_PCH_TYPE(dev) (__I915__(dev)->pch_type) +#define HAS_PCH_KBP(dev) (INTEL_PCH_TYPE(dev) == PCH_KBP) #define HAS_PCH_SPT(dev) (INTEL_PCH_TYPE(dev) == PCH_SPT) #define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT) #define HAS_PCH_LPT_LP(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index 425e721aac58..66571466e9a8 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -40,7 +40,7 @@ static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) if (!mutex_is_locked(mutex)) return false; -#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES) +#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER) return mutex->owner == task; #else /* Since UP may be pre-empted, we cannot assume that we own the lock */ diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index b7ce963fb8f8..44004e3f09e4 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -55,8 +55,10 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, return -ENODEV; /* See the comment at the drm_mm_init() call for more about this check. - * WaSkipStolenMemoryFirstPage:bdw,chv (incomplete) */ - if (INTEL_INFO(dev_priv)->gen == 8 && start < 4096) + * WaSkipStolenMemoryFirstPage:bdw,chv,kbl (incomplete) + */ + if (start < 4096 && (IS_GEN8(dev_priv) || + IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0))) start = 4096; mutex_lock(&dev_priv->mm.stolen_lock); diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 2f6fd33c07ba..aab47f7bb61b 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -2471,7 +2471,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) I915_WRITE(SDEIIR, iir); ret = IRQ_HANDLED; - if (HAS_PCH_SPT(dev_priv)) + if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv)) spt_irq_handler(dev, iir); else cpt_irq_handler(dev, iir); @@ -4661,7 +4661,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv) dev->driver->disable_vblank = gen8_disable_vblank; if (IS_BROXTON(dev)) dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup; - else if (HAS_PCH_SPT(dev)) + else if (HAS_PCH_SPT(dev) || HAS_PCH_KBP(dev)) dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup; else dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index b407411e31ba..3fcf7dd5b6ca 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -220,6 +220,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ECOCHK_PPGTT_WT_HSW (0x2<<3) #define ECOCHK_PPGTT_WB_HSW (0x3<<3) +#define GEN8_CONFIG0 _MMIO(0xD00) +#define GEN9_DEFAULT_FIXES (1 << 3 | 1 << 2 | 1 << 1) + #define GAC_ECO_BITS _MMIO(0x14090) #define ECOBITS_SNB_BIT (1<<13) #define ECOBITS_PPGTT_CACHE64B (3<<8) @@ -1669,6 +1672,9 @@ enum skl_disp_power_wells { #define GEN7_TLB_RD_ADDR _MMIO(0x4700) +#define GAMT_CHKN_BIT_REG _MMIO(0x4ab8) +#define GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING (1<<28) + #if 0 #define PRB0_TAIL _MMIO(0x2030) #define PRB0_HEAD _MMIO(0x2034) @@ -1804,6 +1810,10 @@ enum skl_disp_power_wells { #define GEN9_IZ_HASHING_MASK(slice) (0x3 << ((slice) * 2)) #define GEN9_IZ_HASHING(slice, val) ((val) << ((slice) * 2)) +/* chicken reg for WaConextSwitchWithConcurrentTLBInvalidate */ +#define GEN9_CSFE_CHICKEN1_RCS _MMIO(0x20D4) +#define GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE (1 << 2) + /* WaClearTdlStateAckDirtyBits */ #define GEN8_STATE_ACK _MMIO(0x20F0) #define GEN9_STATE_ACK_SLICE1 _MMIO(0x20F8) @@ -2200,6 +2210,8 @@ enum skl_disp_power_wells { #define ILK_DPFC_STATUS _MMIO(0x43210) #define ILK_DPFC_FENCE_YOFF _MMIO(0x43218) #define ILK_DPFC_CHICKEN _MMIO(0x43224) +#define ILK_DPFC_DISABLE_DUMMY0 (1<<8) +#define ILK_DPFC_NUKE_ON_ANY_MODIFICATION (1<<23) #define ILK_FBC_RT_BASE _MMIO(0x2128) #define ILK_FBC_RT_VALID (1<<0) #define SNB_FBC_FRONT_BUFFER (1<<1) @@ -6031,6 +6043,7 @@ enum skl_disp_power_wells { #define CHICKEN_PAR1_1 _MMIO(0x42080) #define DPA_MASK_VBLANK_SRD (1 << 15) #define FORCE_ARB_IDLE_PLANES (1 << 14) +#define SKL_EDP_PSR_FIX_RDWRAP (1 << 3) #define _CHICKEN_PIPESL_1_A 0x420b0 #define _CHICKEN_PIPESL_1_B 0x420b4 @@ -6039,6 +6052,7 @@ enum skl_disp_power_wells { #define CHICKEN_PIPESL_1(pipe) _MMIO_PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B) #define DISP_ARB_CTL _MMIO(0x45000) +#define DISP_FBC_MEMORY_WAKE (1<<31) #define DISP_TILE_SURFACE_SWIZZLING (1<<13) #define DISP_FBC_WM_DIS (1<<15) #define DISP_ARB_CTL2 _MMIO(0x45004) @@ -6052,6 +6066,9 @@ enum skl_disp_power_wells { #define HSW_NDE_RSTWRN_OPT _MMIO(0x46408) #define RESET_PCH_HANDSHAKE_ENABLE (1<<4) +#define GEN8_CHICKEN_DCPR_1 _MMIO(0x46430) +#define MASK_WAKEMEM (1<<13) + #define SKL_DFSM _MMIO(0x51000) #define SKL_DFSM_CDCLK_LIMIT_MASK (3 << 23) #define SKL_DFSM_CDCLK_LIMIT_675 (0 << 23) @@ -6069,6 +6086,7 @@ enum skl_disp_power_wells { #define GEN9_TSG_BARRIER_ACK_DISABLE (1<<8) #define GEN9_CS_DEBUG_MODE1 _MMIO(0x20ec) +#define GEN9_CTX_PREEMPT_REG _MMIO(0x2248) #define GEN8_CS_CHICKEN1 _MMIO(0x2580) /* GEN7 chicken */ @@ -6076,6 +6094,7 @@ enum skl_disp_power_wells { # define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC ((1<<10) | (1<<26)) # define GEN9_RHWO_OPTIMIZATION_DISABLE (1<<14) #define COMMON_SLICE_CHICKEN2 _MMIO(0x7014) +# define GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION (1<<8) # define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE (1<<0) #define HIZ_CHICKEN _MMIO(0x7018) @@ -6921,6 +6940,7 @@ enum skl_disp_power_wells { #define EDRAM_SETS_IDX(cap) (((cap) >> 8) & 0x3) #define GEN6_UCGCTL1 _MMIO(0x9400) +# define GEN6_GAMUNIT_CLOCK_GATE_DISABLE (1 << 22) # define GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE (1 << 16) # define GEN6_BLBUNIT_CLOCK_GATE_DISABLE (1 << 5) # define GEN6_CSUNIT_CLOCK_GATE_DISABLE (1 << 7) @@ -6937,6 +6957,7 @@ enum skl_disp_power_wells { #define GEN7_UCGCTL4 _MMIO(0x940c) #define GEN7_L3BANK2X_CLOCK_GATE_DISABLE (1<<25) +#define GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE (1<<14) #define GEN6_RCGCTL1 _MMIO(0x9410) #define GEN6_RCGCTL2 _MMIO(0x9414) diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c index a34c23eceba0..2b3b428d9cd2 100644 --- a/drivers/gpu/drm/i915/intel_csr.c +++ b/drivers/gpu/drm/i915/intel_csr.c @@ -41,16 +41,22 @@ * be moved to FW_FAILED. */ +#define I915_CSR_KBL "i915/kbl_dmc_ver1.bin" +MODULE_FIRMWARE(I915_CSR_KBL); +#define KBL_CSR_VERSION_REQUIRED CSR_VERSION(1, 1) + #define I915_CSR_SKL "i915/skl_dmc_ver1.bin" +MODULE_FIRMWARE(I915_CSR_SKL); +#define SKL_CSR_VERSION_REQUIRED CSR_VERSION(1, 23) + #define I915_CSR_BXT "i915/bxt_dmc_ver1.bin" +MODULE_FIRMWARE(I915_CSR_BXT); +#define BXT_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) #define FIRMWARE_URL "https://01.org/linuxgraphics/intel-linux-graphics-firmwares" -MODULE_FIRMWARE(I915_CSR_SKL); -MODULE_FIRMWARE(I915_CSR_BXT); -#define SKL_CSR_VERSION_REQUIRED CSR_VERSION(1, 23) -#define BXT_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) + #define CSR_MAX_FW_SIZE 0x2FFF #define CSR_DEFAULT_FW_OFFSET 0xFFFFFFFF @@ -169,12 +175,10 @@ struct stepping_info { char substepping; }; -/* - * Kabylake derivated from Skylake H0, so SKL H0 - * is the right firmware for KBL A0 (revid 0). - */ static const struct stepping_info kbl_stepping_info[] = { - {'H', '0'}, {'I', '0'} + {'A', '0'}, {'B', '0'}, {'C', '0'}, + {'D', '0'}, {'E', '0'}, {'F', '0'}, + {'G', '0'}, {'H', '0'}, {'I', '0'}, }; static const struct stepping_info skl_stepping_info[] = { @@ -298,7 +302,9 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv, csr->version = css_header->version; - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + if (IS_KABYLAKE(dev_priv)) { + required_min_version = KBL_CSR_VERSION_REQUIRED; + } else if (IS_SKYLAKE(dev_priv)) { required_min_version = SKL_CSR_VERSION_REQUIRED; } else if (IS_BROXTON(dev_priv)) { required_min_version = BXT_CSR_VERSION_REQUIRED; @@ -446,7 +452,9 @@ void intel_csr_ucode_init(struct drm_i915_private *dev_priv) if (!HAS_CSR(dev_priv)) return; - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) + if (IS_KABYLAKE(dev_priv)) + csr->fw_path = I915_CSR_KBL; + else if (IS_SKYLAKE(dev_priv)) csr->fw_path = I915_CSR_SKL; else if (IS_BROXTON(dev_priv)) csr->fw_path = I915_CSR_BXT; diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 56a1637c864f..3074c56a643d 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -8447,16 +8447,16 @@ static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv) tmp |= FDI_MPHY_IOSFSB_RESET_CTL; I915_WRITE(SOUTH_CHICKEN2, tmp); - if (wait_for_atomic_us(I915_READ(SOUTH_CHICKEN2) & - FDI_MPHY_IOSFSB_RESET_STATUS, 100)) + if (wait_for_us(I915_READ(SOUTH_CHICKEN2) & + FDI_MPHY_IOSFSB_RESET_STATUS, 100)) DRM_ERROR("FDI mPHY reset assert timeout\n"); tmp = I915_READ(SOUTH_CHICKEN2); tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL; I915_WRITE(SOUTH_CHICKEN2, tmp); - if (wait_for_atomic_us((I915_READ(SOUTH_CHICKEN2) & - FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100)) + if (wait_for_us((I915_READ(SOUTH_CHICKEN2) & + FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100)) DRM_ERROR("FDI mPHY reset de-assert timeout\n"); } @@ -9440,8 +9440,8 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv, val |= LCPLL_CD_SOURCE_FCLK; I915_WRITE(LCPLL_CTL, val); - if (wait_for_atomic_us(I915_READ(LCPLL_CTL) & - LCPLL_CD_SOURCE_FCLK_DONE, 1)) + if (wait_for_us(I915_READ(LCPLL_CTL) & + LCPLL_CD_SOURCE_FCLK_DONE, 1)) DRM_ERROR("Switching to FCLK failed\n"); val = I915_READ(LCPLL_CTL); @@ -9514,8 +9514,8 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) val &= ~LCPLL_CD_SOURCE_FCLK; I915_WRITE(LCPLL_CTL, val); - if (wait_for_atomic_us((I915_READ(LCPLL_CTL) & - LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1)) + if (wait_for_us((I915_READ(LCPLL_CTL) & + LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1)) DRM_ERROR("Switching back to LCPLL failed\n"); } @@ -11997,6 +11997,12 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, ret = intel_color_check(crtc, crtc_state); if (ret) return ret; + + /* + * Changing color management on Intel hardware is + * handled as part of planes update. + */ + crtc_state->planes_changed = true; } ret = 0; diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index ffe5f8430957..891107f92d9f 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -663,7 +663,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq) done = wait_event_timeout(dev_priv->gmbus_wait_queue, C, msecs_to_jiffies_timeout(10)); else - done = wait_for_atomic(C, 10) == 0; + done = wait_for(C, 10) == 0; if (!done) DRM_ERROR("dp aux hw did not signal timeout (has irq: %i)!\n", has_aux_irq); @@ -4645,7 +4645,7 @@ intel_dp_detect(struct drm_connector *connector, bool force) intel_dp->detect_done = false; - if (intel_connector->detect_edid) + if (is_edp(intel_dp) || intel_connector->detect_edid) return connector_status_connected; else return connector_status_disconnected; @@ -4899,13 +4899,15 @@ static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp) void intel_dp_encoder_reset(struct drm_encoder *encoder) { - struct intel_dp *intel_dp; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + if (!HAS_DDI(dev_priv)) + intel_dp->DP = I915_READ(intel_dp->output_reg); if (to_intel_encoder(encoder)->type != INTEL_OUTPUT_EDP) return; - intel_dp = enc_to_intel_dp(encoder); - pps_lock(intel_dp); /* @@ -4977,9 +4979,6 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) intel_display_power_get(dev_priv, power_domain); if (long_hpd) { - /* indicate that we need to restart link training */ - intel_dp->train_set_valid = false; - intel_dp_long_pulse(intel_dp->attached_connector); if (intel_dp->is_mst) ret = IRQ_HANDLED; diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c index 0b8eefc2acc5..60fb39cd220b 100644 --- a/drivers/gpu/drm/i915/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/intel_dp_link_training.c @@ -85,8 +85,7 @@ static bool intel_dp_reset_link_train(struct intel_dp *intel_dp, uint8_t dp_train_pat) { - if (!intel_dp->train_set_valid) - memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); + memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); intel_dp_set_signal_levels(intel_dp); return intel_dp_set_link_train(intel_dp, dp_train_pat); } @@ -161,23 +160,6 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp) break; } - /* - * if we used previously trained voltage and pre-emphasis values - * and we don't get clock recovery, reset link training values - */ - if (intel_dp->train_set_valid) { - DRM_DEBUG_KMS("clock recovery not ok, reset"); - /* clear the flag as we are not reusing train set */ - intel_dp->train_set_valid = false; - if (!intel_dp_reset_link_train(intel_dp, - DP_TRAINING_PATTERN_1 | - DP_LINK_SCRAMBLING_DISABLE)) { - DRM_ERROR("failed to enable link training\n"); - return; - } - continue; - } - /* Check to see if we've tried the max voltage */ for (i = 0; i < intel_dp->lane_count; i++) if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0) @@ -284,7 +266,6 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) /* Make sure clock is still ok */ if (!drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { - intel_dp->train_set_valid = false; intel_dp_link_training_clock_recovery(intel_dp); intel_dp_set_link_train(intel_dp, training_pattern | @@ -301,7 +282,6 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) /* Try 5 times, then try clock recovery if that fails */ if (tries > 5) { - intel_dp->train_set_valid = false; intel_dp_link_training_clock_recovery(intel_dp); intel_dp_set_link_train(intel_dp, training_pattern | @@ -322,10 +302,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) intel_dp_set_idle_link_train(intel_dp); - if (channel_eq) { - intel_dp->train_set_valid = true; + if (channel_eq) DRM_DEBUG_KMS("Channel EQ done. DP Training successful\n"); - } } void intel_dp_stop_link_train(struct intel_dp *intel_dp) diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c index baf6f5584cbd..58f60b27837e 100644 --- a/drivers/gpu/drm/i915/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c @@ -1377,8 +1377,8 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp); POSTING_READ(BXT_PORT_PLL_ENABLE(port)); - if (wait_for_atomic_us((I915_READ(BXT_PORT_PLL_ENABLE(port)) & - PORT_PLL_LOCK), 200)) + if (wait_for_us((I915_READ(BXT_PORT_PLL_ENABLE(port)) & PORT_PLL_LOCK), + 200)) DRM_ERROR("PLL %d not locked\n", port); /* diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 4a24b0067a3a..f7f0f01814f6 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -863,8 +863,6 @@ struct intel_dp { /* This is called before a link training is starterd */ void (*prepare_link_retrain)(struct intel_dp *intel_dp); - bool train_set_valid; - /* Displayport compliance testing */ unsigned long compliance_test_type; unsigned long compliance_test_data; diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c index d5a7cfec589b..647127f3aaff 100644 --- a/drivers/gpu/drm/i915/intel_fbc.c +++ b/drivers/gpu/drm/i915/intel_fbc.c @@ -824,8 +824,7 @@ static bool intel_fbc_can_choose(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; struct intel_fbc *fbc = &dev_priv->fbc; - bool enable_by_default = IS_HASWELL(dev_priv) || - IS_BROADWELL(dev_priv); + bool enable_by_default = IS_BROADWELL(dev_priv); if (intel_vgpu_active(dev_priv->dev)) { fbc->no_fbc_reason = "VGPU is active"; diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 42eac37de047..7f2d8415ed8b 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -1103,15 +1103,17 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, uint32_t *const batch, uint32_t index) { + struct drm_i915_private *dev_priv = engine->dev->dev_private; uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES); /* - * WaDisableLSQCROPERFforOCL:skl + * WaDisableLSQCROPERFforOCL:skl,kbl * This WA is implemented in skl_init_clock_gating() but since * this batch updates GEN8_L3SQCREG4 with default value we need to * set this bit here to retain the WA during flush. */ - if (IS_SKL_REVID(engine->dev, 0, SKL_REVID_E0)) + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0) || + IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0)) l3sqc4_flush |= GEN8_LQSC_RO_PERF_DIS; wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 | @@ -1273,6 +1275,7 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine, { int ret; struct drm_device *dev = engine->dev; + struct drm_i915_private *dev_priv = dev->dev_private; uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); /* WaDisableCtxRestoreArbitration:skl,bxt */ @@ -1286,6 +1289,22 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine, return ret; index = ret; + /* WaClearSlmSpaceAtContextSwitch:kbl */ + /* Actual scratch location is at 128 bytes offset */ + if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0)) { + uint32_t scratch_addr + = engine->scratch.gtt_offset + 2*CACHELINE_BYTES; + + wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6)); + wa_ctx_emit(batch, index, (PIPE_CONTROL_FLUSH_L3 | + PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_QW_WRITE)); + wa_ctx_emit(batch, index, scratch_addr); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + } /* Pad to end of cacheline */ while (index % CACHELINE_DWORDS) wa_ctx_emit(batch, index, MI_NOOP); @@ -1687,9 +1706,10 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, struct intel_ringbuffer *ringbuf = request->ringbuf; struct intel_engine_cs *engine = ringbuf->engine; u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES; - bool vf_flush_wa = false; + bool vf_flush_wa = false, dc_flush_wa = false; u32 flags = 0; int ret; + int len; flags |= PIPE_CONTROL_CS_STALL; @@ -1716,9 +1736,21 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, */ if (IS_GEN9(engine->dev)) vf_flush_wa = true; + + /* WaForGAMHang:kbl */ + if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0)) + dc_flush_wa = true; } - ret = intel_ring_begin(request, vf_flush_wa ? 12 : 6); + len = 6; + + if (vf_flush_wa) + len += 6; + + if (dc_flush_wa) + len += 12; + + ret = intel_ring_begin(request, len); if (ret) return ret; @@ -1731,12 +1763,31 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, intel_logical_ring_emit(ringbuf, 0); } + if (dc_flush_wa) { + intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6)); + intel_logical_ring_emit(ringbuf, PIPE_CONTROL_DC_FLUSH_ENABLE); + intel_logical_ring_emit(ringbuf, 0); + intel_logical_ring_emit(ringbuf, 0); + intel_logical_ring_emit(ringbuf, 0); + intel_logical_ring_emit(ringbuf, 0); + } + intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6)); intel_logical_ring_emit(ringbuf, flags); intel_logical_ring_emit(ringbuf, scratch_addr); intel_logical_ring_emit(ringbuf, 0); intel_logical_ring_emit(ringbuf, 0); intel_logical_ring_emit(ringbuf, 0); + + if (dc_flush_wa) { + intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6)); + intel_logical_ring_emit(ringbuf, PIPE_CONTROL_CS_STALL); + intel_logical_ring_emit(ringbuf, 0); + intel_logical_ring_emit(ringbuf, 0); + intel_logical_ring_emit(ringbuf, 0); + intel_logical_ring_emit(ringbuf, 0); + } + intel_logical_ring_advance(ringbuf); return 0; diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index 99e26034ae8d..16e209d326b6 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -1038,5 +1038,16 @@ intel_opregion_get_panel_type(struct drm_device *dev) return -ENODEV; } + /* + * FIXME On Dell XPS 13 9350 the OpRegion panel type (0) gives us + * low vswing for eDP, whereas the VBT panel type (2) gives us normal + * vswing instead. Low vswing results in some display flickers, so + * let's simply ignore the OpRegion panel type on SKL for now. + */ + if (IS_SKYLAKE(dev)) { + DRM_DEBUG_KMS("Ignoring OpRegion panel type (%d)\n", ret - 1); + return -ENODEV; + } + return ret - 1; } diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index 8357d571553a..aba94099886b 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -1731,7 +1731,8 @@ intel_panel_init_backlight_funcs(struct intel_panel *panel) panel->backlight.set = bxt_set_backlight; panel->backlight.get = bxt_get_backlight; panel->backlight.hz_to_pwm = bxt_hz_to_pwm; - } else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_SPT(dev_priv)) { + } else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_SPT(dev_priv) || + HAS_PCH_KBP(dev_priv)) { panel->backlight.setup = lpt_setup_backlight; panel->backlight.enable = lpt_enable_backlight; panel->backlight.disable = lpt_disable_backlight; diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index a7ef45da0a9e..2863b92c9da6 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -54,10 +54,38 @@ #define INTEL_RC6p_ENABLE (1<<1) #define INTEL_RC6pp_ENABLE (1<<2) +static void gen9_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl */ + I915_WRITE(CHICKEN_PAR1_1, + I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP); + + I915_WRITE(GEN8_CONFIG0, + I915_READ(GEN8_CONFIG0) | GEN9_DEFAULT_FIXES); + + /* WaEnableChickenDCPR:skl,bxt,kbl */ + I915_WRITE(GEN8_CHICKEN_DCPR_1, + I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); + + /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl */ + /* WaFbcWakeMemOn:skl,bxt,kbl */ + I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | + DISP_FBC_WM_DIS | + DISP_FBC_MEMORY_WAKE); + + /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl */ + I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + ILK_DPFC_DISABLE_DUMMY0); +} + static void bxt_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + gen9_init_clock_gating(dev); + /* WaDisableSDEUnitClockGating:bxt */ I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); @@ -6698,6 +6726,38 @@ static void lpt_suspend_hw(struct drm_device *dev) } } +static void kabylake_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + gen9_init_clock_gating(dev); + + /* WaDisableSDEUnitClockGating:kbl */ + if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + GEN8_SDEUNIT_CLOCK_GATE_DISABLE); + + /* WaDisableGamClockGating:kbl */ + if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | + GEN6_GAMUNIT_CLOCK_GATE_DISABLE); + + /* WaFbcNukeOnHostModify:kbl */ + I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + ILK_DPFC_NUKE_ON_ANY_MODIFICATION); +} + +static void skylake_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + gen9_init_clock_gating(dev); + + /* WaFbcNukeOnHostModify:skl */ + I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + ILK_DPFC_NUKE_ON_ANY_MODIFICATION); +} + static void broadwell_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -7163,9 +7223,9 @@ static void nop_init_clock_gating(struct drm_device *dev) void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) { if (IS_SKYLAKE(dev_priv)) - dev_priv->display.init_clock_gating = nop_init_clock_gating; + dev_priv->display.init_clock_gating = skylake_init_clock_gating; else if (IS_KABYLAKE(dev_priv)) - dev_priv->display.init_clock_gating = nop_init_clock_gating; + dev_priv->display.init_clock_gating = kabylake_init_clock_gating; else if (IS_BROXTON(dev_priv)) dev_priv->display.init_clock_gating = bxt_init_clock_gating; else if (IS_BROADWELL(dev_priv)) diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 04402bb9d26b..68c5af079ef8 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -913,24 +913,26 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine) { struct drm_device *dev = engine->dev; struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t tmp; int ret; - /* WaEnableLbsSlaRetryTimerDecrement:skl */ + /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl */ + I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE)); + + /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl */ I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) | GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE); - /* WaDisableKillLogic:bxt,skl */ + /* WaDisableKillLogic:bxt,skl,kbl */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | ECOCHK_DIS_TLB); - /* WaClearFlowControlGpgpuContextSave:skl,bxt */ - /* WaDisablePartialInstShootdown:skl,bxt */ + /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl */ + /* WaDisablePartialInstShootdown:skl,bxt,kbl */ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, FLOW_CONTROL_ENABLE | PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); - /* Syncing dependencies between camera and graphics:skl,bxt */ + /* Syncing dependencies between camera and graphics:skl,bxt,kbl */ WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC); @@ -952,18 +954,18 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine) */ } - /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */ - /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt */ + /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl */ + /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl */ WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7, GEN9_ENABLE_YV12_BUGFIX | GEN9_ENABLE_GPGPU_PREEMPTION); - /* Wa4x4STCOptimizationDisable:skl,bxt */ - /* WaDisablePartialResolveInVc:skl,bxt */ + /* Wa4x4STCOptimizationDisable:skl,bxt,kbl */ + /* WaDisablePartialResolveInVc:skl,bxt,kbl */ WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE | GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE)); - /* WaCcsTlbPrefetchDisable:skl,bxt */ + /* WaCcsTlbPrefetchDisable:skl,bxt,kbl */ WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5, GEN9_CCS_TLB_PREFETCH_ENABLE); @@ -973,31 +975,57 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine) WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0, PIXEL_MASK_CAMMING_DISABLE); - /* WaForceContextSaveRestoreNonCoherent:skl,bxt */ - tmp = HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT; - if (IS_SKL_REVID(dev, SKL_REVID_F0, REVID_FOREVER) || - IS_BXT_REVID(dev, BXT_REVID_B0, REVID_FOREVER)) - tmp |= HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE; - WA_SET_BIT_MASKED(HDC_CHICKEN0, tmp); + /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl */ + WA_SET_BIT_MASKED(HDC_CHICKEN0, + HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | + HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE); + + /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are + * both tied to WaForceContextSaveRestoreNonCoherent + * in some hsds for skl. We keep the tie for all gen9. The + * documentation is a bit hazy and so we want to get common behaviour, + * even though there is no clear evidence we would need both on kbl/bxt. + * This area has been source of system hangs so we play it safe + * and mimic the skl regardless of what bspec says. + * + * Use Force Non-Coherent whenever executing a 3D context. This + * is a workaround for a possible hang in the unlikely event + * a TLB invalidation occurs during a PSD flush. + */ - /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt */ - if (IS_SKYLAKE(dev) || IS_BXT_REVID(dev, 0, BXT_REVID_B0)) + /* WaForceEnableNonCoherent:skl,bxt,kbl */ + WA_SET_BIT_MASKED(HDC_CHICKEN0, + HDC_FORCE_NON_COHERENT); + + /* WaDisableHDCInvalidation:skl,bxt,kbl */ + I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | + BDW_DISABLE_HDC_INVALIDATION); + + /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl */ + if (IS_SKYLAKE(dev_priv) || + IS_KABYLAKE(dev_priv) || + IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, GEN8_SAMPLER_POWER_BYPASS_DIS); - /* WaDisableSTUnitPowerOptimization:skl,bxt */ + /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl */ WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE); - /* WaOCLCoherentLineFlush:skl,bxt */ + /* WaOCLCoherentLineFlush:skl,bxt,kbl */ I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) | GEN8_LQSC_FLUSH_COHERENT_LINES)); - /* WaEnablePreemptionGranularityControlByUMD:skl,bxt */ + /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt */ + ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG); + if (ret) + return ret; + + /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl */ ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1); if (ret) return ret; - /* WaAllowUMDToModifyHDCChicken1:skl,bxt */ + /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl */ ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1); if (ret) return ret; @@ -1092,22 +1120,6 @@ static int skl_init_workarounds(struct intel_engine_cs *engine) WA_SET_BIT_MASKED(HIZ_CHICKEN, BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE); - /* This is tied to WaForceContextSaveRestoreNonCoherent */ - if (IS_SKL_REVID(dev, 0, REVID_FOREVER)) { - /* - *Use Force Non-Coherent whenever executing a 3D context. This - * is a workaround for a possible hang in the unlikely event - * a TLB invalidation occurs during a PSD flush. - */ - /* WaForceEnableNonCoherent:skl */ - WA_SET_BIT_MASKED(HDC_CHICKEN0, - HDC_FORCE_NON_COHERENT); - - /* WaDisableHDCInvalidation:skl */ - I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | - BDW_DISABLE_HDC_INVALIDATION); - } - /* WaBarrierPerformanceFixDisable:skl */ if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0)) WA_SET_BIT_MASKED(HDC_CHICKEN0, @@ -1120,6 +1132,9 @@ static int skl_init_workarounds(struct intel_engine_cs *engine) GEN7_HALF_SLICE_CHICKEN1, GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); + /* WaDisableGafsUnitClkGating:skl */ + WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); + /* WaDisableLSQCROPERFforOCL:skl */ ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4); if (ret) @@ -1174,6 +1189,63 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine) return ret; } + /* WaInsertDummyPushConstPs:bxt */ + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) + WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, + GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); + + return 0; +} + +static int kbl_init_workarounds(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->dev->dev_private; + int ret; + + ret = gen9_init_workarounds(engine); + if (ret) + return ret; + + /* WaEnableGapsTsvCreditFix:kbl */ + I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) | + GEN9_GAPS_TSV_CREDIT_DISABLE)); + + /* WaDisableDynamicCreditSharing:kbl */ + if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + WA_SET_BIT(GAMT_CHKN_BIT_REG, + GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING); + + /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */ + if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0)) + WA_SET_BIT_MASKED(HDC_CHICKEN0, + HDC_FENCE_DEST_SLM_DISABLE); + + /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes + * involving this register should also be added to WA batch as required. + */ + if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0)) + /* WaDisableLSQCROPERFforOCL:kbl */ + I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | + GEN8_LQSC_RO_PERF_DIS); + + /* WaInsertDummyPushConstPs:kbl */ + if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, + GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); + + /* WaDisableGafsUnitClkGating:kbl */ + WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); + + /* WaDisableSbeCacheDispatchPortSharing:kbl */ + WA_SET_BIT_MASKED( + GEN7_HALF_SLICE_CHICKEN1, + GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); + + /* WaDisableLSQCROPERFforOCL:kbl */ + ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4); + if (ret) + return ret; + return 0; } @@ -1199,6 +1271,9 @@ int init_workarounds_ring(struct intel_engine_cs *engine) if (IS_BROXTON(dev)) return bxt_init_workarounds(engine); + if (IS_KABYLAKE(dev_priv)) + return kbl_init_workarounds(engine); + return 0; } diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c index 300ea03be8f0..d1f248fd3506 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c +++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c @@ -552,7 +552,8 @@ nouveau_fbcon_init(struct drm_device *dev) if (ret) goto fini; - fbcon->helper.fbdev->pixmap.buf_align = 4; + if (fbcon->helper.fbdev) + fbcon->helper.fbdev->pixmap.buf_align = 4; return 0; fini: diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c index 22706c0a54b5..49bd5da194e1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c @@ -40,7 +40,8 @@ static int gf119_sor_dp_pattern(struct nvkm_output_dp *outp, int pattern) { struct nvkm_device *device = outp->base.disp->engine.subdev.device; - nvkm_mask(device, 0x61c110, 0x0f0f0f0f, 0x01010101 * pattern); + const u32 soff = gf119_sor_soff(outp); + nvkm_mask(device, 0x61c110 + soff, 0x0f0f0f0f, 0x01010101 * pattern); return 0; } diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c index e671a7cd3463..6ac717f2056f 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c @@ -148,40 +148,39 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane) struct rcar_du_vsp_plane_state *state = to_rcar_vsp_plane_state(plane->plane.state); struct drm_framebuffer *fb = plane->plane.state->fb; - struct v4l2_rect src; - struct v4l2_rect dst; - dma_addr_t paddr[2] = { 0, }; - u32 pixelformat = 0; + struct vsp1_du_atomic_config cfg = { + .pixelformat = 0, + .pitch = fb->pitches[0], + .alpha = state->alpha, + .zpos = state->zpos, + }; unsigned int i; - src.left = state->state.src_x >> 16; - src.top = state->state.src_y >> 16; - src.width = state->state.src_w >> 16; - src.height = state->state.src_h >> 16; + cfg.src.left = state->state.src_x >> 16; + cfg.src.top = state->state.src_y >> 16; + cfg.src.width = state->state.src_w >> 16; + cfg.src.height = state->state.src_h >> 16; - dst.left = state->state.crtc_x; - dst.top = state->state.crtc_y; - dst.width = state->state.crtc_w; - dst.height = state->state.crtc_h; + cfg.dst.left = state->state.crtc_x; + cfg.dst.top = state->state.crtc_y; + cfg.dst.width = state->state.crtc_w; + cfg.dst.height = state->state.crtc_h; for (i = 0; i < state->format->planes; ++i) { struct drm_gem_cma_object *gem; gem = drm_fb_cma_get_gem_obj(fb, i); - paddr[i] = gem->paddr + fb->offsets[i]; + cfg.mem[i] = gem->paddr + fb->offsets[i]; } for (i = 0; i < ARRAY_SIZE(formats_kms); ++i) { if (formats_kms[i] == state->format->fourcc) { - pixelformat = formats_v4l2[i]; + cfg.pixelformat = formats_v4l2[i]; break; } } - WARN_ON(!pixelformat); - - vsp1_du_atomic_update(plane->vsp->vsp, plane->index, pixelformat, - fb->pitches[0], paddr, &src, &dst); + vsp1_du_atomic_update(plane->vsp->vsp, plane->index, &cfg); } static int rcar_du_vsp_plane_atomic_check(struct drm_plane *plane, @@ -220,8 +219,7 @@ static void rcar_du_vsp_plane_atomic_update(struct drm_plane *plane, if (plane->state->crtc) rcar_du_vsp_plane_setup(rplane); else - vsp1_du_atomic_update(rplane->vsp->vsp, rplane->index, 0, 0, 0, - NULL, NULL); + vsp1_du_atomic_update(rplane->vsp->vsp, rplane->index, NULL); } static const struct drm_plane_helper_funcs rcar_du_vsp_plane_helper_funcs = { @@ -269,6 +267,7 @@ static void rcar_du_vsp_plane_reset(struct drm_plane *plane) return; state->alpha = 255; + state->zpos = plane->type == DRM_PLANE_TYPE_PRIMARY ? 0 : 1; plane->state = &state->state; plane->state->plane = plane; @@ -283,6 +282,8 @@ static int rcar_du_vsp_plane_atomic_set_property(struct drm_plane *plane, if (property == rcdu->props.alpha) rstate->alpha = val; + else if (property == rcdu->props.zpos) + rstate->zpos = val; else return -EINVAL; @@ -299,6 +300,8 @@ static int rcar_du_vsp_plane_atomic_get_property(struct drm_plane *plane, if (property == rcdu->props.alpha) *val = rstate->alpha; + else if (property == rcdu->props.zpos) + *val = rstate->zpos; else return -EINVAL; @@ -378,6 +381,8 @@ int rcar_du_vsp_init(struct rcar_du_vsp *vsp) drm_object_attach_property(&plane->plane.base, rcdu->props.alpha, 255); + drm_object_attach_property(&plane->plane.base, + rcdu->props.zpos, 1); } return 0; diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.h b/drivers/gpu/drm/rcar-du/rcar_du_vsp.h index df3bf3805c69..510dcc9c6816 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.h +++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.h @@ -44,6 +44,7 @@ static inline struct rcar_du_vsp_plane *to_rcar_vsp_plane(struct drm_plane *p) * @state: base DRM plane state * @format: information about the pixel format used by the plane * @alpha: value of the plane alpha property + * @zpos: value of the plane zpos property */ struct rcar_du_vsp_plane_state { struct drm_plane_state state; @@ -51,6 +52,7 @@ struct rcar_du_vsp_plane_state { const struct rcar_du_format_info *format; unsigned int alpha; + unsigned int zpos; }; static inline struct rcar_du_vsp_plane_state * diff --git a/drivers/gpu/drm/sun4i/Kconfig b/drivers/gpu/drm/sun4i/Kconfig index 99510e64e91a..a4b357db8856 100644 --- a/drivers/gpu/drm/sun4i/Kconfig +++ b/drivers/gpu/drm/sun4i/Kconfig @@ -1,6 +1,6 @@ config DRM_SUN4I tristate "DRM Support for Allwinner A10 Display Engine" - depends on DRM && ARM + depends on DRM && ARM && COMMON_CLK depends on ARCH_SUNXI || COMPILE_TEST select DRM_GEM_CMA_HELPER select DRM_KMS_HELPER diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c index f7a15c1a93bf..3ab560450a82 100644 --- a/drivers/gpu/drm/sun4i/sun4i_backend.c +++ b/drivers/gpu/drm/sun4i/sun4i_backend.c @@ -190,7 +190,7 @@ int sun4i_backend_update_layer_buffer(struct sun4i_backend *backend, /* Get the physical address of the buffer in memory */ gem = drm_fb_cma_get_gem_obj(fb, 0); - DRM_DEBUG_DRIVER("Using GEM @ 0x%x\n", gem->paddr); + DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->paddr); /* Compute the start of the displayed memory */ bpp = drm_format_plane_cpp(fb->pixel_format, 0); @@ -198,7 +198,7 @@ int sun4i_backend_update_layer_buffer(struct sun4i_backend *backend, paddr += (state->src_x >> 16) * bpp; paddr += (state->src_y >> 16) * fb->pitches[0]; - DRM_DEBUG_DRIVER("Setting buffer address to 0x%x\n", paddr); + DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &paddr); /* Write the 32 lower bits of the address (in bits) */ lo_paddr = paddr << 3; diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c index 4182a21f5923..41cacecbea9a 100644 --- a/drivers/gpu/drm/sun4i/sun4i_crtc.c +++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c @@ -65,6 +65,14 @@ static void sun4i_crtc_disable(struct drm_crtc *crtc) DRM_DEBUG_DRIVER("Disabling the CRTC\n"); sun4i_tcon_disable(drv->tcon); + + if (crtc->state->event && !crtc->state->active) { + spin_lock_irq(&crtc->dev->event_lock); + drm_crtc_send_vblank_event(crtc, crtc->state->event); + spin_unlock_irq(&crtc->dev->event_lock); + + crtc->state->event = NULL; + } } static void sun4i_crtc_enable(struct drm_crtc *crtc) diff --git a/drivers/gpu/drm/sun4i/sun4i_dotclock.c b/drivers/gpu/drm/sun4i/sun4i_dotclock.c index 3ff668cb463c..5b3463197c48 100644 --- a/drivers/gpu/drm/sun4i/sun4i_dotclock.c +++ b/drivers/gpu/drm/sun4i/sun4i_dotclock.c @@ -72,14 +72,40 @@ static unsigned long sun4i_dclk_recalc_rate(struct clk_hw *hw, static long sun4i_dclk_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *parent_rate) { - return *parent_rate / DIV_ROUND_CLOSEST(*parent_rate, rate); + unsigned long best_parent = 0; + u8 best_div = 1; + int i; + + for (i = 6; i < 127; i++) { + unsigned long ideal = rate * i; + unsigned long rounded; + + rounded = clk_hw_round_rate(clk_hw_get_parent(hw), + ideal); + + if (rounded == ideal) { + best_parent = rounded; + best_div = i; + goto out; + } + + if ((rounded < ideal) && (rounded > best_parent)) { + best_parent = rounded; + best_div = i; + } + } + +out: + *parent_rate = best_parent; + + return best_parent / best_div; } static int sun4i_dclk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { struct sun4i_dclk *dclk = hw_to_dclk(hw); - int div = DIV_ROUND_CLOSEST(parent_rate, rate); + u8 div = parent_rate / rate; return regmap_update_bits(dclk->regmap, SUN4I_TCON0_DCLK_REG, GENMASK(6, 0), div); @@ -127,10 +153,14 @@ int sun4i_dclk_create(struct device *dev, struct sun4i_tcon *tcon) const char *clk_name, *parent_name; struct clk_init_data init; struct sun4i_dclk *dclk; + int ret; parent_name = __clk_get_name(tcon->sclk0); - of_property_read_string_index(dev->of_node, "clock-output-names", 0, - &clk_name); + ret = of_property_read_string_index(dev->of_node, + "clock-output-names", 0, + &clk_name); + if (ret) + return ret; dclk = devm_kzalloc(dev, sizeof(*dclk), GFP_KERNEL); if (!dclk) @@ -140,6 +170,7 @@ int sun4i_dclk_create(struct device *dev, struct sun4i_tcon *tcon) init.ops = &sun4i_dclk_ops; init.parent_names = &parent_name; init.num_parents = 1; + init.flags = CLK_SET_RATE_PARENT; dclk->regmap = tcon->regs; dclk->hw.init = &init; diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c index 76e922bb60e5..937394cbc241 100644 --- a/drivers/gpu/drm/sun4i/sun4i_drv.c +++ b/drivers/gpu/drm/sun4i/sun4i_drv.c @@ -24,34 +24,6 @@ #include "sun4i_layer.h" #include "sun4i_tcon.h" -static int sun4i_drv_connector_plug_all(struct drm_device *drm) -{ - struct drm_connector *connector, *failed; - int ret; - - mutex_lock(&drm->mode_config.mutex); - list_for_each_entry(connector, &drm->mode_config.connector_list, head) { - ret = drm_connector_register(connector); - if (ret) { - failed = connector; - goto err; - } - } - mutex_unlock(&drm->mode_config.mutex); - return 0; - -err: - list_for_each_entry(connector, &drm->mode_config.connector_list, head) { - if (failed == connector) - break; - - drm_connector_unregister(connector); - } - mutex_unlock(&drm->mode_config.mutex); - - return ret; -} - static int sun4i_drv_enable_vblank(struct drm_device *drm, unsigned int pipe) { struct sun4i_drv *drv = drm->dev_private; @@ -120,11 +92,27 @@ static struct drm_driver sun4i_drv_driver = { /* Frame Buffer Operations */ /* VBlank Operations */ - .get_vblank_counter = drm_vblank_count, + .get_vblank_counter = drm_vblank_no_hw_counter, .enable_vblank = sun4i_drv_enable_vblank, .disable_vblank = sun4i_drv_disable_vblank, }; +static void sun4i_remove_framebuffers(void) +{ + struct apertures_struct *ap; + + ap = alloc_apertures(1); + if (!ap) + return; + + /* The framebuffer can be located anywhere in RAM */ + ap->ranges[0].base = 0; + ap->ranges[0].size = ~0; + + remove_conflicting_framebuffers(ap, "sun4i-drm-fb", false); + kfree(ap); +} + static int sun4i_drv_bind(struct device *dev) { struct drm_device *drm; @@ -172,6 +160,9 @@ static int sun4i_drv_bind(struct device *dev) } drm->irq_enabled = true; + /* Remove early framebuffers (ie. simplefb) */ + sun4i_remove_framebuffers(); + /* Create our framebuffer */ drv->fbdev = sun4i_framebuffer_init(drm); if (IS_ERR(drv->fbdev)) { @@ -187,7 +178,7 @@ static int sun4i_drv_bind(struct device *dev) if (ret) goto free_drm; - ret = sun4i_drv_connector_plug_all(drm); + ret = drm_connector_register_all(drm); if (ret) goto unregister_drm; @@ -204,6 +195,7 @@ static void sun4i_drv_unbind(struct device *dev) { struct drm_device *drm = dev_get_drvdata(dev); + drm_connector_unregister_all(drm); drm_dev_unregister(drm); drm_kms_helper_poll_fini(drm); sun4i_framebuffer_free(drm); @@ -318,6 +310,7 @@ static int sun4i_drv_probe(struct platform_device *pdev) count += sun4i_drv_add_endpoints(&pdev->dev, &match, pipeline); + of_node_put(pipeline); DRM_DEBUG_DRIVER("Queued %d outputs on pipeline %d\n", count, i); diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c index ab6494818050..aaffe9e64ffb 100644 --- a/drivers/gpu/drm/sun4i/sun4i_rgb.c +++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c @@ -54,8 +54,13 @@ static int sun4i_rgb_get_modes(struct drm_connector *connector) static int sun4i_rgb_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { + struct sun4i_rgb *rgb = drm_connector_to_sun4i_rgb(connector); + struct sun4i_drv *drv = rgb->drv; + struct sun4i_tcon *tcon = drv->tcon; u32 hsync = mode->hsync_end - mode->hsync_start; u32 vsync = mode->vsync_end - mode->vsync_start; + unsigned long rate = mode->clock * 1000; + long rounded_rate; DRM_DEBUG_DRIVER("Validating modes...\n"); @@ -87,6 +92,15 @@ static int sun4i_rgb_mode_valid(struct drm_connector *connector, DRM_DEBUG_DRIVER("Vertical parameters OK\n"); + rounded_rate = clk_round_rate(tcon->dclk, rate); + if (rounded_rate < rate) + return MODE_CLOCK_LOW; + + if (rounded_rate > rate) + return MODE_CLOCK_HIGH; + + DRM_DEBUG_DRIVER("Clock rate OK\n"); + return MODE_OK; } @@ -203,7 +217,7 @@ int sun4i_rgb_init(struct drm_device *drm) int ret; /* If we don't have a panel, there's no point in going on */ - if (!tcon->panel) + if (IS_ERR(tcon->panel)) return -ENODEV; rgb = devm_kzalloc(drm->dev, sizeof(*rgb), GFP_KERNEL); diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c index 9f19b0e08560..652385f09735 100644 --- a/drivers/gpu/drm/sun4i/sun4i_tcon.c +++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c @@ -425,11 +425,11 @@ static struct drm_panel *sun4i_tcon_find_panel(struct device_node *node) remote = of_graph_get_remote_port_parent(end_node); if (!remote) { - DRM_DEBUG_DRIVER("Enable to parse remote node\n"); + DRM_DEBUG_DRIVER("Unable to parse remote node\n"); return ERR_PTR(-EINVAL); } - return of_drm_find_panel(remote); + return of_drm_find_panel(remote) ?: ERR_PTR(-EPROBE_DEFER); } static int sun4i_tcon_bind(struct device *dev, struct device *master, @@ -490,7 +490,11 @@ static int sun4i_tcon_bind(struct device *dev, struct device *master, return 0; } - return sun4i_rgb_init(drm); + ret = sun4i_rgb_init(drm); + if (ret < 0) + goto err_free_clocks; + + return 0; err_free_clocks: sun4i_tcon_free_clocks(tcon); @@ -522,12 +526,13 @@ static int sun4i_tcon_probe(struct platform_device *pdev) * Defer the probe. */ panel = sun4i_tcon_find_panel(node); - if (IS_ERR(panel)) { - /* - * If we don't have a panel endpoint, just go on - */ - if (PTR_ERR(panel) != -ENODEV) - return -EPROBE_DEFER; + + /* + * If we don't have a panel endpoint, just go on + */ + if (PTR_ERR(panel) == -EPROBE_DEFER) { + DRM_DEBUG_DRIVER("Still waiting for our panel. Deferring...\n"); + return -EPROBE_DEFER; } return component_add(&pdev->dev, &sun4i_tcon_ops); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 39386f50af87..a71cf98c655f 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1034,9 +1034,9 @@ out_unlock: return ret; } -static bool ttm_bo_mem_compat(struct ttm_placement *placement, - struct ttm_mem_reg *mem, - uint32_t *new_flags) +bool ttm_bo_mem_compat(struct ttm_placement *placement, + struct ttm_mem_reg *mem, + uint32_t *new_flags) { int i; @@ -1068,6 +1068,7 @@ static bool ttm_bo_mem_compat(struct ttm_placement *placement, return false; } +EXPORT_SYMBOL(ttm_bo_mem_compat); int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c index 9b078a493996..0cd889015dc5 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c @@ -49,6 +49,7 @@ int vmw_dmabuf_pin_in_placement(struct vmw_private *dev_priv, { struct ttm_buffer_object *bo = &buf->base; int ret; + uint32_t new_flags; ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible); if (unlikely(ret != 0)) @@ -60,7 +61,12 @@ int vmw_dmabuf_pin_in_placement(struct vmw_private *dev_priv, if (unlikely(ret != 0)) goto err; - ret = ttm_bo_validate(bo, placement, interruptible, false); + if (buf->pin_count > 0) + ret = ttm_bo_mem_compat(placement, &bo->mem, + &new_flags) == true ? 0 : -EINVAL; + else + ret = ttm_bo_validate(bo, placement, interruptible, false); + if (!ret) vmw_bo_pin_reserved(buf, true); @@ -91,6 +97,7 @@ int vmw_dmabuf_pin_in_vram_or_gmr(struct vmw_private *dev_priv, { struct ttm_buffer_object *bo = &buf->base; int ret; + uint32_t new_flags; ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible); if (unlikely(ret != 0)) @@ -102,6 +109,12 @@ int vmw_dmabuf_pin_in_vram_or_gmr(struct vmw_private *dev_priv, if (unlikely(ret != 0)) goto err; + if (buf->pin_count > 0) { + ret = ttm_bo_mem_compat(&vmw_vram_gmr_placement, &bo->mem, + &new_flags) == true ? 0 : -EINVAL; + goto out_unreserve; + } + ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, interruptible, false); if (likely(ret == 0) || ret == -ERESTARTSYS) @@ -161,6 +174,7 @@ int vmw_dmabuf_pin_in_start_of_vram(struct vmw_private *dev_priv, struct ttm_placement placement; struct ttm_place place; int ret = 0; + uint32_t new_flags; place = vmw_vram_placement.placement[0]; place.lpfn = bo->num_pages; @@ -185,10 +199,15 @@ int vmw_dmabuf_pin_in_start_of_vram(struct vmw_private *dev_priv, */ if (bo->mem.mem_type == TTM_PL_VRAM && bo->mem.start < bo->num_pages && - bo->mem.start > 0) + bo->mem.start > 0 && + buf->pin_count == 0) (void) ttm_bo_validate(bo, &vmw_sys_placement, false, false); - ret = ttm_bo_validate(bo, &placement, interruptible, false); + if (buf->pin_count > 0) + ret = ttm_bo_mem_compat(&placement, &bo->mem, + &new_flags) == true ? 0 : -EINVAL; + else + ret = ttm_bo_validate(bo, &placement, interruptible, false); /* For some reason we didn't end up at the start of vram */ WARN_ON(ret == 0 && bo->offset != 0); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index 9fcd8200d485..8d528fcf6e96 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -233,6 +233,7 @@ static int vmw_force_iommu; static int vmw_restrict_iommu; static int vmw_force_coherent; static int vmw_restrict_dma_mask; +static int vmw_assume_16bpp; static int vmw_probe(struct pci_dev *, const struct pci_device_id *); static void vmw_master_init(struct vmw_master *); @@ -249,6 +250,8 @@ MODULE_PARM_DESC(force_coherent, "Force coherent TTM pages"); module_param_named(force_coherent, vmw_force_coherent, int, 0600); MODULE_PARM_DESC(restrict_dma_mask, "Restrict DMA mask to 44 bits with IOMMU"); module_param_named(restrict_dma_mask, vmw_restrict_dma_mask, int, 0600); +MODULE_PARM_DESC(assume_16bpp, "Assume 16-bpp when filtering modes"); +module_param_named(assume_16bpp, vmw_assume_16bpp, int, 0600); static void vmw_print_capabilities(uint32_t capabilities) @@ -660,6 +663,8 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset) dev_priv->vram_start = pci_resource_start(dev->pdev, 1); dev_priv->mmio_start = pci_resource_start(dev->pdev, 2); + dev_priv->assume_16bpp = !!vmw_assume_16bpp; + dev_priv->enable_fb = enable_fbdev; vmw_write(dev_priv, SVGA_REG_ID, SVGA_ID_2); @@ -706,6 +711,13 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset) vmw_read(dev_priv, SVGA_REG_SUGGESTED_GBOBJECT_MEM_SIZE_KB); + /* + * Workaround for low memory 2D VMs to compensate for the + * allocation taken by fbdev + */ + if (!(dev_priv->capabilities & SVGA_CAP_3D)) + mem_size *= 2; + dev_priv->max_mob_pages = mem_size * 1024 / PAGE_SIZE; dev_priv->prim_bb_mem = vmw_read(dev_priv, diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h index 1980e2a28265..89fb19443a3f 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h @@ -386,6 +386,7 @@ struct vmw_private { spinlock_t hw_lock; spinlock_t cap_lock; bool has_dx; + bool assume_16bpp; /* * VGA registers. diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c index 679a4cb98ee3..d2d93959b119 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c @@ -517,28 +517,6 @@ static int vmw_fb_kms_framebuffer(struct fb_info *info) par->set_fb = &vfb->base; - if (!par->bo_ptr) { - /* - * Pin before mapping. Since we don't know in what placement - * to pin, call into KMS to do it for us. - */ - ret = vfb->pin(vfb); - if (ret) { - DRM_ERROR("Could not pin the fbdev framebuffer.\n"); - return ret; - } - - ret = ttm_bo_kmap(&par->vmw_bo->base, 0, - par->vmw_bo->base.num_pages, &par->map); - if (ret) { - vfb->unpin(vfb); - DRM_ERROR("Could not map the fbdev framebuffer.\n"); - return ret; - } - - par->bo_ptr = ttm_kmap_obj_virtual(&par->map, &par->bo_iowrite); - } - return 0; } @@ -601,6 +579,31 @@ static int vmw_fb_set_par(struct fb_info *info) if (ret) goto out_unlock; + if (!par->bo_ptr) { + struct vmw_framebuffer *vfb = vmw_framebuffer_to_vfb(set.fb); + + /* + * Pin before mapping. Since we don't know in what placement + * to pin, call into KMS to do it for us. + */ + ret = vfb->pin(vfb); + if (ret) { + DRM_ERROR("Could not pin the fbdev framebuffer.\n"); + goto out_unlock; + } + + ret = ttm_bo_kmap(&par->vmw_bo->base, 0, + par->vmw_bo->base.num_pages, &par->map); + if (ret) { + vfb->unpin(vfb); + DRM_ERROR("Could not map the fbdev framebuffer.\n"); + goto out_unlock; + } + + par->bo_ptr = ttm_kmap_obj_virtual(&par->map, &par->bo_iowrite); + } + + vmw_fb_dirty_mark(par, par->fb_x, par->fb_y, par->set_fb->width, par->set_fb->height); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c index 55231cce73a0..e29da45a2847 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c @@ -1553,14 +1553,10 @@ int vmw_du_connector_fill_modes(struct drm_connector *connector, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) }; int i; - u32 assumed_bpp = 2; + u32 assumed_bpp = 4; - /* - * If using screen objects, then assume 32-bpp because that's what the - * SVGA device is assuming - */ - if (dev_priv->active_display_unit == vmw_du_screen_object) - assumed_bpp = 4; + if (dev_priv->assume_16bpp) + assumed_bpp = 2; if (dev_priv->active_display_unit == vmw_du_screen_target) { max_width = min(max_width, dev_priv->stdu_max_width); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c index f0374f9b56ca..e57a0bad7a62 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c @@ -300,6 +300,9 @@ static int vmw_recv_msg(struct rpc_channel *channel, void **msg, break; } + if (retries == RETRIES) + return -EINVAL; + *msg_len = reply_len; *msg = reply; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c index 9ca818fb034c..41932a7c4f79 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c @@ -399,8 +399,10 @@ static int vmw_stdu_bind_fb(struct vmw_private *dev_priv, WARN_ON_ONCE(!stdu->defined); - if (!vfb->dmabuf && new_fb->width == mode->hdisplay && - new_fb->height == mode->vdisplay) + new_vfbs = (vfb->dmabuf) ? NULL : vmw_framebuffer_to_vfbs(new_fb); + + if (new_vfbs && new_vfbs->surface->base_size.width == mode->hdisplay && + new_vfbs->surface->base_size.height == mode->vdisplay) new_content_type = SAME_AS_DISPLAY; else if (vfb->dmabuf) new_content_type = SEPARATE_DMA; @@ -444,7 +446,6 @@ static int vmw_stdu_bind_fb(struct vmw_private *dev_priv, content_srf.mip_levels[0] = 1; content_srf.multisample_count = 0; } else { - new_vfbs = vmw_framebuffer_to_vfbs(new_fb); content_srf = *new_vfbs->surface; } @@ -464,7 +465,6 @@ static int vmw_stdu_bind_fb(struct vmw_private *dev_priv, return ret; } } else if (new_content_type == SAME_AS_DISPLAY) { - new_vfbs = vmw_framebuffer_to_vfbs(new_fb); new_display_srf = vmw_surface_reference(new_vfbs->surface); } diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index 95b7d61d9910..fb6f1f447279 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -61,6 +61,7 @@ MODULE_LICENSE("GPL"); #define MT_QUIRK_ALWAYS_VALID (1 << 4) #define MT_QUIRK_VALID_IS_INRANGE (1 << 5) #define MT_QUIRK_VALID_IS_CONFIDENCE (1 << 6) +#define MT_QUIRK_CONFIDENCE (1 << 7) #define MT_QUIRK_SLOT_IS_CONTACTID_MINUS_ONE (1 << 8) #define MT_QUIRK_NO_AREA (1 << 9) #define MT_QUIRK_IGNORE_DUPLICATES (1 << 10) @@ -78,6 +79,7 @@ struct mt_slot { __s32 contactid; /* the device ContactID assigned to this slot */ bool touch_state; /* is the touch valid? */ bool inrange_state; /* is the finger in proximity of the sensor? */ + bool confidence_state; /* is the touch made by a finger? */ }; struct mt_class { @@ -503,10 +505,8 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, return 1; case HID_DG_CONFIDENCE: if (cls->name == MT_CLS_WIN_8 && - field->application == HID_DG_TOUCHPAD) { - cls->quirks &= ~MT_QUIRK_ALWAYS_VALID; - cls->quirks |= MT_QUIRK_VALID_IS_CONFIDENCE; - } + field->application == HID_DG_TOUCHPAD) + cls->quirks |= MT_QUIRK_CONFIDENCE; mt_store_field(usage, td, hi); return 1; case HID_DG_TIPSWITCH: @@ -619,6 +619,7 @@ static void mt_complete_slot(struct mt_device *td, struct input_dev *input) return; if (td->curvalid || (td->mtclass.quirks & MT_QUIRK_ALWAYS_VALID)) { + int active; int slotnum = mt_compute_slot(td, input); struct mt_slot *s = &td->curdata; struct input_mt *mt = input->mt; @@ -633,10 +634,14 @@ static void mt_complete_slot(struct mt_device *td, struct input_dev *input) return; } + if (!(td->mtclass.quirks & MT_QUIRK_CONFIDENCE)) + s->confidence_state = 1; + active = (s->touch_state || s->inrange_state) && + s->confidence_state; + input_mt_slot(input, slotnum); - input_mt_report_slot_state(input, MT_TOOL_FINGER, - s->touch_state || s->inrange_state); - if (s->touch_state || s->inrange_state) { + input_mt_report_slot_state(input, MT_TOOL_FINGER, active); + if (active) { /* this finger is in proximity of the sensor */ int wide = (s->w > s->h); /* divided by two to match visual scale of touch */ @@ -701,6 +706,8 @@ static void mt_process_mt_event(struct hid_device *hid, struct hid_field *field, td->curdata.touch_state = value; break; case HID_DG_CONFIDENCE: + if (quirks & MT_QUIRK_CONFIDENCE) + td->curdata.confidence_state = value; if (quirks & MT_QUIRK_VALID_IS_CONFIDENCE) td->curvalid = value; break; diff --git a/drivers/hid/usbhid/hiddev.c b/drivers/hid/usbhid/hiddev.c index 2f1ddca6f2e0..700145b15088 100644 --- a/drivers/hid/usbhid/hiddev.c +++ b/drivers/hid/usbhid/hiddev.c @@ -516,13 +516,13 @@ static noinline int hiddev_ioctl_usage(struct hiddev *hiddev, unsigned int cmd, goto inval; } else if (uref->usage_index >= field->report_count) goto inval; - - else if ((cmd == HIDIOCGUSAGES || cmd == HIDIOCSUSAGES) && - (uref_multi->num_values > HID_MAX_MULTI_USAGES || - uref->usage_index + uref_multi->num_values > field->report_count)) - goto inval; } + if ((cmd == HIDIOCGUSAGES || cmd == HIDIOCSUSAGES) && + (uref_multi->num_values > HID_MAX_MULTI_USAGES || + uref->usage_index + uref_multi->num_values > field->report_count)) + goto inval; + switch (cmd) { case HIDIOCGUSAGE: uref->value = field->value[uref->usage_index]; diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index ff940075bb90..eaf2f916d48c 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -486,6 +486,18 @@ config SENSORS_FSCHMD This driver can also be built as a module. If so, the module will be called fschmd. +config SENSORS_FTSTEUTATES + tristate "Fujitsu Technology Solutions sensor chip Teutates" + depends on I2C && WATCHDOG + select WATCHDOG_CORE + help + If you say yes here you get support for the Fujitsu Technology + Solutions (FTS) sensor chip "Teutates" including support for + the integrated watchdog. + + This driver can also be built as a module. If so, the module + will be called ftsteutates. + config SENSORS_GL518SM tristate "Genesys Logic GL518SM" depends on I2C @@ -645,8 +657,8 @@ config SENSORS_JC42 temperature sensors, which are used on many DDR3 memory modules for mobile devices and servers. Support will include, but not be limited to, ADT7408, AT30TS00, CAT34TS02, CAT6095, MAX6604, MCP9804, MCP9805, - MCP98242, MCP98243, MCP98244, MCP9843, SE97, SE98, STTS424(E), - STTS2002, STTS3000, TSE2002, TSE2004, TS3000, and TS3001. + MCP9808, MCP98242, MCP98243, MCP98244, MCP9843, SE97, SE98, + STTS424(E), STTS2002, STTS3000, TSE2002, TSE2004, TS3000, and TS3001. This driver can also be built as a module. If so, the module will be called jc42. @@ -958,6 +970,7 @@ config SENSORS_LM75 tristate "National Semiconductor LM75 and compatibles" depends on I2C depends on THERMAL || !THERMAL_OF + select REGMAP_I2C help If you say yes here you get support for one common type of temperature sensor chip, with models including: @@ -1265,6 +1278,17 @@ config SENSORS_SHT21 This driver can also be built as a module. If so, the module will be called sht21. +config SENSORS_SHT3x + tristate "Sensiron humidity and temperature sensors. SHT3x and compat." + depends on I2C + select CRC8 + help + If you say yes here you get support for the Sensiron SHT30 and SHT31 + humidity and temperature sensors. + + This driver can also be built as a module. If so, the module + will be called sht3x. + config SENSORS_SHTC1 tristate "Sensiron humidity and temperature sensors. SHTC1 and compat." depends on I2C @@ -1514,6 +1538,17 @@ config SENSORS_INA2XX This driver can also be built as a module. If so, the module will be called ina2xx. +config SENSORS_INA3221 + tristate "Texas Instruments INA3221 Triple Power Monitor" + depends on I2C + select REGMAP_I2C + help + If you say yes here you get support for the TI INA3221 Triple Power + Monitor. + + This driver can also be built as a module. If so, the module + will be called ina3221. + config SENSORS_TC74 tristate "Microchip TC74" depends on I2C @@ -1538,6 +1573,7 @@ config SENSORS_TMP102 tristate "Texas Instruments TMP102" depends on I2C depends on THERMAL || !THERMAL_OF + select REGMAP_I2C help If you say yes here you get support for Texas Instruments TMP102 sensor chips. @@ -1561,7 +1597,7 @@ config SENSORS_TMP401 depends on I2C help If you say yes here you get support for Texas Instruments TMP401, - TMP411, TMP431, TMP432 and TMP435 temperature sensor chips. + TMP411, TMP431, TMP432, TMP435, and TMP461 temperature sensor chips. This driver can also be built as a module. If so, the module will be called tmp401. diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index 2ef5b7c4c54f..fe87d2895a97 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -62,6 +62,7 @@ obj-$(CONFIG_SENSORS_F71882FG) += f71882fg.o obj-$(CONFIG_SENSORS_F75375S) += f75375s.o obj-$(CONFIG_SENSORS_FAM15H_POWER) += fam15h_power.o obj-$(CONFIG_SENSORS_FSCHMD) += fschmd.o +obj-$(CONFIG_SENSORS_FTSTEUTATES) += ftsteutates.o obj-$(CONFIG_SENSORS_G760A) += g760a.o obj-$(CONFIG_SENSORS_G762) += g762.o obj-$(CONFIG_SENSORS_GL518SM) += gl518sm.o @@ -77,6 +78,7 @@ obj-$(CONFIG_SENSORS_IBMPOWERNV)+= ibmpowernv.o obj-$(CONFIG_SENSORS_IIO_HWMON) += iio_hwmon.o obj-$(CONFIG_SENSORS_INA209) += ina209.o obj-$(CONFIG_SENSORS_INA2XX) += ina2xx.o +obj-$(CONFIG_SENSORS_INA3221) += ina3221.o obj-$(CONFIG_SENSORS_IT87) += it87.o obj-$(CONFIG_SENSORS_JC42) += jc42.o obj-$(CONFIG_SENSORS_JZ4740) += jz4740-hwmon.o @@ -138,6 +140,7 @@ obj-$(CONFIG_SENSORS_SCH5627) += sch5627.o obj-$(CONFIG_SENSORS_SCH5636) += sch5636.o obj-$(CONFIG_SENSORS_SHT15) += sht15.o obj-$(CONFIG_SENSORS_SHT21) += sht21.o +obj-$(CONFIG_SENSORS_SHT3x) += sht3x.o obj-$(CONFIG_SENSORS_SHTC1) += shtc1.o obj-$(CONFIG_SENSORS_SIS5595) += sis5595.o obj-$(CONFIG_SENSORS_SMM665) += smm665.o diff --git a/drivers/hwmon/ad7314.c b/drivers/hwmon/ad7314.c index 202c1fbb3407..8ea35932fbaa 100644 --- a/drivers/hwmon/ad7314.c +++ b/drivers/hwmon/ad7314.c @@ -37,7 +37,6 @@ enum ad7314_variant { struct ad7314_data { struct spi_device *spi_dev; - struct device *hwmon_dev; u16 rx ____cacheline_aligned; }; @@ -88,62 +87,30 @@ static ssize_t ad7314_show_temperature(struct device *dev, } } -static ssize_t ad7314_show_name(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - return sprintf(buf, "%s\n", to_spi_device(dev)->modalias); -} - -static DEVICE_ATTR(name, S_IRUGO, ad7314_show_name, NULL); static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, ad7314_show_temperature, NULL, 0); -static struct attribute *ad7314_attributes[] = { - &dev_attr_name.attr, +static struct attribute *ad7314_attrs[] = { &sensor_dev_attr_temp1_input.dev_attr.attr, NULL, }; -static const struct attribute_group ad7314_group = { - .attrs = ad7314_attributes, -}; +ATTRIBUTE_GROUPS(ad7314); static int ad7314_probe(struct spi_device *spi_dev) { - int ret; struct ad7314_data *chip; + struct device *hwmon_dev; chip = devm_kzalloc(&spi_dev->dev, sizeof(*chip), GFP_KERNEL); if (chip == NULL) return -ENOMEM; - spi_set_drvdata(spi_dev, chip); - - ret = sysfs_create_group(&spi_dev->dev.kobj, &ad7314_group); - if (ret < 0) - return ret; - - chip->hwmon_dev = hwmon_device_register(&spi_dev->dev); - if (IS_ERR(chip->hwmon_dev)) { - ret = PTR_ERR(chip->hwmon_dev); - goto error_remove_group; - } chip->spi_dev = spi_dev; - - return 0; -error_remove_group: - sysfs_remove_group(&spi_dev->dev.kobj, &ad7314_group); - return ret; -} - -static int ad7314_remove(struct spi_device *spi_dev) -{ - struct ad7314_data *chip = spi_get_drvdata(spi_dev); - - hwmon_device_unregister(chip->hwmon_dev); - sysfs_remove_group(&spi_dev->dev.kobj, &ad7314_group); - - return 0; + hwmon_dev = devm_hwmon_device_register_with_groups(&spi_dev->dev, + spi_dev->modalias, + chip, ad7314_groups); + return PTR_ERR_OR_ZERO(hwmon_dev); } static const struct spi_device_id ad7314_id[] = { @@ -159,7 +126,6 @@ static struct spi_driver ad7314_driver = { .name = "ad7314", }, .probe = ad7314_probe, - .remove = ad7314_remove, .id_table = ad7314_id, }; diff --git a/drivers/hwmon/ads7871.c b/drivers/hwmon/ads7871.c index 4fd9e4de1972..59bd7b9e1772 100644 --- a/drivers/hwmon/ads7871.c +++ b/drivers/hwmon/ads7871.c @@ -66,14 +66,12 @@ #include #include #include -#include #include #define DEVICE_NAME "ads7871" struct ads7871_data { - struct device *hwmon_dev; - struct mutex update_lock; + struct spi_device *spi; }; static int ads7871_read_reg8(struct spi_device *spi, int reg) @@ -101,7 +99,8 @@ static int ads7871_write_reg8(struct spi_device *spi, int reg, u8 val) static ssize_t show_voltage(struct device *dev, struct device_attribute *da, char *buf) { - struct spi_device *spi = to_spi_device(dev); + struct ads7871_data *pdata = dev_get_drvdata(dev); + struct spi_device *spi = pdata->spi; struct sensor_device_attribute *attr = to_sensor_dev_attr(da); int ret, val, i = 0; uint8_t channel, mux_cnv; @@ -139,12 +138,6 @@ static ssize_t show_voltage(struct device *dev, } } -static ssize_t ads7871_show_name(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - return sprintf(buf, "%s\n", to_spi_device(dev)->modalias); -} - static SENSOR_DEVICE_ATTR(in0_input, S_IRUGO, show_voltage, NULL, 0); static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, show_voltage, NULL, 1); static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, show_voltage, NULL, 2); @@ -154,9 +147,7 @@ static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, show_voltage, NULL, 5); static SENSOR_DEVICE_ATTR(in6_input, S_IRUGO, show_voltage, NULL, 6); static SENSOR_DEVICE_ATTR(in7_input, S_IRUGO, show_voltage, NULL, 7); -static DEVICE_ATTR(name, S_IRUGO, ads7871_show_name, NULL); - -static struct attribute *ads7871_attributes[] = { +static struct attribute *ads7871_attrs[] = { &sensor_dev_attr_in0_input.dev_attr.attr, &sensor_dev_attr_in1_input.dev_attr.attr, &sensor_dev_attr_in2_input.dev_attr.attr, @@ -165,21 +156,18 @@ static struct attribute *ads7871_attributes[] = { &sensor_dev_attr_in5_input.dev_attr.attr, &sensor_dev_attr_in6_input.dev_attr.attr, &sensor_dev_attr_in7_input.dev_attr.attr, - &dev_attr_name.attr, NULL }; -static const struct attribute_group ads7871_group = { - .attrs = ads7871_attributes, -}; +ATTRIBUTE_GROUPS(ads7871); static int ads7871_probe(struct spi_device *spi) { - int ret, err; + struct device *dev = &spi->dev; + int ret; uint8_t val; struct ads7871_data *pdata; - - dev_dbg(&spi->dev, "probe\n"); + struct device *hwmon_dev; /* Configure the SPI bus */ spi->mode = (SPI_MODE_0); @@ -193,7 +181,7 @@ static int ads7871_probe(struct spi_device *spi) ads7871_write_reg8(spi, REG_OSC_CONTROL, val); ret = ads7871_read_reg8(spi, REG_OSC_CONTROL); - dev_dbg(&spi->dev, "REG_OSC_CONTROL write:%x, read:%x\n", val, ret); + dev_dbg(dev, "REG_OSC_CONTROL write:%x, read:%x\n", val, ret); /* * because there is no other error checking on an SPI bus * we need to make sure we really have a chip @@ -201,46 +189,23 @@ static int ads7871_probe(struct spi_device *spi) if (val != ret) return -ENODEV; - pdata = devm_kzalloc(&spi->dev, sizeof(struct ads7871_data), - GFP_KERNEL); + pdata = devm_kzalloc(dev, sizeof(struct ads7871_data), GFP_KERNEL); if (!pdata) return -ENOMEM; - err = sysfs_create_group(&spi->dev.kobj, &ads7871_group); - if (err < 0) - return err; - - spi_set_drvdata(spi, pdata); + pdata->spi = spi; - pdata->hwmon_dev = hwmon_device_register(&spi->dev); - if (IS_ERR(pdata->hwmon_dev)) { - err = PTR_ERR(pdata->hwmon_dev); - goto error_remove; - } - - return 0; - -error_remove: - sysfs_remove_group(&spi->dev.kobj, &ads7871_group); - return err; -} - -static int ads7871_remove(struct spi_device *spi) -{ - struct ads7871_data *pdata = spi_get_drvdata(spi); - - hwmon_device_unregister(pdata->hwmon_dev); - sysfs_remove_group(&spi->dev.kobj, &ads7871_group); - return 0; + hwmon_dev = devm_hwmon_device_register_with_groups(dev, spi->modalias, + pdata, + ads7871_groups); + return PTR_ERR_OR_ZERO(hwmon_dev); } static struct spi_driver ads7871_driver = { .driver = { .name = DEVICE_NAME, }, - .probe = ads7871_probe, - .remove = ads7871_remove, }; module_spi_driver(ads7871_driver); diff --git a/drivers/hwmon/adt7411.c b/drivers/hwmon/adt7411.c index 827c03703128..a7f886961830 100644 --- a/drivers/hwmon/adt7411.c +++ b/drivers/hwmon/adt7411.c @@ -30,6 +30,7 @@ #define ADT7411_REG_CFG1 0x18 #define ADT7411_CFG1_START_MONITOR (1 << 0) +#define ADT7411_CFG1_RESERVED_BIT3 (1 << 3) #define ADT7411_REG_CFG2 0x19 #define ADT7411_CFG2_DISABLE_AVG (1 << 5) @@ -296,8 +297,10 @@ static int adt7411_probe(struct i2c_client *client, mutex_init(&data->device_lock); mutex_init(&data->update_lock); + /* According to the datasheet, we must only write 1 to bit 3 */ ret = adt7411_modify_bit(client, ADT7411_REG_CFG1, - ADT7411_CFG1_START_MONITOR, 1); + ADT7411_CFG1_RESERVED_BIT3 + | ADT7411_CFG1_START_MONITOR, 1); if (ret < 0) return ret; diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index c43318d3416e..acf9c0361d9f 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -35,6 +35,7 @@ #include #include #include +#include #include @@ -66,11 +67,13 @@ static DEFINE_MUTEX(i8k_mutex); static char bios_version[4]; +static char bios_machineid[16]; static struct device *i8k_hwmon_dev; static u32 i8k_hwmon_flags; static uint i8k_fan_mult = I8K_FAN_MULT; static uint i8k_pwm_mult; static uint i8k_fan_max = I8K_FAN_HIGH; +static bool disallow_fan_type_call; #define I8K_HWMON_HAVE_TEMP1 (1 << 0) #define I8K_HWMON_HAVE_TEMP2 (1 << 1) @@ -78,6 +81,7 @@ static uint i8k_fan_max = I8K_FAN_HIGH; #define I8K_HWMON_HAVE_TEMP4 (1 << 3) #define I8K_HWMON_HAVE_FAN1 (1 << 4) #define I8K_HWMON_HAVE_FAN2 (1 << 5) +#define I8K_HWMON_HAVE_FAN3 (1 << 6) MODULE_AUTHOR("Massimo Dal Zotto (dz@debian.org)"); MODULE_AUTHOR("Pali Rohár "); @@ -94,13 +98,13 @@ module_param(ignore_dmi, bool, 0); MODULE_PARM_DESC(ignore_dmi, "Continue probing hardware even if DMI data does not match"); #if IS_ENABLED(CONFIG_I8K) -static bool restricted; +static bool restricted = true; module_param(restricted, bool, 0); -MODULE_PARM_DESC(restricted, "Allow fan control if SYS_ADMIN capability set"); +MODULE_PARM_DESC(restricted, "Restrict fan control and serial number to CAP_SYS_ADMIN (default: 1)"); static bool power_status; module_param(power_status, bool, 0600); -MODULE_PARM_DESC(power_status, "Report power status in /proc/i8k"); +MODULE_PARM_DESC(power_status, "Report power status in /proc/i8k (default: 0)"); #endif static uint fan_mult; @@ -136,6 +140,14 @@ static int i8k_smm(struct smm_regs *regs) int eax = regs->eax; cpumask_var_t old_mask; +#ifdef DEBUG + int ebx = regs->ebx; + unsigned long duration; + ktime_t calltime, delta, rettime; + + calltime = ktime_get(); +#endif + /* SMM requires CPU 0 */ if (!alloc_cpumask_var(&old_mask, GFP_KERNEL)) return -ENOMEM; @@ -207,6 +219,15 @@ static int i8k_smm(struct smm_regs *regs) out: set_cpus_allowed_ptr(current, old_mask); free_cpumask_var(old_mask); + +#ifdef DEBUG + rettime = ktime_get(); + delta = ktime_sub(rettime, calltime); + duration = ktime_to_ns(delta) >> 10; + pr_debug("smm(0x%.4x 0x%.4x) = 0x%.4x (took %7lu usecs)\n", eax, ebx, + (rc ? 0xffff : regs->eax & 0xffff), duration); +#endif + return rc; } @@ -235,14 +256,28 @@ static int i8k_get_fan_speed(int fan) /* * Read the fan type. */ -static int i8k_get_fan_type(int fan) +static int _i8k_get_fan_type(int fan) { struct smm_regs regs = { .eax = I8K_SMM_GET_FAN_TYPE, }; + if (disallow_fan_type_call) + return -EINVAL; + regs.ebx = fan & 0xff; return i8k_smm(®s) ? : regs.eax & 0xff; } +static int i8k_get_fan_type(int fan) +{ + /* I8K_SMM_GET_FAN_TYPE SMM call is expensive, so cache values */ + static int types[3] = { INT_MIN, INT_MIN, INT_MIN }; + + if (types[fan] == INT_MIN) + types[fan] = _i8k_get_fan_type(fan); + + return types[fan]; +} + /* * Read the fan nominal rpm for specific fan speed. */ @@ -387,14 +422,20 @@ i8k_ioctl_unlocked(struct file *fp, unsigned int cmd, unsigned long arg) switch (cmd) { case I8K_BIOS_VERSION: + if (!isdigit(bios_version[0]) || !isdigit(bios_version[1]) || + !isdigit(bios_version[2])) + return -EINVAL; + val = (bios_version[0] << 16) | (bios_version[1] << 8) | bios_version[2]; break; case I8K_MACHINE_ID: - memset(buff, 0, 16); - strlcpy(buff, i8k_get_dmi_data(DMI_PRODUCT_SERIAL), - sizeof(buff)); + if (restricted && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + memset(buff, 0, sizeof(buff)); + strlcpy(buff, bios_machineid, sizeof(buff)); break; case I8K_FN_STATUS: @@ -511,7 +552,7 @@ static int i8k_proc_show(struct seq_file *seq, void *offset) seq_printf(seq, "%s %s %s %d %d %d %d %d %d %d\n", I8K_PROC_FMT, bios_version, - i8k_get_dmi_data(DMI_PRODUCT_SERIAL), + (restricted && !capable(CAP_SYS_ADMIN)) ? "-1" : bios_machineid, cpu_temp, left_fan, right_fan, left_speed, right_speed, ac_power, fn_key); @@ -696,6 +737,12 @@ static SENSOR_DEVICE_ATTR(fan2_label, S_IRUGO, i8k_hwmon_show_fan_label, NULL, 1); static SENSOR_DEVICE_ATTR(pwm2, S_IRUGO | S_IWUSR, i8k_hwmon_show_pwm, i8k_hwmon_set_pwm, 1); +static SENSOR_DEVICE_ATTR(fan3_input, S_IRUGO, i8k_hwmon_show_fan, NULL, + 2); +static SENSOR_DEVICE_ATTR(fan3_label, S_IRUGO, i8k_hwmon_show_fan_label, NULL, + 2); +static SENSOR_DEVICE_ATTR(pwm3, S_IRUGO | S_IWUSR, i8k_hwmon_show_pwm, + i8k_hwmon_set_pwm, 2); static struct attribute *i8k_attrs[] = { &sensor_dev_attr_temp1_input.dev_attr.attr, /* 0 */ @@ -712,12 +759,18 @@ static struct attribute *i8k_attrs[] = { &sensor_dev_attr_fan2_input.dev_attr.attr, /* 11 */ &sensor_dev_attr_fan2_label.dev_attr.attr, /* 12 */ &sensor_dev_attr_pwm2.dev_attr.attr, /* 13 */ + &sensor_dev_attr_fan3_input.dev_attr.attr, /* 14 */ + &sensor_dev_attr_fan3_label.dev_attr.attr, /* 15 */ + &sensor_dev_attr_pwm3.dev_attr.attr, /* 16 */ NULL }; static umode_t i8k_is_visible(struct kobject *kobj, struct attribute *attr, int index) { + if (disallow_fan_type_call && + (index == 9 || index == 12 || index == 15)) + return 0; if (index >= 0 && index <= 1 && !(i8k_hwmon_flags & I8K_HWMON_HAVE_TEMP1)) return 0; @@ -736,6 +789,9 @@ static umode_t i8k_is_visible(struct kobject *kobj, struct attribute *attr, if (index >= 11 && index <= 13 && !(i8k_hwmon_flags & I8K_HWMON_HAVE_FAN2)) return 0; + if (index >= 14 && index <= 16 && + !(i8k_hwmon_flags & I8K_HWMON_HAVE_FAN3)) + return 0; return attr->mode; } @@ -767,16 +823,27 @@ static int __init i8k_init_hwmon(void) if (err >= 0) i8k_hwmon_flags |= I8K_HWMON_HAVE_TEMP4; - /* First fan attributes, if fan type is OK */ - err = i8k_get_fan_type(0); + /* First fan attributes, if fan status or type is OK */ + err = i8k_get_fan_status(0); + if (err < 0) + err = i8k_get_fan_type(0); if (err >= 0) i8k_hwmon_flags |= I8K_HWMON_HAVE_FAN1; - /* Second fan attributes, if fan type is OK */ - err = i8k_get_fan_type(1); + /* Second fan attributes, if fan status or type is OK */ + err = i8k_get_fan_status(1); + if (err < 0) + err = i8k_get_fan_type(1); if (err >= 0) i8k_hwmon_flags |= I8K_HWMON_HAVE_FAN2; + /* Third fan attributes, if fan status or type is OK */ + err = i8k_get_fan_status(2); + if (err < 0) + err = i8k_get_fan_type(2); + if (err >= 0) + i8k_hwmon_flags |= I8K_HWMON_HAVE_FAN3; + i8k_hwmon_dev = hwmon_device_register_with_groups(NULL, "dell_smm", NULL, i8k_groups); if (IS_ERR(i8k_hwmon_dev)) { @@ -929,12 +996,14 @@ static struct dmi_system_id i8k_dmi_table[] __initdata = { MODULE_DEVICE_TABLE(dmi, i8k_dmi_table); -static struct dmi_system_id i8k_blacklist_dmi_table[] __initdata = { +/* + * On some machines once I8K_SMM_GET_FAN_TYPE is issued then CPU fan speed + * randomly going up and down due to bug in Dell SMM or BIOS. Here is blacklist + * of affected Dell machines for which we disallow I8K_SMM_GET_FAN_TYPE call. + * See bug: https://bugzilla.kernel.org/show_bug.cgi?id=100121 + */ +static struct dmi_system_id i8k_blacklist_fan_type_dmi_table[] __initdata = { { - /* - * CPU fan speed going up and down on Dell Studio XPS 8000 - * for unknown reasons. - */ .ident = "Dell Studio XPS 8000", .matches = { DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), @@ -942,16 +1011,19 @@ static struct dmi_system_id i8k_blacklist_dmi_table[] __initdata = { }, }, { - /* - * CPU fan speed going up and down on Dell Studio XPS 8100 - * for unknown reasons. - */ .ident = "Dell Studio XPS 8100", .matches = { DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Studio XPS 8100"), }, }, + { + .ident = "Dell Inspiron 580", + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Inspiron 580 "), + }, + }, { } }; @@ -966,8 +1038,7 @@ static int __init i8k_probe(void) /* * Get DMI information */ - if (!dmi_check_system(i8k_dmi_table) || - dmi_check_system(i8k_blacklist_dmi_table)) { + if (!dmi_check_system(i8k_dmi_table)) { if (!ignore_dmi && !force) return -ENODEV; @@ -978,8 +1049,13 @@ static int __init i8k_probe(void) i8k_get_dmi_data(DMI_BIOS_VERSION)); } + if (dmi_check_system(i8k_blacklist_fan_type_dmi_table)) + disallow_fan_type_call = true; + strlcpy(bios_version, i8k_get_dmi_data(DMI_BIOS_VERSION), sizeof(bios_version)); + strlcpy(bios_machineid, i8k_get_dmi_data(DMI_PRODUCT_SERIAL), + sizeof(bios_machineid)); /* * Get SMM Dell signature diff --git a/drivers/hwmon/emc6w201.c b/drivers/hwmon/emc6w201.c index ada90716448d..f37fe2011640 100644 --- a/drivers/hwmon/emc6w201.c +++ b/drivers/hwmon/emc6w201.c @@ -464,7 +464,7 @@ static int emc6w201_detect(struct i2c_client *client, if (verstep < 0 || (verstep & 0xF0) != 0xB0) return -ENODEV; if ((verstep & 0x0F) > 2) { - dev_dbg(&client->dev, "Unknwown EMC6W201 stepping %d\n", + dev_dbg(&client->dev, "Unknown EMC6W201 stepping %d\n", verstep & 0x0F); return -ENODEV; } diff --git a/drivers/hwmon/ftsteutates.c b/drivers/hwmon/ftsteutates.c new file mode 100644 index 000000000000..2b2ff67026be --- /dev/null +++ b/drivers/hwmon/ftsteutates.c @@ -0,0 +1,819 @@ +/* + * Support for the FTS Systemmonitoring Chip "Teutates" + * + * Copyright (C) 2016 Fujitsu Technology Solutions GmbH, + * Thilo Cestonaro + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FTS_DEVICE_ID_REG 0x0000 +#define FTS_DEVICE_REVISION_REG 0x0001 +#define FTS_DEVICE_STATUS_REG 0x0004 +#define FTS_SATELLITE_STATUS_REG 0x0005 +#define FTS_EVENT_STATUS_REG 0x0006 +#define FTS_GLOBAL_CONTROL_REG 0x0007 + +#define FTS_SENSOR_EVENT_REG 0x0010 + +#define FTS_FAN_EVENT_REG 0x0014 +#define FTS_FAN_PRESENT_REG 0x0015 + +#define FTS_POWER_ON_TIME_COUNTER_A 0x007A +#define FTS_POWER_ON_TIME_COUNTER_B 0x007B +#define FTS_POWER_ON_TIME_COUNTER_C 0x007C + +#define FTS_PAGE_SELECT_REG 0x007F + +#define FTS_WATCHDOG_TIME_PRESET 0x000B +#define FTS_WATCHDOG_CONTROL 0x5081 + +#define FTS_NO_FAN_SENSORS 0x08 +#define FTS_NO_TEMP_SENSORS 0x10 +#define FTS_NO_VOLT_SENSORS 0x04 + +static struct i2c_device_id fts_id[] = { + { "ftsteutates", 0 }, + { } +}; +MODULE_DEVICE_TABLE(i2c, fts_id); + +enum WATCHDOG_RESOLUTION { + seconds = 1, + minutes = 60 +}; + +struct fts_data { + struct i2c_client *client; + /* update sensor data lock */ + struct mutex update_lock; + /* read/write register lock */ + struct mutex access_lock; + unsigned long last_updated; /* in jiffies */ + struct watchdog_device wdd; + enum WATCHDOG_RESOLUTION resolution; + bool valid; /* false until following fields are valid */ + + u8 volt[FTS_NO_VOLT_SENSORS]; + + u8 temp_input[FTS_NO_TEMP_SENSORS]; + u8 temp_alarm; + + u8 fan_present; + u8 fan_input[FTS_NO_FAN_SENSORS]; /* in rps */ + u8 fan_source[FTS_NO_FAN_SENSORS]; + u8 fan_alarm; +}; + +#define FTS_REG_FAN_INPUT(idx) ((idx) + 0x20) +#define FTS_REG_FAN_SOURCE(idx) ((idx) + 0x30) +#define FTS_REG_FAN_CONTROL(idx) (((idx) << 16) + 0x4881) + +#define FTS_REG_TEMP_INPUT(idx) ((idx) + 0x40) +#define FTS_REG_TEMP_CONTROL(idx) (((idx) << 16) + 0x0681) + +#define FTS_REG_VOLT(idx) ((idx) + 0x18) + +/*****************************************************************************/ +/* I2C Helper functions */ +/*****************************************************************************/ +static int fts_read_byte(struct i2c_client *client, unsigned short reg) +{ + int ret; + unsigned char page = reg >> 8; + struct fts_data *data = dev_get_drvdata(&client->dev); + + mutex_lock(&data->access_lock); + + dev_dbg(&client->dev, "page select - page: 0x%.02x\n", page); + ret = i2c_smbus_write_byte_data(client, FTS_PAGE_SELECT_REG, page); + if (ret < 0) + goto error; + + reg &= 0xFF; + ret = i2c_smbus_read_byte_data(client, reg); + dev_dbg(&client->dev, "read - reg: 0x%.02x: val: 0x%.02x\n", reg, ret); + +error: + mutex_unlock(&data->access_lock); + return ret; +} + +static int fts_write_byte(struct i2c_client *client, unsigned short reg, + unsigned char value) +{ + int ret; + unsigned char page = reg >> 8; + struct fts_data *data = dev_get_drvdata(&client->dev); + + mutex_lock(&data->access_lock); + + dev_dbg(&client->dev, "page select - page: 0x%.02x\n", page); + ret = i2c_smbus_write_byte_data(client, FTS_PAGE_SELECT_REG, page); + if (ret < 0) + goto error; + + reg &= 0xFF; + dev_dbg(&client->dev, + "write - reg: 0x%.02x: val: 0x%.02x\n", reg, value); + ret = i2c_smbus_write_byte_data(client, reg, value); + +error: + mutex_unlock(&data->access_lock); + return ret; +} + +/*****************************************************************************/ +/* Data Updater Helper function */ +/*****************************************************************************/ +static int fts_update_device(struct fts_data *data) +{ + int i; + int err = 0; + + mutex_lock(&data->update_lock); + if (!time_after(jiffies, data->last_updated + 2 * HZ) && data->valid) + goto exit; + + err = fts_read_byte(data->client, FTS_DEVICE_STATUS_REG); + if (err < 0) + goto exit; + + data->valid = !!(err & 0x02); /* Data not ready yet */ + if (unlikely(!data->valid)) { + err = -EAGAIN; + goto exit; + } + + err = fts_read_byte(data->client, FTS_FAN_PRESENT_REG); + if (err < 0) + goto exit; + data->fan_present = err; + + err = fts_read_byte(data->client, FTS_FAN_EVENT_REG); + if (err < 0) + goto exit; + data->fan_alarm = err; + + for (i = 0; i < FTS_NO_FAN_SENSORS; i++) { + if (data->fan_present & BIT(i)) { + err = fts_read_byte(data->client, FTS_REG_FAN_INPUT(i)); + if (err < 0) + goto exit; + data->fan_input[i] = err; + + err = fts_read_byte(data->client, + FTS_REG_FAN_SOURCE(i)); + if (err < 0) + goto exit; + data->fan_source[i] = err; + } else { + data->fan_input[i] = 0; + data->fan_source[i] = 0; + } + } + + err = fts_read_byte(data->client, FTS_SENSOR_EVENT_REG); + if (err < 0) + goto exit; + data->temp_alarm = err; + + for (i = 0; i < FTS_NO_TEMP_SENSORS; i++) { + err = fts_read_byte(data->client, FTS_REG_TEMP_INPUT(i)); + if (err < 0) + goto exit; + data->temp_input[i] = err; + } + + for (i = 0; i < FTS_NO_VOLT_SENSORS; i++) { + err = fts_read_byte(data->client, FTS_REG_VOLT(i)); + if (err < 0) + goto exit; + data->volt[i] = err; + } + data->last_updated = jiffies; + err = 0; +exit: + mutex_unlock(&data->update_lock); + return err; +} + +/*****************************************************************************/ +/* Watchdog functions */ +/*****************************************************************************/ +static int fts_wd_set_resolution(struct fts_data *data, + enum WATCHDOG_RESOLUTION resolution) +{ + int ret; + + if (data->resolution == resolution) + return 0; + + ret = fts_read_byte(data->client, FTS_WATCHDOG_CONTROL); + if (ret < 0) + return ret; + + if ((resolution == seconds && ret & BIT(1)) || + (resolution == minutes && (ret & BIT(1)) == 0)) { + data->resolution = resolution; + return 0; + } + + if (resolution == seconds) + set_bit(1, (unsigned long *)&ret); + else + ret &= ~BIT(1); + + ret = fts_write_byte(data->client, FTS_WATCHDOG_CONTROL, ret); + if (ret < 0) + return ret; + + data->resolution = resolution; + return ret; +} + +static int fts_wd_set_timeout(struct watchdog_device *wdd, unsigned int timeout) +{ + struct fts_data *data; + enum WATCHDOG_RESOLUTION resolution = seconds; + int ret; + + data = watchdog_get_drvdata(wdd); + /* switch watchdog resolution to minutes if timeout does not fit + * into a byte + */ + if (timeout > 0xFF) { + timeout = DIV_ROUND_UP(timeout, 60) * 60; + resolution = minutes; + } + + ret = fts_wd_set_resolution(data, resolution); + if (ret < 0) + return ret; + + wdd->timeout = timeout; + return 0; +} + +static int fts_wd_start(struct watchdog_device *wdd) +{ + struct fts_data *data = watchdog_get_drvdata(wdd); + + return fts_write_byte(data->client, FTS_WATCHDOG_TIME_PRESET, + wdd->timeout / (u8)data->resolution); +} + +static int fts_wd_stop(struct watchdog_device *wdd) +{ + struct fts_data *data; + + data = watchdog_get_drvdata(wdd); + return fts_write_byte(data->client, FTS_WATCHDOG_TIME_PRESET, 0); +} + +static const struct watchdog_info fts_wd_info = { + .options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE, + .identity = "FTS Teutates Hardware Watchdog", +}; + +static const struct watchdog_ops fts_wd_ops = { + .owner = THIS_MODULE, + .start = fts_wd_start, + .stop = fts_wd_stop, + .set_timeout = fts_wd_set_timeout, +}; + +static int fts_watchdog_init(struct fts_data *data) +{ + int timeout, ret; + + watchdog_set_drvdata(&data->wdd, data); + + timeout = fts_read_byte(data->client, FTS_WATCHDOG_TIME_PRESET); + if (timeout < 0) + return timeout; + + /* watchdog not running, set timeout to a default of 60 sec. */ + if (timeout == 0) { + ret = fts_wd_set_resolution(data, seconds); + if (ret < 0) + return ret; + data->wdd.timeout = 60; + } else { + ret = fts_read_byte(data->client, FTS_WATCHDOG_CONTROL); + if (ret < 0) + return ret; + + data->resolution = ret & BIT(1) ? seconds : minutes; + data->wdd.timeout = timeout * (u8)data->resolution; + set_bit(WDOG_HW_RUNNING, &data->wdd.status); + } + + /* Register our watchdog part */ + data->wdd.info = &fts_wd_info; + data->wdd.ops = &fts_wd_ops; + data->wdd.parent = &data->client->dev; + data->wdd.min_timeout = 1; + + /* max timeout 255 minutes. */ + data->wdd.max_hw_heartbeat_ms = 0xFF * 60 * MSEC_PER_SEC; + + return watchdog_register_device(&data->wdd); +} + +/*****************************************************************************/ +/* SysFS handler functions */ +/*****************************************************************************/ +static ssize_t show_in_value(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + int err; + + err = fts_update_device(data); + if (err < 0) + return err; + + return sprintf(buf, "%u\n", data->volt[index]); +} + +static ssize_t show_temp_value(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + int err; + + err = fts_update_device(data); + if (err < 0) + return err; + + return sprintf(buf, "%u\n", data->temp_input[index]); +} + +static ssize_t show_temp_fault(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + int err; + + err = fts_update_device(data); + if (err < 0) + return err; + + /* 00h Temperature = Sensor Error */ + return sprintf(buf, "%d\n", data->temp_input[index] == 0); +} + +static ssize_t show_temp_alarm(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + int err; + + err = fts_update_device(data); + if (err < 0) + return err; + + return sprintf(buf, "%u\n", !!(data->temp_alarm & BIT(index))); +} + +static ssize_t +clear_temp_alarm(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + long ret; + + ret = fts_update_device(data); + if (ret < 0) + return ret; + + if (kstrtoul(buf, 10, &ret) || ret != 0) + return -EINVAL; + + mutex_lock(&data->update_lock); + ret = fts_read_byte(data->client, FTS_REG_TEMP_CONTROL(index)); + if (ret < 0) + goto error; + + ret = fts_write_byte(data->client, FTS_REG_TEMP_CONTROL(index), + ret | 0x1); + if (ret < 0) + goto error; + + data->valid = false; +error: + mutex_unlock(&data->update_lock); + return ret; +} + +static ssize_t show_fan_value(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + int err; + + err = fts_update_device(data); + if (err < 0) + return err; + + return sprintf(buf, "%u\n", data->fan_input[index]); +} + +static ssize_t show_fan_source(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + int err; + + err = fts_update_device(data); + if (err < 0) + return err; + + return sprintf(buf, "%u\n", data->fan_source[index]); +} + +static ssize_t show_fan_alarm(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + int err; + + err = fts_update_device(data); + if (err < 0) + return err; + + return sprintf(buf, "%d\n", !!(data->fan_alarm & BIT(index))); +} + +static ssize_t +clear_fan_alarm(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct fts_data *data = dev_get_drvdata(dev); + int index = to_sensor_dev_attr(devattr)->index; + long ret; + + ret = fts_update_device(data); + if (ret < 0) + return ret; + + if (kstrtoul(buf, 10, &ret) || ret != 0) + return -EINVAL; + + mutex_lock(&data->update_lock); + ret = fts_read_byte(data->client, FTS_REG_FAN_CONTROL(index)); + if (ret < 0) + goto error; + + ret = fts_write_byte(data->client, FTS_REG_FAN_CONTROL(index), + ret | 0x1); + if (ret < 0) + goto error; + + data->valid = false; +error: + mutex_unlock(&data->update_lock); + return ret; +} + +/*****************************************************************************/ +/* SysFS structs */ +/*****************************************************************************/ + +/* Temprature sensors */ +static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_temp_value, NULL, 0); +static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, show_temp_value, NULL, 1); +static SENSOR_DEVICE_ATTR(temp3_input, S_IRUGO, show_temp_value, NULL, 2); +static SENSOR_DEVICE_ATTR(temp4_input, S_IRUGO, show_temp_value, NULL, 3); +static SENSOR_DEVICE_ATTR(temp5_input, S_IRUGO, show_temp_value, NULL, 4); +static SENSOR_DEVICE_ATTR(temp6_input, S_IRUGO, show_temp_value, NULL, 5); +static SENSOR_DEVICE_ATTR(temp7_input, S_IRUGO, show_temp_value, NULL, 6); +static SENSOR_DEVICE_ATTR(temp8_input, S_IRUGO, show_temp_value, NULL, 7); +static SENSOR_DEVICE_ATTR(temp9_input, S_IRUGO, show_temp_value, NULL, 8); +static SENSOR_DEVICE_ATTR(temp10_input, S_IRUGO, show_temp_value, NULL, 9); +static SENSOR_DEVICE_ATTR(temp11_input, S_IRUGO, show_temp_value, NULL, 10); +static SENSOR_DEVICE_ATTR(temp12_input, S_IRUGO, show_temp_value, NULL, 11); +static SENSOR_DEVICE_ATTR(temp13_input, S_IRUGO, show_temp_value, NULL, 12); +static SENSOR_DEVICE_ATTR(temp14_input, S_IRUGO, show_temp_value, NULL, 13); +static SENSOR_DEVICE_ATTR(temp15_input, S_IRUGO, show_temp_value, NULL, 14); +static SENSOR_DEVICE_ATTR(temp16_input, S_IRUGO, show_temp_value, NULL, 15); + +static SENSOR_DEVICE_ATTR(temp1_fault, S_IRUGO, show_temp_fault, NULL, 0); +static SENSOR_DEVICE_ATTR(temp2_fault, S_IRUGO, show_temp_fault, NULL, 1); +static SENSOR_DEVICE_ATTR(temp3_fault, S_IRUGO, show_temp_fault, NULL, 2); +static SENSOR_DEVICE_ATTR(temp4_fault, S_IRUGO, show_temp_fault, NULL, 3); +static SENSOR_DEVICE_ATTR(temp5_fault, S_IRUGO, show_temp_fault, NULL, 4); +static SENSOR_DEVICE_ATTR(temp6_fault, S_IRUGO, show_temp_fault, NULL, 5); +static SENSOR_DEVICE_ATTR(temp7_fault, S_IRUGO, show_temp_fault, NULL, 6); +static SENSOR_DEVICE_ATTR(temp8_fault, S_IRUGO, show_temp_fault, NULL, 7); +static SENSOR_DEVICE_ATTR(temp9_fault, S_IRUGO, show_temp_fault, NULL, 8); +static SENSOR_DEVICE_ATTR(temp10_fault, S_IRUGO, show_temp_fault, NULL, 9); +static SENSOR_DEVICE_ATTR(temp11_fault, S_IRUGO, show_temp_fault, NULL, 10); +static SENSOR_DEVICE_ATTR(temp12_fault, S_IRUGO, show_temp_fault, NULL, 11); +static SENSOR_DEVICE_ATTR(temp13_fault, S_IRUGO, show_temp_fault, NULL, 12); +static SENSOR_DEVICE_ATTR(temp14_fault, S_IRUGO, show_temp_fault, NULL, 13); +static SENSOR_DEVICE_ATTR(temp15_fault, S_IRUGO, show_temp_fault, NULL, 14); +static SENSOR_DEVICE_ATTR(temp16_fault, S_IRUGO, show_temp_fault, NULL, 15); + +static SENSOR_DEVICE_ATTR(temp1_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 0); +static SENSOR_DEVICE_ATTR(temp2_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 1); +static SENSOR_DEVICE_ATTR(temp3_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 2); +static SENSOR_DEVICE_ATTR(temp4_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 3); +static SENSOR_DEVICE_ATTR(temp5_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 4); +static SENSOR_DEVICE_ATTR(temp6_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 5); +static SENSOR_DEVICE_ATTR(temp7_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 6); +static SENSOR_DEVICE_ATTR(temp8_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 7); +static SENSOR_DEVICE_ATTR(temp9_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 8); +static SENSOR_DEVICE_ATTR(temp10_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 9); +static SENSOR_DEVICE_ATTR(temp11_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 10); +static SENSOR_DEVICE_ATTR(temp12_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 11); +static SENSOR_DEVICE_ATTR(temp13_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 12); +static SENSOR_DEVICE_ATTR(temp14_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 13); +static SENSOR_DEVICE_ATTR(temp15_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 14); +static SENSOR_DEVICE_ATTR(temp16_alarm, S_IRUGO | S_IWUSR, show_temp_alarm, + clear_temp_alarm, 15); + +static struct attribute *fts_temp_attrs[] = { + &sensor_dev_attr_temp1_input.dev_attr.attr, + &sensor_dev_attr_temp2_input.dev_attr.attr, + &sensor_dev_attr_temp3_input.dev_attr.attr, + &sensor_dev_attr_temp4_input.dev_attr.attr, + &sensor_dev_attr_temp5_input.dev_attr.attr, + &sensor_dev_attr_temp6_input.dev_attr.attr, + &sensor_dev_attr_temp7_input.dev_attr.attr, + &sensor_dev_attr_temp8_input.dev_attr.attr, + &sensor_dev_attr_temp9_input.dev_attr.attr, + &sensor_dev_attr_temp10_input.dev_attr.attr, + &sensor_dev_attr_temp11_input.dev_attr.attr, + &sensor_dev_attr_temp12_input.dev_attr.attr, + &sensor_dev_attr_temp13_input.dev_attr.attr, + &sensor_dev_attr_temp14_input.dev_attr.attr, + &sensor_dev_attr_temp15_input.dev_attr.attr, + &sensor_dev_attr_temp16_input.dev_attr.attr, + + &sensor_dev_attr_temp1_fault.dev_attr.attr, + &sensor_dev_attr_temp2_fault.dev_attr.attr, + &sensor_dev_attr_temp3_fault.dev_attr.attr, + &sensor_dev_attr_temp4_fault.dev_attr.attr, + &sensor_dev_attr_temp5_fault.dev_attr.attr, + &sensor_dev_attr_temp6_fault.dev_attr.attr, + &sensor_dev_attr_temp7_fault.dev_attr.attr, + &sensor_dev_attr_temp8_fault.dev_attr.attr, + &sensor_dev_attr_temp9_fault.dev_attr.attr, + &sensor_dev_attr_temp10_fault.dev_attr.attr, + &sensor_dev_attr_temp11_fault.dev_attr.attr, + &sensor_dev_attr_temp12_fault.dev_attr.attr, + &sensor_dev_attr_temp13_fault.dev_attr.attr, + &sensor_dev_attr_temp14_fault.dev_attr.attr, + &sensor_dev_attr_temp15_fault.dev_attr.attr, + &sensor_dev_attr_temp16_fault.dev_attr.attr, + + &sensor_dev_attr_temp1_alarm.dev_attr.attr, + &sensor_dev_attr_temp2_alarm.dev_attr.attr, + &sensor_dev_attr_temp3_alarm.dev_attr.attr, + &sensor_dev_attr_temp4_alarm.dev_attr.attr, + &sensor_dev_attr_temp5_alarm.dev_attr.attr, + &sensor_dev_attr_temp6_alarm.dev_attr.attr, + &sensor_dev_attr_temp7_alarm.dev_attr.attr, + &sensor_dev_attr_temp8_alarm.dev_attr.attr, + &sensor_dev_attr_temp9_alarm.dev_attr.attr, + &sensor_dev_attr_temp10_alarm.dev_attr.attr, + &sensor_dev_attr_temp11_alarm.dev_attr.attr, + &sensor_dev_attr_temp12_alarm.dev_attr.attr, + &sensor_dev_attr_temp13_alarm.dev_attr.attr, + &sensor_dev_attr_temp14_alarm.dev_attr.attr, + &sensor_dev_attr_temp15_alarm.dev_attr.attr, + &sensor_dev_attr_temp16_alarm.dev_attr.attr, + NULL +}; + +/* Fans */ +static SENSOR_DEVICE_ATTR(fan1_input, S_IRUGO, show_fan_value, NULL, 0); +static SENSOR_DEVICE_ATTR(fan2_input, S_IRUGO, show_fan_value, NULL, 1); +static SENSOR_DEVICE_ATTR(fan3_input, S_IRUGO, show_fan_value, NULL, 2); +static SENSOR_DEVICE_ATTR(fan4_input, S_IRUGO, show_fan_value, NULL, 3); +static SENSOR_DEVICE_ATTR(fan5_input, S_IRUGO, show_fan_value, NULL, 4); +static SENSOR_DEVICE_ATTR(fan6_input, S_IRUGO, show_fan_value, NULL, 5); +static SENSOR_DEVICE_ATTR(fan7_input, S_IRUGO, show_fan_value, NULL, 6); +static SENSOR_DEVICE_ATTR(fan8_input, S_IRUGO, show_fan_value, NULL, 7); + +static SENSOR_DEVICE_ATTR(fan1_source, S_IRUGO, show_fan_source, NULL, 0); +static SENSOR_DEVICE_ATTR(fan2_source, S_IRUGO, show_fan_source, NULL, 1); +static SENSOR_DEVICE_ATTR(fan3_source, S_IRUGO, show_fan_source, NULL, 2); +static SENSOR_DEVICE_ATTR(fan4_source, S_IRUGO, show_fan_source, NULL, 3); +static SENSOR_DEVICE_ATTR(fan5_source, S_IRUGO, show_fan_source, NULL, 4); +static SENSOR_DEVICE_ATTR(fan6_source, S_IRUGO, show_fan_source, NULL, 5); +static SENSOR_DEVICE_ATTR(fan7_source, S_IRUGO, show_fan_source, NULL, 6); +static SENSOR_DEVICE_ATTR(fan8_source, S_IRUGO, show_fan_source, NULL, 7); + +static SENSOR_DEVICE_ATTR(fan1_alarm, S_IRUGO | S_IWUSR, + show_fan_alarm, clear_fan_alarm, 0); +static SENSOR_DEVICE_ATTR(fan2_alarm, S_IRUGO | S_IWUSR, + show_fan_alarm, clear_fan_alarm, 1); +static SENSOR_DEVICE_ATTR(fan3_alarm, S_IRUGO | S_IWUSR, + show_fan_alarm, clear_fan_alarm, 2); +static SENSOR_DEVICE_ATTR(fan4_alarm, S_IRUGO | S_IWUSR, + show_fan_alarm, clear_fan_alarm, 3); +static SENSOR_DEVICE_ATTR(fan5_alarm, S_IRUGO | S_IWUSR, + show_fan_alarm, clear_fan_alarm, 4); +static SENSOR_DEVICE_ATTR(fan6_alarm, S_IRUGO | S_IWUSR, + show_fan_alarm, clear_fan_alarm, 5); +static SENSOR_DEVICE_ATTR(fan7_alarm, S_IRUGO | S_IWUSR, + show_fan_alarm, clear_fan_alarm, 6); +static SENSOR_DEVICE_ATTR(fan8_alarm, S_IRUGO | S_IWUSR, + show_fan_alarm, clear_fan_alarm, 7); + +static struct attribute *fts_fan_attrs[] = { + &sensor_dev_attr_fan1_input.dev_attr.attr, + &sensor_dev_attr_fan2_input.dev_attr.attr, + &sensor_dev_attr_fan3_input.dev_attr.attr, + &sensor_dev_attr_fan4_input.dev_attr.attr, + &sensor_dev_attr_fan5_input.dev_attr.attr, + &sensor_dev_attr_fan6_input.dev_attr.attr, + &sensor_dev_attr_fan7_input.dev_attr.attr, + &sensor_dev_attr_fan8_input.dev_attr.attr, + + &sensor_dev_attr_fan1_source.dev_attr.attr, + &sensor_dev_attr_fan2_source.dev_attr.attr, + &sensor_dev_attr_fan3_source.dev_attr.attr, + &sensor_dev_attr_fan4_source.dev_attr.attr, + &sensor_dev_attr_fan5_source.dev_attr.attr, + &sensor_dev_attr_fan6_source.dev_attr.attr, + &sensor_dev_attr_fan7_source.dev_attr.attr, + &sensor_dev_attr_fan8_source.dev_attr.attr, + + &sensor_dev_attr_fan1_alarm.dev_attr.attr, + &sensor_dev_attr_fan2_alarm.dev_attr.attr, + &sensor_dev_attr_fan3_alarm.dev_attr.attr, + &sensor_dev_attr_fan4_alarm.dev_attr.attr, + &sensor_dev_attr_fan5_alarm.dev_attr.attr, + &sensor_dev_attr_fan6_alarm.dev_attr.attr, + &sensor_dev_attr_fan7_alarm.dev_attr.attr, + &sensor_dev_attr_fan8_alarm.dev_attr.attr, + NULL +}; + +/* Voltages */ +static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, show_in_value, NULL, 0); +static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, show_in_value, NULL, 1); +static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, show_in_value, NULL, 2); +static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, show_in_value, NULL, 3); +static struct attribute *fts_voltage_attrs[] = { + &sensor_dev_attr_in1_input.dev_attr.attr, + &sensor_dev_attr_in2_input.dev_attr.attr, + &sensor_dev_attr_in3_input.dev_attr.attr, + &sensor_dev_attr_in4_input.dev_attr.attr, + NULL +}; + +static const struct attribute_group fts_voltage_attr_group = { + .attrs = fts_voltage_attrs +}; + +static const struct attribute_group fts_temp_attr_group = { + .attrs = fts_temp_attrs +}; + +static const struct attribute_group fts_fan_attr_group = { + .attrs = fts_fan_attrs +}; + +static const struct attribute_group *fts_attr_groups[] = { + &fts_voltage_attr_group, + &fts_temp_attr_group, + &fts_fan_attr_group, + NULL +}; + +/*****************************************************************************/ +/* Module initialization / remove functions */ +/*****************************************************************************/ +static int fts_remove(struct i2c_client *client) +{ + struct fts_data *data = dev_get_drvdata(&client->dev); + + watchdog_unregister_device(&data->wdd); + return 0; +} + +static int fts_probe(struct i2c_client *client, const struct i2c_device_id *id) +{ + u8 revision; + struct fts_data *data; + int err; + s8 deviceid; + struct device *hwmon_dev; + + if (client->addr != 0x73) + return -ENODEV; + + /* Baseboard Management Controller check */ + deviceid = i2c_smbus_read_byte_data(client, FTS_DEVICE_ID_REG); + if (deviceid > 0 && (deviceid & 0xF0) == 0x10) { + switch (deviceid & 0x0F) { + case 0x01: + break; + default: + dev_dbg(&client->dev, + "No Baseboard Management Controller\n"); + return -ENODEV; + } + } else { + dev_dbg(&client->dev, "No fujitsu board\n"); + return -ENODEV; + } + + data = devm_kzalloc(&client->dev, sizeof(struct fts_data), + GFP_KERNEL); + if (!data) + return -ENOMEM; + + mutex_init(&data->update_lock); + mutex_init(&data->access_lock); + data->client = client; + dev_set_drvdata(&client->dev, data); + + err = i2c_smbus_read_byte_data(client, FTS_DEVICE_REVISION_REG); + if (err < 0) + return err; + revision = err; + + hwmon_dev = devm_hwmon_device_register_with_groups(&client->dev, + "ftsteutates", + data, + fts_attr_groups); + if (IS_ERR(hwmon_dev)) + return PTR_ERR(hwmon_dev); + + err = fts_watchdog_init(data); + if (err) + return err; + + dev_info(&client->dev, "Detected FTS Teutates chip, revision: %d.%d\n", + (revision & 0xF0) >> 4, revision & 0x0F); + return 0; +} + +/*****************************************************************************/ +/* Module Details */ +/*****************************************************************************/ +static struct i2c_driver fts_driver = { + .driver = { + .name = "ftsteutates", + }, + .id_table = fts_id, + .probe = fts_probe, + .remove = fts_remove, +}; + +module_i2c_driver(fts_driver); + +MODULE_AUTHOR("Thilo Cestonaro "); +MODULE_DESCRIPTION("FTS Teutates driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c new file mode 100644 index 000000000000..e6b49500c52a --- /dev/null +++ b/drivers/hwmon/ina3221.c @@ -0,0 +1,445 @@ +/* + * INA3221 Triple Current/Voltage Monitor + * + * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include + +#define INA3221_DRIVER_NAME "ina3221" + +#define INA3221_CONFIG 0x00 +#define INA3221_SHUNT1 0x01 +#define INA3221_BUS1 0x02 +#define INA3221_SHUNT2 0x03 +#define INA3221_BUS2 0x04 +#define INA3221_SHUNT3 0x05 +#define INA3221_BUS3 0x06 +#define INA3221_CRIT1 0x07 +#define INA3221_WARN1 0x08 +#define INA3221_CRIT2 0x09 +#define INA3221_WARN2 0x0a +#define INA3221_CRIT3 0x0b +#define INA3221_WARN3 0x0c +#define INA3221_MASK_ENABLE 0x0f + +#define INA3221_CONFIG_MODE_SHUNT BIT(1) +#define INA3221_CONFIG_MODE_BUS BIT(2) +#define INA3221_CONFIG_MODE_CONTINUOUS BIT(3) + +#define INA3221_RSHUNT_DEFAULT 10000 + +enum ina3221_fields { + /* Configuration */ + F_RST, + + /* Alert Flags */ + F_WF3, F_WF2, F_WF1, + F_CF3, F_CF2, F_CF1, + + /* sentinel */ + F_MAX_FIELDS +}; + +static const struct reg_field ina3221_reg_fields[] = { + [F_RST] = REG_FIELD(INA3221_CONFIG, 15, 15), + + [F_WF3] = REG_FIELD(INA3221_MASK_ENABLE, 3, 3), + [F_WF2] = REG_FIELD(INA3221_MASK_ENABLE, 4, 4), + [F_WF1] = REG_FIELD(INA3221_MASK_ENABLE, 5, 5), + [F_CF3] = REG_FIELD(INA3221_MASK_ENABLE, 7, 7), + [F_CF2] = REG_FIELD(INA3221_MASK_ENABLE, 8, 8), + [F_CF1] = REG_FIELD(INA3221_MASK_ENABLE, 9, 9), +}; + +enum ina3221_channels { + INA3221_CHANNEL1, + INA3221_CHANNEL2, + INA3221_CHANNEL3, + INA3221_NUM_CHANNELS +}; + +static const unsigned int register_channel[] = { + [INA3221_SHUNT1] = INA3221_CHANNEL1, + [INA3221_SHUNT2] = INA3221_CHANNEL2, + [INA3221_SHUNT3] = INA3221_CHANNEL3, + [INA3221_CRIT1] = INA3221_CHANNEL1, + [INA3221_CRIT2] = INA3221_CHANNEL2, + [INA3221_CRIT3] = INA3221_CHANNEL3, + [INA3221_WARN1] = INA3221_CHANNEL1, + [INA3221_WARN2] = INA3221_CHANNEL2, + [INA3221_WARN3] = INA3221_CHANNEL3, +}; + +/** + * struct ina3221_data - device specific information + * @regmap: Register map of the device + * @fields: Register fields of the device + * @shunt_resistors: Array of resistor values per channel + */ +struct ina3221_data { + struct regmap *regmap; + struct regmap_field *fields[F_MAX_FIELDS]; + int shunt_resistors[INA3221_NUM_CHANNELS]; +}; + +static int ina3221_read_value(struct ina3221_data *ina, unsigned int reg, + int *val) +{ + unsigned int regval; + int ret; + + ret = regmap_read(ina->regmap, reg, ®val); + if (ret) + return ret; + + *val = sign_extend32(regval >> 3, 12); + + return 0; +} + +static ssize_t ina3221_show_bus_voltage(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); + struct ina3221_data *ina = dev_get_drvdata(dev); + unsigned int reg = sd_attr->index; + int val, voltage_mv, ret; + + ret = ina3221_read_value(ina, reg, &val); + if (ret) + return ret; + + voltage_mv = val * 8; + + return snprintf(buf, PAGE_SIZE, "%d\n", voltage_mv); +} + +static ssize_t ina3221_show_shunt_voltage(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); + struct ina3221_data *ina = dev_get_drvdata(dev); + unsigned int reg = sd_attr->index; + int val, voltage_uv, ret; + + ret = ina3221_read_value(ina, reg, &val); + if (ret) + return ret; + voltage_uv = val * 40; + + return snprintf(buf, PAGE_SIZE, "%d\n", voltage_uv); +} + +static ssize_t ina3221_show_current(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); + struct ina3221_data *ina = dev_get_drvdata(dev); + unsigned int reg = sd_attr->index; + unsigned int channel = register_channel[reg]; + int resistance_uo = ina->shunt_resistors[channel]; + int val, current_ma, voltage_nv, ret; + + ret = ina3221_read_value(ina, reg, &val); + if (ret) + return ret; + voltage_nv = val * 40000; + + current_ma = DIV_ROUND_CLOSEST(voltage_nv, resistance_uo); + + return snprintf(buf, PAGE_SIZE, "%d\n", current_ma); +} + +static ssize_t ina3221_set_current(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); + struct ina3221_data *ina = dev_get_drvdata(dev); + unsigned int reg = sd_attr->index; + unsigned int channel = register_channel[reg]; + int resistance_uo = ina->shunt_resistors[channel]; + int val, current_ma, voltage_uv, ret; + + ret = kstrtoint(buf, 0, ¤t_ma); + if (ret) + return ret; + + /* clamp current */ + current_ma = clamp_val(current_ma, + INT_MIN / resistance_uo, + INT_MAX / resistance_uo); + + voltage_uv = DIV_ROUND_CLOSEST(current_ma * resistance_uo, 1000); + + /* clamp voltage */ + voltage_uv = clamp_val(voltage_uv, -163800, 163800); + + /* 1 / 40uV(scale) << 3(register shift) = 5 */ + val = DIV_ROUND_CLOSEST(voltage_uv, 5) & 0xfff8; + + ret = regmap_write(ina->regmap, reg, val); + if (ret) + return ret; + + return count; +} + +static ssize_t ina3221_show_shunt(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); + struct ina3221_data *ina = dev_get_drvdata(dev); + unsigned int channel = sd_attr->index; + unsigned int resistance_uo; + + resistance_uo = ina->shunt_resistors[channel]; + + return snprintf(buf, PAGE_SIZE, "%d\n", resistance_uo); +} + +static ssize_t ina3221_set_shunt(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); + struct ina3221_data *ina = dev_get_drvdata(dev); + unsigned int channel = sd_attr->index; + int val; + int ret; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + val = clamp_val(val, 1, INT_MAX); + + ina->shunt_resistors[channel] = val; + + return count; +} + +static ssize_t ina3221_show_alert(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sensor_device_attribute *sd_attr = to_sensor_dev_attr(attr); + struct ina3221_data *ina = dev_get_drvdata(dev); + unsigned int field = sd_attr->index; + unsigned int regval; + int ret; + + ret = regmap_field_read(ina->fields[field], ®val); + if (ret) + return ret; + + return snprintf(buf, PAGE_SIZE, "%d\n", regval); +} + +/* bus voltage */ +static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, + ina3221_show_bus_voltage, NULL, INA3221_BUS1); +static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, + ina3221_show_bus_voltage, NULL, INA3221_BUS2); +static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, + ina3221_show_bus_voltage, NULL, INA3221_BUS3); + +/* calculated current */ +static SENSOR_DEVICE_ATTR(curr1_input, S_IRUGO, + ina3221_show_current, NULL, INA3221_SHUNT1); +static SENSOR_DEVICE_ATTR(curr2_input, S_IRUGO, + ina3221_show_current, NULL, INA3221_SHUNT2); +static SENSOR_DEVICE_ATTR(curr3_input, S_IRUGO, + ina3221_show_current, NULL, INA3221_SHUNT3); + +/* shunt resistance */ +static SENSOR_DEVICE_ATTR(shunt1_resistor, S_IRUGO | S_IWUSR, + ina3221_show_shunt, ina3221_set_shunt, INA3221_CHANNEL1); +static SENSOR_DEVICE_ATTR(shunt2_resistor, S_IRUGO | S_IWUSR, + ina3221_show_shunt, ina3221_set_shunt, INA3221_CHANNEL2); +static SENSOR_DEVICE_ATTR(shunt3_resistor, S_IRUGO | S_IWUSR, + ina3221_show_shunt, ina3221_set_shunt, INA3221_CHANNEL3); + +/* critical current */ +static SENSOR_DEVICE_ATTR(curr1_crit, S_IRUGO | S_IWUSR, + ina3221_show_current, ina3221_set_current, INA3221_CRIT1); +static SENSOR_DEVICE_ATTR(curr2_crit, S_IRUGO | S_IWUSR, + ina3221_show_current, ina3221_set_current, INA3221_CRIT2); +static SENSOR_DEVICE_ATTR(curr3_crit, S_IRUGO | S_IWUSR, + ina3221_show_current, ina3221_set_current, INA3221_CRIT3); + +/* critical current alert */ +static SENSOR_DEVICE_ATTR(curr1_crit_alarm, S_IRUGO, + ina3221_show_alert, NULL, F_CF1); +static SENSOR_DEVICE_ATTR(curr2_crit_alarm, S_IRUGO, + ina3221_show_alert, NULL, F_CF2); +static SENSOR_DEVICE_ATTR(curr3_crit_alarm, S_IRUGO, + ina3221_show_alert, NULL, F_CF3); + +/* warning current */ +static SENSOR_DEVICE_ATTR(curr1_max, S_IRUGO | S_IWUSR, + ina3221_show_current, ina3221_set_current, INA3221_WARN1); +static SENSOR_DEVICE_ATTR(curr2_max, S_IRUGO | S_IWUSR, + ina3221_show_current, ina3221_set_current, INA3221_WARN2); +static SENSOR_DEVICE_ATTR(curr3_max, S_IRUGO | S_IWUSR, + ina3221_show_current, ina3221_set_current, INA3221_WARN3); + +/* warning current alert */ +static SENSOR_DEVICE_ATTR(curr1_max_alarm, S_IRUGO, + ina3221_show_alert, NULL, F_WF1); +static SENSOR_DEVICE_ATTR(curr2_max_alarm, S_IRUGO, + ina3221_show_alert, NULL, F_WF2); +static SENSOR_DEVICE_ATTR(curr3_max_alarm, S_IRUGO, + ina3221_show_alert, NULL, F_WF3); + +/* shunt voltage */ +static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, + ina3221_show_shunt_voltage, NULL, INA3221_SHUNT1); +static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, + ina3221_show_shunt_voltage, NULL, INA3221_SHUNT2); +static SENSOR_DEVICE_ATTR(in6_input, S_IRUGO, + ina3221_show_shunt_voltage, NULL, INA3221_SHUNT3); + +static struct attribute *ina3221_attrs[] = { + /* channel 1 */ + &sensor_dev_attr_in1_input.dev_attr.attr, + &sensor_dev_attr_curr1_input.dev_attr.attr, + &sensor_dev_attr_shunt1_resistor.dev_attr.attr, + &sensor_dev_attr_curr1_crit.dev_attr.attr, + &sensor_dev_attr_curr1_crit_alarm.dev_attr.attr, + &sensor_dev_attr_curr1_max.dev_attr.attr, + &sensor_dev_attr_curr1_max_alarm.dev_attr.attr, + &sensor_dev_attr_in4_input.dev_attr.attr, + + /* channel 2 */ + &sensor_dev_attr_in2_input.dev_attr.attr, + &sensor_dev_attr_curr2_input.dev_attr.attr, + &sensor_dev_attr_shunt2_resistor.dev_attr.attr, + &sensor_dev_attr_curr2_crit.dev_attr.attr, + &sensor_dev_attr_curr2_crit_alarm.dev_attr.attr, + &sensor_dev_attr_curr2_max.dev_attr.attr, + &sensor_dev_attr_curr2_max_alarm.dev_attr.attr, + &sensor_dev_attr_in5_input.dev_attr.attr, + + /* channel 3 */ + &sensor_dev_attr_in3_input.dev_attr.attr, + &sensor_dev_attr_curr3_input.dev_attr.attr, + &sensor_dev_attr_shunt3_resistor.dev_attr.attr, + &sensor_dev_attr_curr3_crit.dev_attr.attr, + &sensor_dev_attr_curr3_crit_alarm.dev_attr.attr, + &sensor_dev_attr_curr3_max.dev_attr.attr, + &sensor_dev_attr_curr3_max_alarm.dev_attr.attr, + &sensor_dev_attr_in6_input.dev_attr.attr, + + NULL, +}; +ATTRIBUTE_GROUPS(ina3221); + +static const struct regmap_range ina3221_yes_ranges[] = { + regmap_reg_range(INA3221_SHUNT1, INA3221_BUS3), + regmap_reg_range(INA3221_MASK_ENABLE, INA3221_MASK_ENABLE), +}; + +static const struct regmap_access_table ina3221_volatile_table = { + .yes_ranges = ina3221_yes_ranges, + .n_yes_ranges = ARRAY_SIZE(ina3221_yes_ranges), +}; + +static const struct regmap_config ina3221_regmap_config = { + .reg_bits = 8, + .val_bits = 16, + + .cache_type = REGCACHE_RBTREE, + .volatile_table = &ina3221_volatile_table, +}; + +static int ina3221_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct device *dev = &client->dev; + struct ina3221_data *ina; + struct device *hwmon_dev; + int i, ret; + + ina = devm_kzalloc(dev, sizeof(*ina), GFP_KERNEL); + if (!ina) + return -ENOMEM; + + ina->regmap = devm_regmap_init_i2c(client, &ina3221_regmap_config); + if (IS_ERR(ina->regmap)) { + dev_err(dev, "Unable to allocate register map\n"); + return PTR_ERR(ina->regmap); + } + + for (i = 0; i < F_MAX_FIELDS; i++) { + ina->fields[i] = devm_regmap_field_alloc(dev, + ina->regmap, + ina3221_reg_fields[i]); + if (IS_ERR(ina->fields[i])) { + dev_err(dev, "Unable to allocate regmap fields\n"); + return PTR_ERR(ina->fields[i]); + } + } + + for (i = 0; i < INA3221_NUM_CHANNELS; i++) + ina->shunt_resistors[i] = INA3221_RSHUNT_DEFAULT; + + ret = regmap_field_write(ina->fields[F_RST], true); + if (ret) { + dev_err(dev, "Unable to reset device\n"); + return ret; + } + + hwmon_dev = devm_hwmon_device_register_with_groups(dev, + client->name, + ina, ina3221_groups); + if (IS_ERR(hwmon_dev)) { + dev_err(dev, "Unable to register hwmon device\n"); + return PTR_ERR(hwmon_dev); + } + + return 0; +} + +static const struct of_device_id ina3221_of_match_table[] = { + { .compatible = "ti,ina3221", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ina3221_of_match_table); + +static const struct i2c_device_id ina3221_ids[] = { + { "ina3221", 0 }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(i2c, ina3221_ids); + +static struct i2c_driver ina3221_i2c_driver = { + .probe = ina3221_probe, + .driver = { + .name = INA3221_DRIVER_NAME, + .of_match_table = ina3221_of_match_table, + }, + .id_table = ina3221_ids, +}; +module_i2c_driver(ina3221_i2c_driver); + +MODULE_AUTHOR("Andrew F. Davis "); +MODULE_DESCRIPTION("Texas Instruments INA3221 HWMon Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c index 9887d3224a86..9d5f85f3384f 100644 --- a/drivers/hwmon/jc42.c +++ b/drivers/hwmon/jc42.c @@ -31,6 +31,7 @@ #include #include #include +#include /* Addresses to scan */ static const unsigned short normal_i2c[] = { @@ -104,6 +105,9 @@ static const unsigned short normal_i2c[] = { #define MCP9804_DEVID 0x0200 #define MCP9804_DEVID_MASK 0xfffc +#define MCP9808_DEVID 0x0400 +#define MCP9808_DEVID_MASK 0xfffc + #define MCP98242_DEVID 0x2000 #define MCP98242_DEVID_MASK 0xfffc @@ -160,6 +164,7 @@ static struct jc42_chips jc42_chips[] = { { IDT_MANID, TS3001_DEVID, TS3001_DEVID_MASK }, { MAX_MANID, MAX6604_DEVID, MAX6604_DEVID_MASK }, { MCP_MANID, MCP9804_DEVID, MCP9804_DEVID_MASK }, + { MCP_MANID, MCP9808_DEVID, MCP9808_DEVID_MASK }, { MCP_MANID, MCP98242_DEVID, MCP98242_DEVID_MASK }, { MCP_MANID, MCP98243_DEVID, MCP98243_DEVID_MASK }, { MCP_MANID, MCP98244_DEVID, MCP98244_DEVID_MASK }, @@ -537,11 +542,20 @@ static const struct i2c_device_id jc42_id[] = { }; MODULE_DEVICE_TABLE(i2c, jc42_id); +#ifdef CONFIG_OF +static const struct of_device_id jc42_of_ids[] = { + { .compatible = "jedec,jc-42.4-temp", }, + { } +}; +MODULE_DEVICE_TABLE(of, jc42_of_ids); +#endif + static struct i2c_driver jc42_driver = { - .class = I2C_CLASS_SPD, + .class = I2C_CLASS_SPD | I2C_CLASS_HWMON, .driver = { .name = "jc42", .pm = JC42_DEV_PM_OPS, + .of_match_table = of_match_ptr(jc42_of_ids), }, .probe = jc42_probe, .remove = jc42_remove, diff --git a/drivers/hwmon/jz4740-hwmon.c b/drivers/hwmon/jz4740-hwmon.c index df9b3447f2a8..0621ee1b3c98 100644 --- a/drivers/hwmon/jz4740-hwmon.c +++ b/drivers/hwmon/jz4740-hwmon.c @@ -29,23 +29,13 @@ struct jz4740_hwmon { void __iomem *base; - int irq; - const struct mfd_cell *cell; - struct device *hwmon; - + struct platform_device *pdev; struct completion read_completion; - struct mutex lock; }; -static ssize_t jz4740_hwmon_show_name(struct device *dev, - struct device_attribute *dev_attr, char *buf) -{ - return sprintf(buf, "jz4740\n"); -} - static irqreturn_t jz4740_hwmon_irq(int irq, void *data) { struct jz4740_hwmon *hwmon = data; @@ -58,6 +48,7 @@ static ssize_t jz4740_hwmon_read_adcin(struct device *dev, struct device_attribute *dev_attr, char *buf) { struct jz4740_hwmon *hwmon = dev_get_drvdata(dev); + struct platform_device *pdev = hwmon->pdev; struct completion *completion = &hwmon->read_completion; long t; unsigned long val; @@ -68,7 +59,7 @@ static ssize_t jz4740_hwmon_read_adcin(struct device *dev, reinit_completion(completion); enable_irq(hwmon->irq); - hwmon->cell->enable(to_platform_device(dev)); + hwmon->cell->enable(pdev); t = wait_for_completion_interruptible_timeout(completion, HZ); @@ -80,7 +71,7 @@ static ssize_t jz4740_hwmon_read_adcin(struct device *dev, ret = t ? t : -ETIMEDOUT; } - hwmon->cell->disable(to_platform_device(dev)); + hwmon->cell->disable(pdev); disable_irq(hwmon->irq); mutex_unlock(&hwmon->lock); @@ -88,26 +79,24 @@ static ssize_t jz4740_hwmon_read_adcin(struct device *dev, return ret; } -static DEVICE_ATTR(name, S_IRUGO, jz4740_hwmon_show_name, NULL); static DEVICE_ATTR(in0_input, S_IRUGO, jz4740_hwmon_read_adcin, NULL); -static struct attribute *jz4740_hwmon_attributes[] = { - &dev_attr_name.attr, +static struct attribute *jz4740_attrs[] = { &dev_attr_in0_input.attr, NULL }; -static const struct attribute_group jz4740_hwmon_attr_group = { - .attrs = jz4740_hwmon_attributes, -}; +ATTRIBUTE_GROUPS(jz4740); static int jz4740_hwmon_probe(struct platform_device *pdev) { int ret; + struct device *dev = &pdev->dev; struct jz4740_hwmon *hwmon; + struct device *hwmon_dev; struct resource *mem; - hwmon = devm_kzalloc(&pdev->dev, sizeof(*hwmon), GFP_KERNEL); + hwmon = devm_kzalloc(dev, sizeof(*hwmon), GFP_KERNEL); if (!hwmon) return -ENOMEM; @@ -125,12 +114,11 @@ static int jz4740_hwmon_probe(struct platform_device *pdev) if (IS_ERR(hwmon->base)) return PTR_ERR(hwmon->base); + hwmon->pdev = pdev; init_completion(&hwmon->read_completion); mutex_init(&hwmon->lock); - platform_set_drvdata(pdev, hwmon); - - ret = devm_request_irq(&pdev->dev, hwmon->irq, jz4740_hwmon_irq, 0, + ret = devm_request_irq(dev, hwmon->irq, jz4740_hwmon_irq, 0, pdev->name, hwmon); if (ret) { dev_err(&pdev->dev, "Failed to request irq: %d\n", ret); @@ -138,38 +126,13 @@ static int jz4740_hwmon_probe(struct platform_device *pdev) } disable_irq(hwmon->irq); - ret = sysfs_create_group(&pdev->dev.kobj, &jz4740_hwmon_attr_group); - if (ret) { - dev_err(&pdev->dev, "Failed to create sysfs group: %d\n", ret); - return ret; - } - - hwmon->hwmon = hwmon_device_register(&pdev->dev); - if (IS_ERR(hwmon->hwmon)) { - ret = PTR_ERR(hwmon->hwmon); - goto err_remove_file; - } - - return 0; - -err_remove_file: - sysfs_remove_group(&pdev->dev.kobj, &jz4740_hwmon_attr_group); - return ret; -} - -static int jz4740_hwmon_remove(struct platform_device *pdev) -{ - struct jz4740_hwmon *hwmon = platform_get_drvdata(pdev); - - hwmon_device_unregister(hwmon->hwmon); - sysfs_remove_group(&pdev->dev.kobj, &jz4740_hwmon_attr_group); - - return 0; + hwmon_dev = devm_hwmon_device_register_with_groups(dev, "jz4740", hwmon, + jz4740_groups); + return PTR_ERR_OR_ZERO(hwmon_dev); } static struct platform_driver jz4740_hwmon_driver = { .probe = jz4740_hwmon_probe, - .remove = jz4740_hwmon_remove, .driver = { .name = "jz4740-hwmon", }, diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c index 69166ab3151d..547a9c87c68c 100644 --- a/drivers/hwmon/lm75.c +++ b/drivers/hwmon/lm75.c @@ -26,8 +26,8 @@ #include #include #include -#include #include +#include #include #include "lm75.h" @@ -66,35 +66,21 @@ static const unsigned short normal_i2c[] = { 0x48, 0x49, 0x4a, 0x4b, 0x4c, /* The LM75 registers */ +#define LM75_REG_TEMP 0x00 #define LM75_REG_CONF 0x01 -static const u8 LM75_REG_TEMP[3] = { - 0x00, /* input */ - 0x03, /* max */ - 0x02, /* hyst */ -}; +#define LM75_REG_HYST 0x02 +#define LM75_REG_MAX 0x03 /* Each client has this additional data */ struct lm75_data { struct i2c_client *client; - struct device *hwmon_dev; - struct mutex update_lock; + struct regmap *regmap; u8 orig_conf; u8 resolution; /* In bits, between 9 and 12 */ u8 resolution_limits; - char valid; /* !=0 if registers are valid */ - unsigned long last_updated; /* In jiffies */ - unsigned long sample_time; /* In jiffies */ - s16 temp[3]; /* Register values, - 0 = input - 1 = max - 2 = hyst */ + unsigned int sample_time; /* In ms */ }; -static int lm75_read_value(struct i2c_client *client, u8 reg); -static int lm75_write_value(struct i2c_client *client, u8 reg, u16 value); -static struct lm75_data *lm75_update_device(struct device *dev); - - /*-----------------------------------------------------------------------*/ static inline long lm75_reg_to_mc(s16 temp, u8 resolution) @@ -106,12 +92,15 @@ static inline long lm75_reg_to_mc(s16 temp, u8 resolution) static int lm75_read_temp(void *dev, int *temp) { - struct lm75_data *data = lm75_update_device(dev); + struct lm75_data *data = dev_get_drvdata(dev); + unsigned int _temp; + int err; - if (IS_ERR(data)) - return PTR_ERR(data); + err = regmap_read(data->regmap, LM75_REG_TEMP, &_temp); + if (err < 0) + return err; - *temp = lm75_reg_to_mc(data->temp[0], data->resolution); + *temp = lm75_reg_to_mc(_temp, data->resolution); return 0; } @@ -120,13 +109,15 @@ static ssize_t show_temp(struct device *dev, struct device_attribute *da, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(da); - struct lm75_data *data = lm75_update_device(dev); + struct lm75_data *data = dev_get_drvdata(dev); + unsigned int temp = 0; + int err; - if (IS_ERR(data)) - return PTR_ERR(data); + err = regmap_read(data->regmap, attr->index, &temp); + if (err < 0) + return err; - return sprintf(buf, "%ld\n", lm75_reg_to_mc(data->temp[attr->index], - data->resolution)); + return sprintf(buf, "%ld\n", lm75_reg_to_mc(temp, data->resolution)); } static ssize_t set_temp(struct device *dev, struct device_attribute *da, @@ -134,8 +125,6 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *da, { struct sensor_device_attribute *attr = to_sensor_dev_attr(da); struct lm75_data *data = dev_get_drvdata(dev); - struct i2c_client *client = data->client; - int nr = attr->index; long temp; int error; u8 resolution; @@ -153,25 +142,36 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *da, else resolution = data->resolution; - mutex_lock(&data->update_lock); temp = clamp_val(temp, LM75_TEMP_MIN, LM75_TEMP_MAX); - data->temp[nr] = DIV_ROUND_CLOSEST(temp << (resolution - 8), - 1000) << (16 - resolution); - lm75_write_value(client, LM75_REG_TEMP[nr], data->temp[nr]); - mutex_unlock(&data->update_lock); + temp = DIV_ROUND_CLOSEST(temp << (resolution - 8), + 1000) << (16 - resolution); + error = regmap_write(data->regmap, attr->index, temp); + if (error < 0) + return error; + return count; } +static ssize_t show_update_interval(struct device *dev, + struct device_attribute *da, char *buf) +{ + struct lm75_data *data = dev_get_drvdata(dev); + + return sprintf(buf, "%u\n", data->sample_time); +} + static SENSOR_DEVICE_ATTR(temp1_max, S_IWUSR | S_IRUGO, - show_temp, set_temp, 1); + show_temp, set_temp, LM75_REG_MAX); static SENSOR_DEVICE_ATTR(temp1_max_hyst, S_IWUSR | S_IRUGO, - show_temp, set_temp, 2); -static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_temp, NULL, 0); + show_temp, set_temp, LM75_REG_HYST); +static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_temp, NULL, LM75_REG_TEMP); +static DEVICE_ATTR(update_interval, S_IRUGO, show_update_interval, NULL); static struct attribute *lm75_attrs[] = { &sensor_dev_attr_temp1_input.dev_attr.attr, &sensor_dev_attr_temp1_max.dev_attr.attr, &sensor_dev_attr_temp1_max_hyst.dev_attr.attr, + &dev_attr_update_interval.attr, NULL }; @@ -185,10 +185,40 @@ static const struct thermal_zone_of_device_ops lm75_of_thermal_ops = { /* device probe and removal */ +static bool lm75_is_writeable_reg(struct device *dev, unsigned int reg) +{ + return reg != LM75_REG_TEMP; +} + +static bool lm75_is_volatile_reg(struct device *dev, unsigned int reg) +{ + return reg == LM75_REG_TEMP; +} + +static const struct regmap_config lm75_regmap_config = { + .reg_bits = 8, + .val_bits = 16, + .max_register = LM75_REG_MAX, + .writeable_reg = lm75_is_writeable_reg, + .volatile_reg = lm75_is_volatile_reg, + .val_format_endian = REGMAP_ENDIAN_BIG, + .cache_type = REGCACHE_RBTREE, + .use_single_rw = true, +}; + +static void lm75_remove(void *data) +{ + struct lm75_data *lm75 = data; + struct i2c_client *client = lm75->client; + + i2c_smbus_write_byte_data(client, LM75_REG_CONF, lm75->orig_conf); +} + static int lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct device *dev = &client->dev; + struct device *hwmon_dev; struct lm75_data *data; int status; u8 set_mask, clr_mask; @@ -204,8 +234,10 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) return -ENOMEM; data->client = client; - i2c_set_clientdata(client, data); - mutex_init(&data->update_lock); + + data->regmap = devm_regmap_init_i2c(client, &lm75_regmap_config); + if (IS_ERR(data->regmap)) + return PTR_ERR(data->regmap); /* Set to LM75 resolution (9 bits, 1/2 degree C) and range. * Then tweak to be more precise when appropriate. @@ -217,7 +249,7 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) case adt75: clr_mask |= 1 << 5; /* not one-shot mode */ data->resolution = 12; - data->sample_time = HZ / 8; + data->sample_time = MSEC_PER_SEC / 8; break; case ds1775: case ds75: @@ -225,35 +257,35 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) clr_mask |= 3 << 5; set_mask |= 2 << 5; /* 11-bit mode */ data->resolution = 11; - data->sample_time = HZ; + data->sample_time = MSEC_PER_SEC; break; case ds7505: set_mask |= 3 << 5; /* 12-bit mode */ data->resolution = 12; - data->sample_time = HZ / 4; + data->sample_time = MSEC_PER_SEC / 4; break; case g751: case lm75: case lm75a: data->resolution = 9; - data->sample_time = HZ / 2; + data->sample_time = MSEC_PER_SEC / 2; break; case lm75b: data->resolution = 11; - data->sample_time = HZ / 4; + data->sample_time = MSEC_PER_SEC / 4; break; case max6625: data->resolution = 9; - data->sample_time = HZ / 4; + data->sample_time = MSEC_PER_SEC / 4; break; case max6626: data->resolution = 12; data->resolution_limits = 9; - data->sample_time = HZ / 4; + data->sample_time = MSEC_PER_SEC / 4; break; case tcn75: data->resolution = 9; - data->sample_time = HZ / 8; + data->sample_time = MSEC_PER_SEC / 8; break; case mcp980x: data->resolution_limits = 9; @@ -262,14 +294,14 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) case tmp101: set_mask |= 3 << 5; /* 12-bit mode */ data->resolution = 12; - data->sample_time = HZ; + data->sample_time = MSEC_PER_SEC; clr_mask |= 1 << 7; /* not one-shot mode */ break; case tmp112: set_mask |= 3 << 5; /* 12-bit mode */ clr_mask |= 1 << 7; /* not one-shot mode */ data->resolution = 12; - data->sample_time = HZ / 4; + data->sample_time = MSEC_PER_SEC / 4; break; case tmp105: case tmp175: @@ -278,17 +310,17 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) set_mask |= 3 << 5; /* 12-bit mode */ clr_mask |= 1 << 7; /* not one-shot mode */ data->resolution = 12; - data->sample_time = HZ / 2; + data->sample_time = MSEC_PER_SEC / 2; break; case tmp75c: clr_mask |= 1 << 5; /* not one-shot mode */ data->resolution = 12; - data->sample_time = HZ / 4; + data->sample_time = MSEC_PER_SEC / 4; break; } /* configure as specified */ - status = lm75_read_value(client, LM75_REG_CONF); + status = i2c_smbus_read_byte_data(client, LM75_REG_CONF); if (status < 0) { dev_dbg(dev, "Can't read config? %d\n", status); return status; @@ -297,30 +329,23 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) new = status & ~clr_mask; new |= set_mask; if (status != new) - lm75_write_value(client, LM75_REG_CONF, new); - dev_dbg(dev, "Config %02x\n", new); + i2c_smbus_write_byte_data(client, LM75_REG_CONF, new); - data->hwmon_dev = hwmon_device_register_with_groups(dev, client->name, - data, lm75_groups); - if (IS_ERR(data->hwmon_dev)) - return PTR_ERR(data->hwmon_dev); + devm_add_action(dev, lm75_remove, data); - devm_thermal_zone_of_sensor_register(data->hwmon_dev, 0, - data->hwmon_dev, - &lm75_of_thermal_ops); + dev_dbg(dev, "Config %02x\n", new); - dev_info(dev, "%s: sensor '%s'\n", - dev_name(data->hwmon_dev), client->name); + hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name, + data, lm75_groups); + if (IS_ERR(hwmon_dev)) + return PTR_ERR(hwmon_dev); - return 0; -} + devm_thermal_zone_of_sensor_register(hwmon_dev, 0, + hwmon_dev, + &lm75_of_thermal_ops); -static int lm75_remove(struct i2c_client *client) -{ - struct lm75_data *data = i2c_get_clientdata(client); + dev_info(dev, "%s: sensor '%s'\n", dev_name(hwmon_dev), client->name); - hwmon_device_unregister(data->hwmon_dev); - lm75_write_value(client, LM75_REG_CONF, data->orig_conf); return 0; } @@ -449,13 +474,13 @@ static int lm75_suspend(struct device *dev) { int status; struct i2c_client *client = to_i2c_client(dev); - status = lm75_read_value(client, LM75_REG_CONF); + status = i2c_smbus_read_byte_data(client, LM75_REG_CONF); if (status < 0) { dev_dbg(&client->dev, "Can't read config? %d\n", status); return status; } status = status | LM75_SHUTDOWN; - lm75_write_value(client, LM75_REG_CONF, status); + i2c_smbus_write_byte_data(client, LM75_REG_CONF, status); return 0; } @@ -463,13 +488,13 @@ static int lm75_resume(struct device *dev) { int status; struct i2c_client *client = to_i2c_client(dev); - status = lm75_read_value(client, LM75_REG_CONF); + status = i2c_smbus_read_byte_data(client, LM75_REG_CONF); if (status < 0) { dev_dbg(&client->dev, "Can't read config? %d\n", status); return status; } status = status & ~LM75_SHUTDOWN; - lm75_write_value(client, LM75_REG_CONF, status); + i2c_smbus_write_byte_data(client, LM75_REG_CONF, status); return 0; } @@ -489,73 +514,11 @@ static struct i2c_driver lm75_driver = { .pm = LM75_DEV_PM_OPS, }, .probe = lm75_probe, - .remove = lm75_remove, .id_table = lm75_ids, .detect = lm75_detect, .address_list = normal_i2c, }; -/*-----------------------------------------------------------------------*/ - -/* register access */ - -/* - * All registers are word-sized, except for the configuration register. - * LM75 uses a high-byte first convention, which is exactly opposite to - * the SMBus standard. - */ -static int lm75_read_value(struct i2c_client *client, u8 reg) -{ - if (reg == LM75_REG_CONF) - return i2c_smbus_read_byte_data(client, reg); - else - return i2c_smbus_read_word_swapped(client, reg); -} - -static int lm75_write_value(struct i2c_client *client, u8 reg, u16 value) -{ - if (reg == LM75_REG_CONF) - return i2c_smbus_write_byte_data(client, reg, value); - else - return i2c_smbus_write_word_swapped(client, reg, value); -} - -static struct lm75_data *lm75_update_device(struct device *dev) -{ - struct lm75_data *data = dev_get_drvdata(dev); - struct i2c_client *client = data->client; - struct lm75_data *ret = data; - - mutex_lock(&data->update_lock); - - if (time_after(jiffies, data->last_updated + data->sample_time) - || !data->valid) { - int i; - dev_dbg(&client->dev, "Starting lm75 update\n"); - - for (i = 0; i < ARRAY_SIZE(data->temp); i++) { - int status; - - status = lm75_read_value(client, LM75_REG_TEMP[i]); - if (unlikely(status < 0)) { - dev_dbg(dev, - "LM75: Failed to read value: reg %d, error %d\n", - LM75_REG_TEMP[i], status); - ret = ERR_PTR(status); - data->valid = 0; - goto abort; - } - data->temp[i] = status; - } - data->last_updated = jiffies; - data->valid = 1; - } - -abort: - mutex_unlock(&data->update_lock); - return ret; -} - module_i2c_driver(lm75_driver); MODULE_AUTHOR("Frodo Looijaard "); diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c index e30a5939dc0d..f51e758ba529 100644 --- a/drivers/hwmon/lm90.c +++ b/drivers/hwmon/lm90.c @@ -171,7 +171,6 @@ enum chips { lm90, adm1032, lm99, lm86, max6657, max6659, adt7461, max6680, #define SA56004_REG_R_LOCAL_TEMPL 0x22 -#define LM90_DEF_CONVRATE_RVAL 6 /* Def conversion rate register value */ #define LM90_MAX_CONVRATE_MS 16000 /* Maximum conversion rate in ms */ /* TMP451 registers */ @@ -366,11 +365,9 @@ enum lm90_temp11_reg_index { struct lm90_data { struct i2c_client *client; - struct device *hwmon_dev; const struct attribute_group *groups[6]; struct mutex update_lock; - struct regulator *regulator; - char valid; /* zero until following fields are valid */ + bool valid; /* true if register values are valid */ unsigned long last_updated; /* in jiffies */ int kind; u32 flags; @@ -412,7 +409,7 @@ static inline s32 adm1032_write_byte(struct i2c_client *client, u8 value) * because we don't want the address pointer to change between the write * byte and the read byte transactions. */ -static int lm90_read_reg(struct i2c_client *client, u8 reg, u8 *value) +static int lm90_read_reg(struct i2c_client *client, u8 reg) { int err; @@ -423,20 +420,12 @@ static int lm90_read_reg(struct i2c_client *client, u8 reg, u8 *value) } else err = i2c_smbus_read_byte_data(client, reg); - if (err < 0) { - dev_warn(&client->dev, "Register %#02x read failed (%d)\n", - reg, err); - return err; - } - *value = err; - - return 0; + return err; } -static int lm90_read16(struct i2c_client *client, u8 regh, u8 regl, u16 *value) +static int lm90_read16(struct i2c_client *client, u8 regh, u8 regl) { - int err; - u8 oldh, newh, l; + int oldh, newh, l; /* * There is a trick here. We have to read two registers to have the @@ -451,18 +440,21 @@ static int lm90_read16(struct i2c_client *client, u8 regh, u8 regl, u16 *value) * we have to read the low byte again, and now we believe we have a * correct reading. */ - if ((err = lm90_read_reg(client, regh, &oldh)) - || (err = lm90_read_reg(client, regl, &l)) - || (err = lm90_read_reg(client, regh, &newh))) - return err; + oldh = lm90_read_reg(client, regh); + if (oldh < 0) + return oldh; + l = lm90_read_reg(client, regl); + if (l < 0) + return l; + newh = lm90_read_reg(client, regh); + if (newh < 0) + return newh; if (oldh != newh) { - err = lm90_read_reg(client, regl, &l); - if (err) - return err; + l = lm90_read_reg(client, regl); + if (l < 0) + return l; } - *value = (newh << 8) | l; - - return 0; + return (newh << 8) | l; } /* @@ -473,20 +465,23 @@ static int lm90_read16(struct i2c_client *client, u8 regh, u8 regl, u16 *value) * various registers have different meanings as a result of selecting a * non-default remote channel. */ -static inline void lm90_select_remote_channel(struct i2c_client *client, - struct lm90_data *data, - int channel) +static inline int lm90_select_remote_channel(struct i2c_client *client, + struct lm90_data *data, + int channel) { - u8 config; + int config; if (data->kind == max6696) { - lm90_read_reg(client, LM90_REG_R_CONFIG1, &config); + config = lm90_read_reg(client, LM90_REG_R_CONFIG1); + if (config < 0) + return config; config &= ~0x08; if (channel) config |= 0x08; i2c_smbus_write_byte_data(client, LM90_REG_W_CONFIG1, config); } + return 0; } /* @@ -513,118 +508,204 @@ static void lm90_set_convrate(struct i2c_client *client, struct lm90_data *data, data->update_interval = DIV_ROUND_CLOSEST(update_interval, 64); } +static int lm90_update_limits(struct device *dev) +{ + struct lm90_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + int val; + + val = lm90_read_reg(client, LM90_REG_R_LOCAL_CRIT); + if (val < 0) + return val; + data->temp8[LOCAL_CRIT] = val; + + val = lm90_read_reg(client, LM90_REG_R_REMOTE_CRIT); + if (val < 0) + return val; + data->temp8[REMOTE_CRIT] = val; + + val = lm90_read_reg(client, LM90_REG_R_TCRIT_HYST); + if (val < 0) + return val; + data->temp_hyst = val; + + lm90_read_reg(client, LM90_REG_R_REMOTE_LOWH); + if (val < 0) + return val; + data->temp11[REMOTE_LOW] = val << 8; + + if (data->flags & LM90_HAVE_REM_LIMIT_EXT) { + val = lm90_read_reg(client, LM90_REG_R_REMOTE_LOWL); + if (val < 0) + return val; + data->temp11[REMOTE_LOW] |= val; + } + + val = lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHH); + if (val < 0) + return val; + data->temp11[REMOTE_HIGH] = val << 8; + + if (data->flags & LM90_HAVE_REM_LIMIT_EXT) { + val = lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHL); + if (val < 0) + return val; + data->temp11[REMOTE_HIGH] |= val; + } + + if (data->flags & LM90_HAVE_OFFSET) { + val = lm90_read16(client, LM90_REG_R_REMOTE_OFFSH, + LM90_REG_R_REMOTE_OFFSL); + if (val < 0) + return val; + data->temp11[REMOTE_OFFSET] = val; + } + + if (data->flags & LM90_HAVE_EMERGENCY) { + val = lm90_read_reg(client, MAX6659_REG_R_LOCAL_EMERG); + if (val < 0) + return val; + data->temp8[LOCAL_EMERG] = val; + + val = lm90_read_reg(client, MAX6659_REG_R_REMOTE_EMERG); + if (val < 0) + return val; + data->temp8[REMOTE_EMERG] = val; + } + + if (data->kind == max6696) { + val = lm90_select_remote_channel(client, data, 1); + if (val < 0) + return val; + + val = lm90_read_reg(client, LM90_REG_R_REMOTE_CRIT); + if (val < 0) + return val; + data->temp8[REMOTE2_CRIT] = val; + + val = lm90_read_reg(client, MAX6659_REG_R_REMOTE_EMERG); + if (val < 0) + return val; + data->temp8[REMOTE2_EMERG] = val; + + val = lm90_read_reg(client, LM90_REG_R_REMOTE_LOWH); + if (val < 0) + return val; + data->temp11[REMOTE2_LOW] = val << 8; + + val = lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHH); + if (val < 0) + return val; + data->temp11[REMOTE2_HIGH] = val << 8; + + lm90_select_remote_channel(client, data, 0); + } + + return 0; +} + static struct lm90_data *lm90_update_device(struct device *dev) { struct lm90_data *data = dev_get_drvdata(dev); struct i2c_client *client = data->client; unsigned long next_update; + int val = 0; mutex_lock(&data->update_lock); + if (!data->valid) { + val = lm90_update_limits(dev); + if (val < 0) + goto error; + } + next_update = data->last_updated + msecs_to_jiffies(data->update_interval); if (time_after(jiffies, next_update) || !data->valid) { - u8 h, l; - u8 alarms; - dev_dbg(&client->dev, "Updating lm90 data.\n"); - lm90_read_reg(client, LM90_REG_R_LOCAL_LOW, - &data->temp8[LOCAL_LOW]); - lm90_read_reg(client, LM90_REG_R_LOCAL_HIGH, - &data->temp8[LOCAL_HIGH]); - lm90_read_reg(client, LM90_REG_R_LOCAL_CRIT, - &data->temp8[LOCAL_CRIT]); - lm90_read_reg(client, LM90_REG_R_REMOTE_CRIT, - &data->temp8[REMOTE_CRIT]); - lm90_read_reg(client, LM90_REG_R_TCRIT_HYST, &data->temp_hyst); + + data->valid = false; + + val = lm90_read_reg(client, LM90_REG_R_LOCAL_LOW); + if (val < 0) + goto error; + data->temp8[LOCAL_LOW] = val; + + val = lm90_read_reg(client, LM90_REG_R_LOCAL_HIGH); + if (val < 0) + goto error; + data->temp8[LOCAL_HIGH] = val; if (data->reg_local_ext) { - lm90_read16(client, LM90_REG_R_LOCAL_TEMP, - data->reg_local_ext, - &data->temp11[LOCAL_TEMP]); + val = lm90_read16(client, LM90_REG_R_LOCAL_TEMP, + data->reg_local_ext); + if (val < 0) + goto error; + data->temp11[LOCAL_TEMP] = val; } else { - if (lm90_read_reg(client, LM90_REG_R_LOCAL_TEMP, - &h) == 0) - data->temp11[LOCAL_TEMP] = h << 8; - } - lm90_read16(client, LM90_REG_R_REMOTE_TEMPH, - LM90_REG_R_REMOTE_TEMPL, - &data->temp11[REMOTE_TEMP]); - - if (lm90_read_reg(client, LM90_REG_R_REMOTE_LOWH, &h) == 0) { - data->temp11[REMOTE_LOW] = h << 8; - if ((data->flags & LM90_HAVE_REM_LIMIT_EXT) - && lm90_read_reg(client, LM90_REG_R_REMOTE_LOWL, - &l) == 0) - data->temp11[REMOTE_LOW] |= l; - } - if (lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHH, &h) == 0) { - data->temp11[REMOTE_HIGH] = h << 8; - if ((data->flags & LM90_HAVE_REM_LIMIT_EXT) - && lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHL, - &l) == 0) - data->temp11[REMOTE_HIGH] |= l; + val = lm90_read_reg(client, LM90_REG_R_LOCAL_TEMP); + if (val < 0) + goto error; + data->temp11[LOCAL_TEMP] = val << 8; } + val = lm90_read16(client, LM90_REG_R_REMOTE_TEMPH, + LM90_REG_R_REMOTE_TEMPL); + if (val < 0) + goto error; + data->temp11[REMOTE_TEMP] = val; - if (data->flags & LM90_HAVE_OFFSET) { - if (lm90_read_reg(client, LM90_REG_R_REMOTE_OFFSH, - &h) == 0 - && lm90_read_reg(client, LM90_REG_R_REMOTE_OFFSL, - &l) == 0) - data->temp11[REMOTE_OFFSET] = (h << 8) | l; - } - if (data->flags & LM90_HAVE_EMERGENCY) { - lm90_read_reg(client, MAX6659_REG_R_LOCAL_EMERG, - &data->temp8[LOCAL_EMERG]); - lm90_read_reg(client, MAX6659_REG_R_REMOTE_EMERG, - &data->temp8[REMOTE_EMERG]); - } - lm90_read_reg(client, LM90_REG_R_STATUS, &alarms); - data->alarms = alarms; /* save as 16 bit value */ + val = lm90_read_reg(client, LM90_REG_R_STATUS); + if (val < 0) + goto error; + data->alarms = val; /* lower 8 bit of alarms */ if (data->kind == max6696) { - lm90_select_remote_channel(client, data, 1); - lm90_read_reg(client, LM90_REG_R_REMOTE_CRIT, - &data->temp8[REMOTE2_CRIT]); - lm90_read_reg(client, MAX6659_REG_R_REMOTE_EMERG, - &data->temp8[REMOTE2_EMERG]); - lm90_read16(client, LM90_REG_R_REMOTE_TEMPH, - LM90_REG_R_REMOTE_TEMPL, - &data->temp11[REMOTE2_TEMP]); - if (!lm90_read_reg(client, LM90_REG_R_REMOTE_LOWH, &h)) - data->temp11[REMOTE2_LOW] = h << 8; - if (!lm90_read_reg(client, LM90_REG_R_REMOTE_HIGHH, &h)) - data->temp11[REMOTE2_HIGH] = h << 8; + val = lm90_select_remote_channel(client, data, 1); + if (val < 0) + goto error; + + val = lm90_read16(client, LM90_REG_R_REMOTE_TEMPH, + LM90_REG_R_REMOTE_TEMPL); + if (val < 0) + goto error; + data->temp11[REMOTE2_TEMP] = val; + lm90_select_remote_channel(client, data, 0); - if (!lm90_read_reg(client, MAX6696_REG_R_STATUS2, - &alarms)) - data->alarms |= alarms << 8; + val = lm90_read_reg(client, MAX6696_REG_R_STATUS2); + if (val < 0) + goto error; + data->alarms |= val << 8; } /* * Re-enable ALERT# output if it was originally enabled and * relevant alarms are all clear */ - if ((data->config_orig & 0x80) == 0 - && (data->alarms & data->alert_alarms) == 0) { - u8 config; + if (!(data->config_orig & 0x80) && + !(data->alarms & data->alert_alarms)) { + val = lm90_read_reg(client, LM90_REG_R_CONFIG1); + if (val < 0) + goto error; - lm90_read_reg(client, LM90_REG_R_CONFIG1, &config); - if (config & 0x80) { + if (val & 0x80) { dev_dbg(&client->dev, "Re-enabling ALERT#\n"); i2c_smbus_write_byte_data(client, LM90_REG_W_CONFIG1, - config & ~0x80); + val & ~0x80); } } data->last_updated = jiffies; - data->valid = 1; + data->valid = true; } +error: mutex_unlock(&data->update_lock); + if (val < 0) + return ERR_PTR(val); + return data; } @@ -709,16 +790,14 @@ static inline int temp_from_u8_adt7461(struct lm90_data *data, u8 val) { if (data->flags & LM90_FLAG_ADT7461_EXT) return (val - 64) * 1000; - else - return temp_from_s8(val); + return temp_from_s8(val); } static inline int temp_from_u16_adt7461(struct lm90_data *data, u16 val) { if (data->flags & LM90_FLAG_ADT7461_EXT) return (val - 0x4000) / 64 * 250; - else - return temp_from_s16(val); + return temp_from_s16(val); } static u8 temp_to_u8_adt7461(struct lm90_data *data, long val) @@ -729,13 +808,12 @@ static u8 temp_to_u8_adt7461(struct lm90_data *data, long val) if (val >= 191000) return 0xFF; return (val + 500 + 64000) / 1000; - } else { - if (val <= 0) - return 0; - if (val >= 127000) - return 127; - return (val + 500) / 1000; } + if (val <= 0) + return 0; + if (val >= 127000) + return 127; + return (val + 500) / 1000; } static u16 temp_to_u16_adt7461(struct lm90_data *data, long val) @@ -746,13 +824,12 @@ static u16 temp_to_u16_adt7461(struct lm90_data *data, long val) if (val >= 191750) return 0xFFC0; return (val + 64000 + 125) / 250 * 64; - } else { - if (val <= 0) - return 0; - if (val >= 127750) - return 0x7FC0; - return (val + 125) / 250 * 64; } + if (val <= 0) + return 0; + if (val >= 127750) + return 0x7FC0; + return (val + 125) / 250 * 64; } /* @@ -766,6 +843,9 @@ static ssize_t show_temp8(struct device *dev, struct device_attribute *devattr, struct lm90_data *data = lm90_update_device(dev); int temp; + if (IS_ERR(data)) + return PTR_ERR(data); + if (data->kind == adt7461 || data->kind == tmp451) temp = temp_from_u8_adt7461(data, data->temp8[attr->index]); else if (data->kind == max6646) @@ -832,6 +912,9 @@ static ssize_t show_temp11(struct device *dev, struct device_attribute *devattr, struct lm90_data *data = lm90_update_device(dev); int temp; + if (IS_ERR(data)) + return PTR_ERR(data); + if (data->kind == adt7461 || data->kind == tmp451) temp = temp_from_u16_adt7461(data, data->temp11[attr->index]); else if (data->kind == max6646) @@ -907,6 +990,9 @@ static ssize_t show_temphyst(struct device *dev, struct lm90_data *data = lm90_update_device(dev); int temp; + if (IS_ERR(data)) + return PTR_ERR(data); + if (data->kind == adt7461 || data->kind == tmp451) temp = temp_from_u8_adt7461(data, data->temp8[attr->index]); else if (data->kind == max6646) @@ -953,6 +1039,10 @@ static ssize_t show_alarms(struct device *dev, struct device_attribute *dummy, char *buf) { struct lm90_data *data = lm90_update_device(dev); + + if (IS_ERR(data)) + return PTR_ERR(data); + return sprintf(buf, "%d\n", data->alarms); } @@ -963,6 +1053,9 @@ static ssize_t show_alarm(struct device *dev, struct device_attribute struct lm90_data *data = lm90_update_device(dev); int bitnr = attr->index; + if (IS_ERR(data)) + return PTR_ERR(data); + return sprintf(buf, "%d\n", (data->alarms >> bitnr) & 1); } @@ -1404,8 +1497,11 @@ static int lm90_detect(struct i2c_client *client, return 0; } -static void lm90_restore_conf(struct i2c_client *client, struct lm90_data *data) +static void lm90_restore_conf(void *_data) { + struct lm90_data *data = _data; + struct i2c_client *client = data->client; + /* Restore initial configuration */ i2c_smbus_write_byte_data(client, LM90_REG_W_CONVRATE, data->convrate_orig); @@ -1413,24 +1509,22 @@ static void lm90_restore_conf(struct i2c_client *client, struct lm90_data *data) data->config_orig); } -static void lm90_init_client(struct i2c_client *client, struct lm90_data *data) +static int lm90_init_client(struct i2c_client *client, struct lm90_data *data) { - u8 config, convrate; + int config, convrate; - if (lm90_read_reg(client, LM90_REG_R_CONVRATE, &convrate) < 0) { - dev_warn(&client->dev, "Failed to read convrate register!\n"); - convrate = LM90_DEF_CONVRATE_RVAL; - } + convrate = lm90_read_reg(client, LM90_REG_R_CONVRATE); + if (convrate < 0) + return convrate; data->convrate_orig = convrate; /* * Start the conversions. */ lm90_set_convrate(client, data, 500); /* 500ms; 2Hz conversion rate */ - if (lm90_read_reg(client, LM90_REG_R_CONFIG1, &config) < 0) { - dev_warn(&client->dev, "Initialization failed!\n"); - return; - } + config = lm90_read_reg(client, LM90_REG_R_CONFIG1); + if (config < 0) + return config; data->config_orig = config; /* Check Temperature Range Select */ @@ -1456,17 +1550,26 @@ static void lm90_init_client(struct i2c_client *client, struct lm90_data *data) config &= 0xBF; /* run */ if (config != data->config_orig) /* Only write if changed */ i2c_smbus_write_byte_data(client, LM90_REG_W_CONFIG1, config); + + devm_add_action(&client->dev, lm90_restore_conf, data); + + return 0; } static bool lm90_is_tripped(struct i2c_client *client, u16 *status) { struct lm90_data *data = i2c_get_clientdata(client); - u8 st, st2 = 0; + int st, st2 = 0; - lm90_read_reg(client, LM90_REG_R_STATUS, &st); + st = lm90_read_reg(client, LM90_REG_R_STATUS); + if (st < 0) + return false; - if (data->kind == max6696) - lm90_read_reg(client, MAX6696_REG_R_STATUS2, &st2); + if (data->kind == max6696) { + st2 = lm90_read_reg(client, MAX6696_REG_R_STATUS2); + if (st2 < 0) + return false; + } *status = st | (st2 << 8); @@ -1506,6 +1609,16 @@ static irqreturn_t lm90_irq_thread(int irq, void *dev_id) return IRQ_NONE; } +static void lm90_remove_pec(void *dev) +{ + device_remove_file(dev, &dev_attr_pec); +} + +static void lm90_regulator_disable(void *regulator) +{ + regulator_disable(regulator); +} + static int lm90_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -1513,6 +1626,7 @@ static int lm90_probe(struct i2c_client *client, struct i2c_adapter *adapter = to_i2c_adapter(dev->parent); struct lm90_data *data; struct regulator *regulator; + struct device *hwmon_dev; int groups = 0; int err; @@ -1526,6 +1640,8 @@ static int lm90_probe(struct i2c_client *client, return err; } + devm_add_action(dev, lm90_regulator_disable, regulator); + data = devm_kzalloc(dev, sizeof(struct lm90_data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -1534,8 +1650,6 @@ static int lm90_probe(struct i2c_client *client, i2c_set_clientdata(client, data); mutex_init(&data->update_lock); - data->regulator = regulator; - /* Set the device type */ data->kind = id->driver_data; if (data->kind == adm1032) { @@ -1557,7 +1671,11 @@ static int lm90_probe(struct i2c_client *client, data->max_convrate = lm90_params[data->kind].max_convrate; /* Initialize the LM90 chip */ - lm90_init_client(client, data); + err = lm90_init_client(client, data); + if (err < 0) { + dev_err(dev, "Failed to initialize device\n"); + return err; + } /* Register sysfs hooks */ data->groups[groups++] = &lm90_group; @@ -1577,15 +1695,14 @@ static int lm90_probe(struct i2c_client *client, if (client->flags & I2C_CLIENT_PEC) { err = device_create_file(dev, &dev_attr_pec); if (err) - goto exit_restore; + return err; + devm_add_action(dev, lm90_remove_pec, dev); } - data->hwmon_dev = hwmon_device_register_with_groups(dev, client->name, - data, data->groups); - if (IS_ERR(data->hwmon_dev)) { - err = PTR_ERR(data->hwmon_dev); - goto exit_remove_pec; - } + hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name, + data, data->groups); + if (IS_ERR(hwmon_dev)) + return PTR_ERR(hwmon_dev); if (client->irq) { dev_dbg(dev, "IRQ: %d\n", client->irq); @@ -1595,32 +1712,10 @@ static int lm90_probe(struct i2c_client *client, "lm90", client); if (err < 0) { dev_err(dev, "cannot request IRQ %d\n", client->irq); - goto exit_unregister; + return err; } } - return 0; - -exit_unregister: - hwmon_device_unregister(data->hwmon_dev); -exit_remove_pec: - device_remove_file(dev, &dev_attr_pec); -exit_restore: - lm90_restore_conf(client, data); - regulator_disable(data->regulator); - - return err; -} - -static int lm90_remove(struct i2c_client *client) -{ - struct lm90_data *data = i2c_get_clientdata(client); - - hwmon_device_unregister(data->hwmon_dev); - device_remove_file(&client->dev, &dev_attr_pec); - lm90_restore_conf(client, data); - regulator_disable(data->regulator); - return 0; } @@ -1636,13 +1731,16 @@ static void lm90_alert(struct i2c_client *client, unsigned int flag) */ struct lm90_data *data = i2c_get_clientdata(client); - if ((data->flags & LM90_HAVE_BROKEN_ALERT) - && (alarms & data->alert_alarms)) { - u8 config; + if ((data->flags & LM90_HAVE_BROKEN_ALERT) && + (alarms & data->alert_alarms)) { + int config; + dev_dbg(&client->dev, "Disabling ALERT#\n"); - lm90_read_reg(client, LM90_REG_R_CONFIG1, &config); - i2c_smbus_write_byte_data(client, LM90_REG_W_CONFIG1, - config | 0x80); + config = lm90_read_reg(client, LM90_REG_R_CONFIG1); + if (config >= 0) + i2c_smbus_write_byte_data(client, + LM90_REG_W_CONFIG1, + config | 0x80); } } else { dev_info(&client->dev, "Everything OK\n"); @@ -1655,7 +1753,6 @@ static struct i2c_driver lm90_driver = { .name = "lm90", }, .probe = lm90_probe, - .remove = lm90_remove, .alert = lm90_alert, .id_table = lm90_id, .detect = lm90_detect, diff --git a/drivers/hwmon/sht3x.c b/drivers/hwmon/sht3x.c new file mode 100644 index 000000000000..b73a48832732 --- /dev/null +++ b/drivers/hwmon/sht3x.c @@ -0,0 +1,775 @@ +/* Sensirion SHT3x-DIS humidity and temperature sensor driver. + * The SHT3x comes in many different versions, this driver is for the + * I2C version only. + * + * Copyright (C) 2016 Sensirion AG, Switzerland + * Author: David Frey + * Author: Pascal Sachs + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* commands (high precision mode) */ +static const unsigned char sht3x_cmd_measure_blocking_hpm[] = { 0x2c, 0x06 }; +static const unsigned char sht3x_cmd_measure_nonblocking_hpm[] = { 0x24, 0x00 }; + +/* commands (low power mode) */ +static const unsigned char sht3x_cmd_measure_blocking_lpm[] = { 0x2c, 0x10 }; +static const unsigned char sht3x_cmd_measure_nonblocking_lpm[] = { 0x24, 0x16 }; + +/* commands for periodic mode */ +static const unsigned char sht3x_cmd_measure_periodic_mode[] = { 0xe0, 0x00 }; +static const unsigned char sht3x_cmd_break[] = { 0x30, 0x93 }; + +/* commands for heater control */ +static const unsigned char sht3x_cmd_heater_on[] = { 0x30, 0x6d }; +static const unsigned char sht3x_cmd_heater_off[] = { 0x30, 0x66 }; + +/* other commands */ +static const unsigned char sht3x_cmd_read_status_reg[] = { 0xf3, 0x2d }; +static const unsigned char sht3x_cmd_clear_status_reg[] = { 0x30, 0x41 }; + +/* delays for non-blocking i2c commands, both in us */ +#define SHT3X_NONBLOCKING_WAIT_TIME_HPM 15000 +#define SHT3X_NONBLOCKING_WAIT_TIME_LPM 4000 + +#define SHT3X_WORD_LEN 2 +#define SHT3X_CMD_LENGTH 2 +#define SHT3X_CRC8_LEN 1 +#define SHT3X_RESPONSE_LENGTH 6 +#define SHT3X_CRC8_POLYNOMIAL 0x31 +#define SHT3X_CRC8_INIT 0xFF +#define SHT3X_MIN_TEMPERATURE -45000 +#define SHT3X_MAX_TEMPERATURE 130000 +#define SHT3X_MIN_HUMIDITY 0 +#define SHT3X_MAX_HUMIDITY 100000 + +enum sht3x_chips { + sht3x, + sts3x, +}; + +enum sht3x_limits { + limit_max = 0, + limit_max_hyst, + limit_min, + limit_min_hyst, +}; + +DECLARE_CRC8_TABLE(sht3x_crc8_table); + +/* periodic measure commands (high precision mode) */ +static const char periodic_measure_commands_hpm[][SHT3X_CMD_LENGTH] = { + /* 0.5 measurements per second */ + {0x20, 0x32}, + /* 1 measurements per second */ + {0x21, 0x30}, + /* 2 measurements per second */ + {0x22, 0x36}, + /* 4 measurements per second */ + {0x23, 0x34}, + /* 10 measurements per second */ + {0x27, 0x37}, +}; + +/* periodic measure commands (low power mode) */ +static const char periodic_measure_commands_lpm[][SHT3X_CMD_LENGTH] = { + /* 0.5 measurements per second */ + {0x20, 0x2f}, + /* 1 measurements per second */ + {0x21, 0x2d}, + /* 2 measurements per second */ + {0x22, 0x2b}, + /* 4 measurements per second */ + {0x23, 0x29}, + /* 10 measurements per second */ + {0x27, 0x2a}, +}; + +struct sht3x_limit_commands { + const char read_command[SHT3X_CMD_LENGTH]; + const char write_command[SHT3X_CMD_LENGTH]; +}; + +static const struct sht3x_limit_commands limit_commands[] = { + /* temp1_max, humidity1_max */ + [limit_max] = { {0xe1, 0x1f}, {0x61, 0x1d} }, + /* temp_1_max_hyst, humidity1_max_hyst */ + [limit_max_hyst] = { {0xe1, 0x14}, {0x61, 0x16} }, + /* temp1_min, humidity1_min */ + [limit_min] = { {0xe1, 0x02}, {0x61, 0x00} }, + /* temp_1_min_hyst, humidity1_min_hyst */ + [limit_min_hyst] = { {0xe1, 0x09}, {0x61, 0x0B} }, +}; + +#define SHT3X_NUM_LIMIT_CMD ARRAY_SIZE(limit_commands) + +static const u16 mode_to_update_interval[] = { + 0, + 2000, + 1000, + 500, + 250, + 100, +}; + +struct sht3x_data { + struct i2c_client *client; + struct mutex i2c_lock; /* lock for sending i2c commands */ + struct mutex data_lock; /* lock for updating driver data */ + + u8 mode; + const unsigned char *command; + u32 wait_time; /* in us*/ + unsigned long last_update; /* last update in periodic mode*/ + + struct sht3x_platform_data setup; + + /* + * cached values for temperature and humidity and limits + * the limits arrays have the following order: + * max, max_hyst, min, min_hyst + */ + int temperature; + int temperature_limits[SHT3X_NUM_LIMIT_CMD]; + u32 humidity; + u32 humidity_limits[SHT3X_NUM_LIMIT_CMD]; +}; + +static u8 get_mode_from_update_interval(u16 value) +{ + size_t index; + u8 number_of_modes = ARRAY_SIZE(mode_to_update_interval); + + if (value == 0) + return 0; + + /* find next faster update interval */ + for (index = 1; index < number_of_modes; index++) { + if (mode_to_update_interval[index] <= value) + return index; + } + + return number_of_modes - 1; +} + +static int sht3x_read_from_command(struct i2c_client *client, + struct sht3x_data *data, + const char *command, + char *buf, int length, u32 wait_time) +{ + int ret; + + mutex_lock(&data->i2c_lock); + ret = i2c_master_send(client, command, SHT3X_CMD_LENGTH); + + if (ret != SHT3X_CMD_LENGTH) { + ret = ret < 0 ? ret : -EIO; + goto out; + } + + if (wait_time) + usleep_range(wait_time, wait_time + 1000); + + ret = i2c_master_recv(client, buf, length); + if (ret != length) { + ret = ret < 0 ? ret : -EIO; + goto out; + } + + ret = 0; +out: + mutex_unlock(&data->i2c_lock); + return ret; +} + +static int sht3x_extract_temperature(u16 raw) +{ + /* + * From datasheet: + * T = -45 + 175 * ST / 2^16 + * Adapted for integer fixed point (3 digit) arithmetic. + */ + return ((21875 * (int)raw) >> 13) - 45000; +} + +static u32 sht3x_extract_humidity(u16 raw) +{ + /* + * From datasheet: + * RH = 100 * SRH / 2^16 + * Adapted for integer fixed point (3 digit) arithmetic. + */ + return (12500 * (u32)raw) >> 13; +} + +static struct sht3x_data *sht3x_update_client(struct device *dev) +{ + struct sht3x_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + u16 interval_ms = mode_to_update_interval[data->mode]; + unsigned long interval_jiffies = msecs_to_jiffies(interval_ms); + unsigned char buf[SHT3X_RESPONSE_LENGTH]; + u16 val; + int ret = 0; + + mutex_lock(&data->data_lock); + /* + * Only update cached readings once per update interval in periodic + * mode. In single shot mode the sensor measures values on demand, so + * every time the sysfs interface is called, a measurement is triggered. + * In periodic mode however, the measurement process is handled + * internally by the sensor and reading out sensor values only makes + * sense if a new reading is available. + */ + if (time_after(jiffies, data->last_update + interval_jiffies)) { + ret = sht3x_read_from_command(client, data, data->command, buf, + sizeof(buf), data->wait_time); + if (ret) + goto out; + + val = be16_to_cpup((__be16 *)buf); + data->temperature = sht3x_extract_temperature(val); + val = be16_to_cpup((__be16 *)(buf + 3)); + data->humidity = sht3x_extract_humidity(val); + data->last_update = jiffies; + } + +out: + mutex_unlock(&data->data_lock); + if (ret) + return ERR_PTR(ret); + + return data; +} + +/* sysfs attributes */ +static ssize_t temp1_input_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sht3x_data *data = sht3x_update_client(dev); + + if (IS_ERR(data)) + return PTR_ERR(data); + + return sprintf(buf, "%d\n", data->temperature); +} + +static ssize_t humidity1_input_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sht3x_data *data = sht3x_update_client(dev); + + if (IS_ERR(data)) + return PTR_ERR(data); + + return sprintf(buf, "%u\n", data->humidity); +} + +/* + * limits_update must only be called from probe or with data_lock held + */ +static int limits_update(struct sht3x_data *data) +{ + int ret; + u8 index; + int temperature; + u32 humidity; + u16 raw; + char buffer[SHT3X_RESPONSE_LENGTH]; + const struct sht3x_limit_commands *commands; + struct i2c_client *client = data->client; + + for (index = 0; index < SHT3X_NUM_LIMIT_CMD; index++) { + commands = &limit_commands[index]; + ret = sht3x_read_from_command(client, data, + commands->read_command, buffer, + SHT3X_RESPONSE_LENGTH, 0); + + if (ret) + return ret; + + raw = be16_to_cpup((__be16 *)buffer); + temperature = sht3x_extract_temperature((raw & 0x01ff) << 7); + humidity = sht3x_extract_humidity(raw & 0xfe00); + data->temperature_limits[index] = temperature; + data->humidity_limits[index] = humidity; + } + + return ret; +} + +static ssize_t temp1_limit_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct sht3x_data *data = dev_get_drvdata(dev); + u8 index = to_sensor_dev_attr(attr)->index; + int temperature_limit = data->temperature_limits[index]; + + return scnprintf(buf, PAGE_SIZE, "%d\n", temperature_limit); +} + +static ssize_t humidity1_limit_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct sht3x_data *data = dev_get_drvdata(dev); + u8 index = to_sensor_dev_attr(attr)->index; + u32 humidity_limit = data->humidity_limits[index]; + + return scnprintf(buf, PAGE_SIZE, "%u\n", humidity_limit); +} + +/* + * limit_store must only be called with data_lock held + */ +static size_t limit_store(struct device *dev, + size_t count, + u8 index, + int temperature, + u32 humidity) +{ + char buffer[SHT3X_CMD_LENGTH + SHT3X_WORD_LEN + SHT3X_CRC8_LEN]; + char *position = buffer; + int ret; + u16 raw; + struct sht3x_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + const struct sht3x_limit_commands *commands; + + commands = &limit_commands[index]; + + memcpy(position, commands->write_command, SHT3X_CMD_LENGTH); + position += SHT3X_CMD_LENGTH; + /* + * ST = (T + 45) / 175 * 2^16 + * SRH = RH / 100 * 2^16 + * adapted for fixed point arithmetic and packed the same as + * in limit_show() + */ + raw = ((u32)(temperature + 45000) * 24543) >> (16 + 7); + raw |= ((humidity * 42950) >> 16) & 0xfe00; + + *((__be16 *)position) = cpu_to_be16(raw); + position += SHT3X_WORD_LEN; + *position = crc8(sht3x_crc8_table, + position - SHT3X_WORD_LEN, + SHT3X_WORD_LEN, + SHT3X_CRC8_INIT); + + mutex_lock(&data->i2c_lock); + ret = i2c_master_send(client, buffer, sizeof(buffer)); + mutex_unlock(&data->i2c_lock); + + if (ret != sizeof(buffer)) + return ret < 0 ? ret : -EIO; + + data->temperature_limits[index] = temperature; + data->humidity_limits[index] = humidity; + return count; +} + +static ssize_t temp1_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + int temperature; + int ret; + struct sht3x_data *data = dev_get_drvdata(dev); + u8 index = to_sensor_dev_attr(attr)->index; + + ret = kstrtoint(buf, 0, &temperature); + if (ret) + return ret; + + temperature = clamp_val(temperature, SHT3X_MIN_TEMPERATURE, + SHT3X_MAX_TEMPERATURE); + mutex_lock(&data->data_lock); + ret = limit_store(dev, count, index, temperature, + data->humidity_limits[index]); + mutex_unlock(&data->data_lock); + + return ret; +} + +static ssize_t humidity1_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + u32 humidity; + int ret; + struct sht3x_data *data = dev_get_drvdata(dev); + u8 index = to_sensor_dev_attr(attr)->index; + + ret = kstrtou32(buf, 0, &humidity); + if (ret) + return ret; + + humidity = clamp_val(humidity, SHT3X_MIN_HUMIDITY, SHT3X_MAX_HUMIDITY); + mutex_lock(&data->data_lock); + ret = limit_store(dev, count, index, data->temperature_limits[index], + humidity); + mutex_unlock(&data->data_lock); + + return ret; +} + +static void sht3x_select_command(struct sht3x_data *data) +{ + /* + * In blocking mode (clock stretching mode) the I2C bus + * is blocked for other traffic, thus the call to i2c_master_recv() + * will wait until the data is ready. For non blocking mode, we + * have to wait ourselves. + */ + if (data->mode > 0) { + data->command = sht3x_cmd_measure_periodic_mode; + data->wait_time = 0; + } else if (data->setup.blocking_io) { + data->command = data->setup.high_precision ? + sht3x_cmd_measure_blocking_hpm : + sht3x_cmd_measure_blocking_lpm; + data->wait_time = 0; + } else { + if (data->setup.high_precision) { + data->command = sht3x_cmd_measure_nonblocking_hpm; + data->wait_time = SHT3X_NONBLOCKING_WAIT_TIME_HPM; + } else { + data->command = sht3x_cmd_measure_nonblocking_lpm; + data->wait_time = SHT3X_NONBLOCKING_WAIT_TIME_LPM; + } + } +} + +static int status_register_read(struct device *dev, + struct device_attribute *attr, + char *buffer, int length) +{ + int ret; + struct sht3x_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + + ret = sht3x_read_from_command(client, data, sht3x_cmd_read_status_reg, + buffer, length, 0); + + return ret; +} + +static ssize_t temp1_alarm_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + char buffer[SHT3X_WORD_LEN + SHT3X_CRC8_LEN]; + int ret; + + ret = status_register_read(dev, attr, buffer, + SHT3X_WORD_LEN + SHT3X_CRC8_LEN); + if (ret) + return ret; + + return scnprintf(buf, PAGE_SIZE, "%d\n", !!(buffer[0] & 0x04)); +} + +static ssize_t humidity1_alarm_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + char buffer[SHT3X_WORD_LEN + SHT3X_CRC8_LEN]; + int ret; + + ret = status_register_read(dev, attr, buffer, + SHT3X_WORD_LEN + SHT3X_CRC8_LEN); + if (ret) + return ret; + + return scnprintf(buf, PAGE_SIZE, "%d\n", !!(buffer[0] & 0x08)); +} + +static ssize_t heater_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + char buffer[SHT3X_WORD_LEN + SHT3X_CRC8_LEN]; + int ret; + + ret = status_register_read(dev, attr, buffer, + SHT3X_WORD_LEN + SHT3X_CRC8_LEN); + if (ret) + return ret; + + return scnprintf(buf, PAGE_SIZE, "%d\n", !!(buffer[0] & 0x20)); +} + +static ssize_t heater_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct sht3x_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + int ret; + bool status; + + ret = kstrtobool(buf, &status); + if (ret) + return ret; + + mutex_lock(&data->i2c_lock); + + if (status) + ret = i2c_master_send(client, (char *)&sht3x_cmd_heater_on, + SHT3X_CMD_LENGTH); + else + ret = i2c_master_send(client, (char *)&sht3x_cmd_heater_off, + SHT3X_CMD_LENGTH); + + mutex_unlock(&data->i2c_lock); + + return ret; +} + +static ssize_t update_interval_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct sht3x_data *data = dev_get_drvdata(dev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", + mode_to_update_interval[data->mode]); +} + +static ssize_t update_interval_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + u16 update_interval; + u8 mode; + int ret; + const char *command; + struct sht3x_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + + ret = kstrtou16(buf, 0, &update_interval); + if (ret) + return ret; + + mode = get_mode_from_update_interval(update_interval); + + mutex_lock(&data->data_lock); + /* mode did not change */ + if (mode == data->mode) { + mutex_unlock(&data->data_lock); + return count; + } + + mutex_lock(&data->i2c_lock); + /* + * Abort periodic measure mode. + * To do any changes to the configuration while in periodic mode, we + * have to send a break command to the sensor, which then falls back + * to single shot (mode = 0). + */ + if (data->mode > 0) { + ret = i2c_master_send(client, sht3x_cmd_break, + SHT3X_CMD_LENGTH); + if (ret != SHT3X_CMD_LENGTH) + goto out; + data->mode = 0; + } + + if (mode > 0) { + if (data->setup.high_precision) + command = periodic_measure_commands_hpm[mode - 1]; + else + command = periodic_measure_commands_lpm[mode - 1]; + + /* select mode */ + ret = i2c_master_send(client, command, SHT3X_CMD_LENGTH); + if (ret != SHT3X_CMD_LENGTH) + goto out; + } + + /* select mode and command */ + data->mode = mode; + sht3x_select_command(data); + +out: + mutex_unlock(&data->i2c_lock); + mutex_unlock(&data->data_lock); + if (ret != SHT3X_CMD_LENGTH) + return ret < 0 ? ret : -EIO; + + return count; +} + +static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, temp1_input_show, NULL, 0); +static SENSOR_DEVICE_ATTR(humidity1_input, S_IRUGO, humidity1_input_show, + NULL, 0); +static SENSOR_DEVICE_ATTR(temp1_max, S_IRUGO | S_IWUSR, + temp1_limit_show, temp1_limit_store, + limit_max); +static SENSOR_DEVICE_ATTR(humidity1_max, S_IRUGO | S_IWUSR, + humidity1_limit_show, humidity1_limit_store, + limit_max); +static SENSOR_DEVICE_ATTR(temp1_max_hyst, S_IRUGO | S_IWUSR, + temp1_limit_show, temp1_limit_store, + limit_max_hyst); +static SENSOR_DEVICE_ATTR(humidity1_max_hyst, S_IRUGO | S_IWUSR, + humidity1_limit_show, humidity1_limit_store, + limit_max_hyst); +static SENSOR_DEVICE_ATTR(temp1_min, S_IRUGO | S_IWUSR, + temp1_limit_show, temp1_limit_store, + limit_min); +static SENSOR_DEVICE_ATTR(humidity1_min, S_IRUGO | S_IWUSR, + humidity1_limit_show, humidity1_limit_store, + limit_min); +static SENSOR_DEVICE_ATTR(temp1_min_hyst, S_IRUGO | S_IWUSR, + temp1_limit_show, temp1_limit_store, + limit_min_hyst); +static SENSOR_DEVICE_ATTR(humidity1_min_hyst, S_IRUGO | S_IWUSR, + humidity1_limit_show, humidity1_limit_store, + limit_min_hyst); +static SENSOR_DEVICE_ATTR(temp1_alarm, S_IRUGO, temp1_alarm_show, NULL, 0); +static SENSOR_DEVICE_ATTR(humidity1_alarm, S_IRUGO, humidity1_alarm_show, + NULL, 0); +static SENSOR_DEVICE_ATTR(heater_enable, S_IRUGO | S_IWUSR, + heater_enable_show, heater_enable_store, 0); +static SENSOR_DEVICE_ATTR(update_interval, S_IRUGO | S_IWUSR, + update_interval_show, update_interval_store, 0); + +static struct attribute *sht3x_attrs[] = { + &sensor_dev_attr_temp1_input.dev_attr.attr, + &sensor_dev_attr_humidity1_input.dev_attr.attr, + &sensor_dev_attr_temp1_max.dev_attr.attr, + &sensor_dev_attr_temp1_max_hyst.dev_attr.attr, + &sensor_dev_attr_humidity1_max.dev_attr.attr, + &sensor_dev_attr_humidity1_max_hyst.dev_attr.attr, + &sensor_dev_attr_temp1_min.dev_attr.attr, + &sensor_dev_attr_temp1_min_hyst.dev_attr.attr, + &sensor_dev_attr_humidity1_min.dev_attr.attr, + &sensor_dev_attr_humidity1_min_hyst.dev_attr.attr, + &sensor_dev_attr_temp1_alarm.dev_attr.attr, + &sensor_dev_attr_humidity1_alarm.dev_attr.attr, + &sensor_dev_attr_heater_enable.dev_attr.attr, + &sensor_dev_attr_update_interval.dev_attr.attr, + NULL +}; + +static struct attribute *sts3x_attrs[] = { + &sensor_dev_attr_temp1_input.dev_attr.attr, + NULL +}; + +ATTRIBUTE_GROUPS(sht3x); +ATTRIBUTE_GROUPS(sts3x); + +static int sht3x_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + int ret; + struct sht3x_data *data; + struct device *hwmon_dev; + struct i2c_adapter *adap = client->adapter; + struct device *dev = &client->dev; + const struct attribute_group **attribute_groups; + + /* + * we require full i2c support since the sht3x uses multi-byte read and + * writes as well as multi-byte commands which are not supported by + * the smbus protocol + */ + if (!i2c_check_functionality(adap, I2C_FUNC_I2C)) + return -ENODEV; + + ret = i2c_master_send(client, sht3x_cmd_clear_status_reg, + SHT3X_CMD_LENGTH); + if (ret != SHT3X_CMD_LENGTH) + return ret < 0 ? ret : -ENODEV; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->setup.blocking_io = false; + data->setup.high_precision = true; + data->mode = 0; + data->last_update = 0; + data->client = client; + crc8_populate_msb(sht3x_crc8_table, SHT3X_CRC8_POLYNOMIAL); + + if (client->dev.platform_data) + data->setup = *(struct sht3x_platform_data *)dev->platform_data; + + sht3x_select_command(data); + + mutex_init(&data->i2c_lock); + mutex_init(&data->data_lock); + + ret = limits_update(data); + if (ret) + return ret; + + if (id->driver_data == sts3x) + attribute_groups = sts3x_groups; + else + attribute_groups = sht3x_groups; + + hwmon_dev = devm_hwmon_device_register_with_groups(dev, + client->name, + data, + attribute_groups); + + if (IS_ERR(hwmon_dev)) + dev_dbg(dev, "unable to register hwmon device\n"); + + return PTR_ERR_OR_ZERO(hwmon_dev); +} + +/* device ID table */ +static const struct i2c_device_id sht3x_ids[] = { + {"sht3x", sht3x}, + {"sts3x", sts3x}, + {} +}; + +MODULE_DEVICE_TABLE(i2c, sht3x_ids); + +static struct i2c_driver sht3x_i2c_driver = { + .driver.name = "sht3x", + .probe = sht3x_probe, + .id_table = sht3x_ids, +}; + +module_i2c_driver(sht3x_i2c_driver); + +MODULE_AUTHOR("David Frey "); +MODULE_AUTHOR("Pascal Sachs "); +MODULE_DESCRIPTION("Sensirion SHT3x humidity and temperature sensor driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c index f1e96fd7f445..a942a2574a4d 100644 --- a/drivers/hwmon/tmp102.c +++ b/drivers/hwmon/tmp102.c @@ -11,12 +11,9 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA */ +#include #include #include #include @@ -27,6 +24,7 @@ #include #include #include +#include #include #include @@ -50,14 +48,23 @@ #define TMP102_TLOW_REG 0x02 #define TMP102_THIGH_REG 0x03 +#define TMP102_CONFREG_MASK (TMP102_CONF_SD | TMP102_CONF_TM | \ + TMP102_CONF_POL | TMP102_CONF_F0 | \ + TMP102_CONF_F1 | TMP102_CONF_OS | \ + TMP102_CONF_EM | TMP102_CONF_AL | \ + TMP102_CONF_CR0 | TMP102_CONF_CR1) + +#define TMP102_CONFIG_CLEAR (TMP102_CONF_SD | TMP102_CONF_OS | \ + TMP102_CONF_CR0) +#define TMP102_CONFIG_SET (TMP102_CONF_TM | TMP102_CONF_EM | \ + TMP102_CONF_CR1) + +#define CONVERSION_TIME_MS 35 /* in milli-seconds */ + struct tmp102 { - struct i2c_client *client; - struct device *hwmon_dev; - struct mutex lock; + struct regmap *regmap; u16 config_orig; - unsigned long last_update; - int temp[3]; - bool first_time; + unsigned long ready_time; }; /* convert left adjusted 13-bit TMP102 register value to milliCelsius */ @@ -72,44 +79,22 @@ static inline u16 tmp102_mC_to_reg(int val) return (val * 128) / 1000; } -static const u8 tmp102_reg[] = { - TMP102_TEMP_REG, - TMP102_TLOW_REG, - TMP102_THIGH_REG, -}; - -static struct tmp102 *tmp102_update_device(struct device *dev) -{ - struct tmp102 *tmp102 = dev_get_drvdata(dev); - struct i2c_client *client = tmp102->client; - - mutex_lock(&tmp102->lock); - if (time_after(jiffies, tmp102->last_update + HZ / 3)) { - int i; - for (i = 0; i < ARRAY_SIZE(tmp102->temp); ++i) { - int status = i2c_smbus_read_word_swapped(client, - tmp102_reg[i]); - if (status > -1) - tmp102->temp[i] = tmp102_reg_to_mC(status); - } - tmp102->last_update = jiffies; - tmp102->first_time = false; - } - mutex_unlock(&tmp102->lock); - return tmp102; -} - static int tmp102_read_temp(void *dev, int *temp) { - struct tmp102 *tmp102 = tmp102_update_device(dev); + struct tmp102 *tmp102 = dev_get_drvdata(dev); + unsigned int reg; + int ret; - /* Is it too early even to return a conversion? */ - if (tmp102->first_time) { + if (time_before(jiffies, tmp102->ready_time)) { dev_dbg(dev, "%s: Conversion not ready yet..\n", __func__); return -EAGAIN; } - *temp = tmp102->temp[0]; + ret = regmap_read(tmp102->regmap, TMP102_TEMP_REG, ®); + if (ret < 0) + return ret; + + *temp = tmp102_reg_to_mC(reg); return 0; } @@ -119,13 +104,20 @@ static ssize_t tmp102_show_temp(struct device *dev, char *buf) { struct sensor_device_attribute *sda = to_sensor_dev_attr(attr); - struct tmp102 *tmp102 = tmp102_update_device(dev); + struct tmp102 *tmp102 = dev_get_drvdata(dev); + int regaddr = sda->index; + unsigned int reg; + int err; - /* Is it too early even to return a read? */ - if (tmp102->first_time) + if (regaddr == TMP102_TEMP_REG && + time_before(jiffies, tmp102->ready_time)) return -EAGAIN; - return sprintf(buf, "%d\n", tmp102->temp[sda->index]); + err = regmap_read(tmp102->regmap, regaddr, ®); + if (err < 0) + return err; + + return sprintf(buf, "%d\n", tmp102_reg_to_mC(reg)); } static ssize_t tmp102_set_temp(struct device *dev, @@ -134,29 +126,26 @@ static ssize_t tmp102_set_temp(struct device *dev, { struct sensor_device_attribute *sda = to_sensor_dev_attr(attr); struct tmp102 *tmp102 = dev_get_drvdata(dev); - struct i2c_client *client = tmp102->client; + int reg = sda->index; long val; - int status; + int err; if (kstrtol(buf, 10, &val) < 0) return -EINVAL; val = clamp_val(val, -256000, 255000); - mutex_lock(&tmp102->lock); - tmp102->temp[sda->index] = val; - status = i2c_smbus_write_word_swapped(client, tmp102_reg[sda->index], - tmp102_mC_to_reg(val)); - mutex_unlock(&tmp102->lock); - return status ? : count; + err = regmap_write(tmp102->regmap, reg, tmp102_mC_to_reg(val)); + return err ? : count; } -static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, tmp102_show_temp, NULL , 0); +static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, tmp102_show_temp, NULL, + TMP102_TEMP_REG); static SENSOR_DEVICE_ATTR(temp1_max_hyst, S_IWUSR | S_IRUGO, tmp102_show_temp, - tmp102_set_temp, 1); + tmp102_set_temp, TMP102_TLOW_REG); static SENSOR_DEVICE_ATTR(temp1_max, S_IWUSR | S_IRUGO, tmp102_show_temp, - tmp102_set_temp, 2); + tmp102_set_temp, TMP102_THIGH_REG); static struct attribute *tmp102_attrs[] = { &sensor_dev_attr_temp1_input.dev_attr.attr, @@ -166,20 +155,46 @@ static struct attribute *tmp102_attrs[] = { }; ATTRIBUTE_GROUPS(tmp102); -#define TMP102_CONFIG (TMP102_CONF_TM | TMP102_CONF_EM | TMP102_CONF_CR1) -#define TMP102_CONFIG_RD_ONLY (TMP102_CONF_R0 | TMP102_CONF_R1 | TMP102_CONF_AL) - static const struct thermal_zone_of_device_ops tmp102_of_thermal_ops = { .get_temp = tmp102_read_temp, }; +static void tmp102_restore_config(void *data) +{ + struct tmp102 *tmp102 = data; + + regmap_write(tmp102->regmap, TMP102_CONF_REG, tmp102->config_orig); +} + +static bool tmp102_is_writeable_reg(struct device *dev, unsigned int reg) +{ + return reg != TMP102_TEMP_REG; +} + +static bool tmp102_is_volatile_reg(struct device *dev, unsigned int reg) +{ + return reg == TMP102_TEMP_REG; +} + +static const struct regmap_config tmp102_regmap_config = { + .reg_bits = 8, + .val_bits = 16, + .max_register = TMP102_THIGH_REG, + .writeable_reg = tmp102_is_writeable_reg, + .volatile_reg = tmp102_is_volatile_reg, + .val_format_endian = REGMAP_ENDIAN_BIG, + .cache_type = REGCACHE_RBTREE, + .use_single_rw = true, +}; + static int tmp102_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct device *dev = &client->dev; struct device *hwmon_dev; struct tmp102 *tmp102; - int status; + unsigned int regval; + int err; if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WORD_DATA)) { @@ -193,73 +208,57 @@ static int tmp102_probe(struct i2c_client *client, return -ENOMEM; i2c_set_clientdata(client, tmp102); - tmp102->client = client; - status = i2c_smbus_read_word_swapped(client, TMP102_CONF_REG); - if (status < 0) { + tmp102->regmap = devm_regmap_init_i2c(client, &tmp102_regmap_config); + if (IS_ERR(tmp102->regmap)) + return PTR_ERR(tmp102->regmap); + + err = regmap_read(tmp102->regmap, TMP102_CONF_REG, ®val); + if (err < 0) { dev_err(dev, "error reading config register\n"); - return status; + return err; } - tmp102->config_orig = status; - status = i2c_smbus_write_word_swapped(client, TMP102_CONF_REG, - TMP102_CONFIG); - if (status < 0) { - dev_err(dev, "error writing config register\n"); - goto fail_restore_config; + + if ((regval & ~TMP102_CONFREG_MASK) != + (TMP102_CONF_R0 | TMP102_CONF_R1)) { + dev_err(dev, "unexpected config register value\n"); + return -ENODEV; } - status = i2c_smbus_read_word_swapped(client, TMP102_CONF_REG); - if (status < 0) { - dev_err(dev, "error reading config register\n"); - goto fail_restore_config; + + tmp102->config_orig = regval; + + devm_add_action(dev, tmp102_restore_config, tmp102); + + regval &= ~TMP102_CONFIG_CLEAR; + regval |= TMP102_CONFIG_SET; + + err = regmap_write(tmp102->regmap, TMP102_CONF_REG, regval); + if (err < 0) { + dev_err(dev, "error writing config register\n"); + return err; } - status &= ~TMP102_CONFIG_RD_ONLY; - if (status != TMP102_CONFIG) { - dev_err(dev, "config settings did not stick\n"); - status = -ENODEV; - goto fail_restore_config; + + tmp102->ready_time = jiffies; + if (tmp102->config_orig & TMP102_CONF_SD) { + /* + * Mark that we are not ready with data until the first + * conversion is complete + */ + tmp102->ready_time += msecs_to_jiffies(CONVERSION_TIME_MS); } - tmp102->last_update = jiffies; - /* Mark that we are not ready with data until conversion is complete */ - tmp102->first_time = true; - mutex_init(&tmp102->lock); - hwmon_dev = hwmon_device_register_with_groups(dev, client->name, - tmp102, tmp102_groups); + hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name, + tmp102, + tmp102_groups); if (IS_ERR(hwmon_dev)) { dev_dbg(dev, "unable to register hwmon device\n"); - status = PTR_ERR(hwmon_dev); - goto fail_restore_config; + return PTR_ERR(hwmon_dev); } - tmp102->hwmon_dev = hwmon_dev; devm_thermal_zone_of_sensor_register(hwmon_dev, 0, hwmon_dev, &tmp102_of_thermal_ops); dev_info(dev, "initialized\n"); - return 0; - -fail_restore_config: - i2c_smbus_write_word_swapped(client, TMP102_CONF_REG, - tmp102->config_orig); - return status; -} - -static int tmp102_remove(struct i2c_client *client) -{ - struct tmp102 *tmp102 = i2c_get_clientdata(client); - - hwmon_device_unregister(tmp102->hwmon_dev); - - /* Stop monitoring if device was stopped originally */ - if (tmp102->config_orig & TMP102_CONF_SD) { - int config; - - config = i2c_smbus_read_word_swapped(client, TMP102_CONF_REG); - if (config >= 0) - i2c_smbus_write_word_swapped(client, TMP102_CONF_REG, - config | TMP102_CONF_SD); - } - return 0; } @@ -267,27 +266,24 @@ static int tmp102_remove(struct i2c_client *client) static int tmp102_suspend(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); - int config; - - config = i2c_smbus_read_word_swapped(client, TMP102_CONF_REG); - if (config < 0) - return config; + struct tmp102 *tmp102 = i2c_get_clientdata(client); - config |= TMP102_CONF_SD; - return i2c_smbus_write_word_swapped(client, TMP102_CONF_REG, config); + return regmap_update_bits(tmp102->regmap, TMP102_CONF_REG, + TMP102_CONF_SD, TMP102_CONF_SD); } static int tmp102_resume(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); - int config; + struct tmp102 *tmp102 = i2c_get_clientdata(client); + int err; + + err = regmap_update_bits(tmp102->regmap, TMP102_CONF_REG, + TMP102_CONF_SD, 0); - config = i2c_smbus_read_word_swapped(client, TMP102_CONF_REG); - if (config < 0) - return config; + tmp102->ready_time = jiffies + msecs_to_jiffies(CONVERSION_TIME_MS); - config &= ~TMP102_CONF_SD; - return i2c_smbus_write_word_swapped(client, TMP102_CONF_REG, config); + return err; } #endif /* CONFIG_PM */ @@ -303,7 +299,6 @@ static struct i2c_driver tmp102_driver = { .driver.name = DRIVER_NAME, .driver.pm = &tmp102_dev_pm_ops, .probe = tmp102_probe, - .remove = tmp102_remove, .id_table = tmp102_id, }; diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c index ccf4cffe0ee1..eeeed2c7d081 100644 --- a/drivers/hwmon/tmp401.c +++ b/drivers/hwmon/tmp401.c @@ -47,7 +47,7 @@ static const unsigned short normal_i2c[] = { 0x48, 0x49, 0x4a, 0x4c, 0x4d, 0x4e, 0x4f, I2C_CLIENT_END }; -enum chips { tmp401, tmp411, tmp431, tmp432, tmp435 }; +enum chips { tmp401, tmp411, tmp431, tmp432, tmp435, tmp461 }; /* * The TMP401 registers, note some registers have different addresses for @@ -62,31 +62,34 @@ enum chips { tmp401, tmp411, tmp431, tmp432, tmp435 }; #define TMP401_MANUFACTURER_ID_REG 0xFE #define TMP401_DEVICE_ID_REG 0xFF -static const u8 TMP401_TEMP_MSB_READ[6][2] = { +static const u8 TMP401_TEMP_MSB_READ[7][2] = { { 0x00, 0x01 }, /* temp */ { 0x06, 0x08 }, /* low limit */ { 0x05, 0x07 }, /* high limit */ { 0x20, 0x19 }, /* therm (crit) limit */ { 0x30, 0x34 }, /* lowest */ { 0x32, 0x36 }, /* highest */ + { 0, 0x11 }, /* offset */ }; -static const u8 TMP401_TEMP_MSB_WRITE[6][2] = { +static const u8 TMP401_TEMP_MSB_WRITE[7][2] = { { 0, 0 }, /* temp (unused) */ { 0x0C, 0x0E }, /* low limit */ { 0x0B, 0x0D }, /* high limit */ { 0x20, 0x19 }, /* therm (crit) limit */ { 0x30, 0x34 }, /* lowest */ { 0x32, 0x36 }, /* highest */ + { 0, 0x11 }, /* offset */ }; -static const u8 TMP401_TEMP_LSB[6][2] = { +static const u8 TMP401_TEMP_LSB[7][2] = { { 0x15, 0x10 }, /* temp */ { 0x17, 0x14 }, /* low limit */ { 0x16, 0x13 }, /* high limit */ { 0, 0 }, /* therm (crit) limit (unused) */ { 0x31, 0x35 }, /* lowest */ { 0x33, 0x37 }, /* highest */ + { 0, 0x12 }, /* offset */ }; static const u8 TMP432_TEMP_MSB_READ[4][3] = { @@ -149,6 +152,7 @@ static const struct i2c_device_id tmp401_id[] = { { "tmp431", tmp431 }, { "tmp432", tmp432 }, { "tmp435", tmp435 }, + { "tmp461", tmp461 }, { } }; MODULE_DEVICE_TABLE(i2c, tmp401_id); @@ -170,7 +174,7 @@ struct tmp401_data { /* register values */ u8 status[4]; u8 config; - u16 temp[6][3]; + u16 temp[7][3]; u8 temp_crit_hyst; }; @@ -612,6 +616,22 @@ static const struct attribute_group tmp432_group = { .attrs = tmp432_attributes, }; +/* + * Additional features of the TMP461 chip. + * The TMP461 temperature offset for the remote channel. + */ +static SENSOR_DEVICE_ATTR_2(temp2_offset, S_IWUSR | S_IRUGO, show_temp, + store_temp, 6, 1); + +static struct attribute *tmp461_attributes[] = { + &sensor_dev_attr_temp2_offset.dev_attr.attr, + NULL +}; + +static const struct attribute_group tmp461_group = { + .attrs = tmp461_attributes, +}; + /* * Begin non sysfs callback code (aka Real code) */ @@ -714,7 +734,7 @@ static int tmp401_probe(struct i2c_client *client, const struct i2c_device_id *id) { static const char * const names[] = { - "TMP401", "TMP411", "TMP431", "TMP432", "TMP435" + "TMP401", "TMP411", "TMP431", "TMP432", "TMP435", "TMP461" }; struct device *dev = &client->dev; struct device *hwmon_dev; @@ -745,6 +765,9 @@ static int tmp401_probe(struct i2c_client *client, if (data->kind == tmp432) data->groups[groups++] = &tmp432_group; + if (data->kind == tmp461) + data->groups[groups++] = &tmp461_group; + hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name, data, data->groups); if (IS_ERR(hwmon_dev)) diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c index 1be543e8e42f..6f0a51a2c6ec 100644 --- a/drivers/hwtracing/intel_th/core.c +++ b/drivers/hwtracing/intel_th/core.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "intel_th.h" @@ -67,23 +68,33 @@ static int intel_th_probe(struct device *dev) hubdrv = to_intel_th_driver(hub->dev.driver); + pm_runtime_set_active(dev); + pm_runtime_no_callbacks(dev); + pm_runtime_enable(dev); + ret = thdrv->probe(to_intel_th_device(dev)); if (ret) - return ret; + goto out_pm; if (thdrv->attr_group) { ret = sysfs_create_group(&thdev->dev.kobj, thdrv->attr_group); - if (ret) { - thdrv->remove(thdev); - - return ret; - } + if (ret) + goto out; } if (thdev->type == INTEL_TH_OUTPUT && !intel_th_output_assigned(thdev)) + /* does not talk to hardware */ ret = hubdrv->assign(hub, thdev); +out: + if (ret) + thdrv->remove(thdev); + +out_pm: + if (ret) + pm_runtime_disable(dev); + return ret; } @@ -103,6 +114,8 @@ static int intel_th_remove(struct device *dev) if (thdrv->attr_group) sysfs_remove_group(&thdev->dev.kobj, thdrv->attr_group); + pm_runtime_get_sync(dev); + thdrv->remove(thdev); if (intel_th_output_assigned(thdev)) { @@ -110,9 +123,14 @@ static int intel_th_remove(struct device *dev) to_intel_th_driver(dev->parent->driver); if (hub->dev.driver) + /* does not talk to hardware */ hubdrv->unassign(hub, thdev); } + pm_runtime_disable(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + return 0; } @@ -185,6 +203,7 @@ static int intel_th_output_activate(struct intel_th_device *thdev) { struct intel_th_driver *thdrv = to_intel_th_driver_or_null(thdev->dev.driver); + int ret = 0; if (!thdrv) return -ENODEV; @@ -192,12 +211,17 @@ static int intel_th_output_activate(struct intel_th_device *thdev) if (!try_module_get(thdrv->driver.owner)) return -ENODEV; + pm_runtime_get_sync(&thdev->dev); + if (thdrv->activate) - return thdrv->activate(thdev); + ret = thdrv->activate(thdev); + else + intel_th_trace_enable(thdev); - intel_th_trace_enable(thdev); + if (ret) + pm_runtime_put(&thdev->dev); - return 0; + return ret; } static void intel_th_output_deactivate(struct intel_th_device *thdev) @@ -213,6 +237,7 @@ static void intel_th_output_deactivate(struct intel_th_device *thdev) else intel_th_trace_disable(thdev); + pm_runtime_put(&thdev->dev); module_put(thdrv->driver.owner); } @@ -465,6 +490,38 @@ static struct intel_th_subdevice { }, }; +#ifdef CONFIG_MODULES +static void __intel_th_request_hub_module(struct work_struct *work) +{ + struct intel_th *th = container_of(work, struct intel_th, + request_module_work); + + request_module("intel_th_%s", th->hub->name); +} + +static int intel_th_request_hub_module(struct intel_th *th) +{ + INIT_WORK(&th->request_module_work, __intel_th_request_hub_module); + schedule_work(&th->request_module_work); + + return 0; +} + +static void intel_th_request_hub_module_flush(struct intel_th *th) +{ + flush_work(&th->request_module_work); +} +#else +static inline int intel_th_request_hub_module(struct intel_th *th) +{ + return -EINVAL; +} + +static inline void intel_th_request_hub_module_flush(struct intel_th *th) +{ +} +#endif /* CONFIG_MODULES */ + static int intel_th_populate(struct intel_th *th, struct resource *devres, unsigned int ndevres, int irq) { @@ -535,7 +592,7 @@ static int intel_th_populate(struct intel_th *th, struct resource *devres, /* need switch driver to be loaded to enumerate the rest */ if (subdev->type == INTEL_TH_SWITCH && !req) { th->hub = thdev; - err = request_module("intel_th_%s", subdev->name); + err = intel_th_request_hub_module(th); if (!err) req++; } @@ -628,6 +685,10 @@ intel_th_alloc(struct device *dev, struct resource *devres, dev_set_drvdata(dev, th); + pm_runtime_no_callbacks(dev); + pm_runtime_put(dev); + pm_runtime_allow(dev); + err = intel_th_populate(th, devres, ndevres, irq); if (err) goto err_chrdev; @@ -635,6 +696,8 @@ intel_th_alloc(struct device *dev, struct resource *devres, return th; err_chrdev: + pm_runtime_forbid(dev); + __unregister_chrdev(th->major, 0, TH_POSSIBLE_OUTPUTS, "intel_th/output"); @@ -652,12 +715,16 @@ void intel_th_free(struct intel_th *th) { int i; + intel_th_request_hub_module_flush(th); for (i = 0; i < TH_SUBDEVICE_MAX; i++) if (th->thdev[i] != th->hub) intel_th_device_remove(th->thdev[i]); intel_th_device_remove(th->hub); + pm_runtime_get_sync(th->dev); + pm_runtime_forbid(th->dev); + __unregister_chrdev(th->major, 0, TH_POSSIBLE_OUTPUTS, "intel_th/output"); @@ -682,6 +749,7 @@ int intel_th_trace_enable(struct intel_th_device *thdev) if (WARN_ON_ONCE(thdev->type != INTEL_TH_OUTPUT)) return -EINVAL; + pm_runtime_get_sync(&thdev->dev); hubdrv->enable(hub, &thdev->output); return 0; @@ -702,6 +770,7 @@ int intel_th_trace_disable(struct intel_th_device *thdev) return -EINVAL; hubdrv->disable(hub, &thdev->output); + pm_runtime_put(&thdev->dev); return 0; } diff --git a/drivers/hwtracing/intel_th/gth.c b/drivers/hwtracing/intel_th/gth.c index 9beea0b54231..33e09369a491 100644 --- a/drivers/hwtracing/intel_th/gth.c +++ b/drivers/hwtracing/intel_th/gth.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "intel_th.h" #include "gth.h" @@ -190,6 +191,11 @@ static ssize_t master_attr_store(struct device *dev, if (old_port >= 0) { gth->master[ma->master] = -1; clear_bit(ma->master, gth->output[old_port].master); + + /* + * if the port is active, program this setting, + * implies that runtime PM is on + */ if (gth->output[old_port].output->active) gth_master_set(gth, ma->master, -1); } @@ -204,7 +210,7 @@ static ssize_t master_attr_store(struct device *dev, set_bit(ma->master, gth->output[port].master); - /* if the port is active, program this setting */ + /* if the port is active, program this setting, see above */ if (gth->output[port].output->active) gth_master_set(gth, ma->master, port); } @@ -326,11 +332,15 @@ static ssize_t output_attr_show(struct device *dev, struct gth_device *gth = oa->gth; size_t count; + pm_runtime_get_sync(dev); + spin_lock(>h->gth_lock); count = snprintf(buf, PAGE_SIZE, "%x\n", gth_output_parm_get(gth, oa->port, oa->parm)); spin_unlock(>h->gth_lock); + pm_runtime_put(dev); + return count; } @@ -346,10 +356,14 @@ static ssize_t output_attr_store(struct device *dev, if (kstrtouint(buf, 16, &config) < 0) return -EINVAL; + pm_runtime_get_sync(dev); + spin_lock(>h->gth_lock); gth_output_parm_set(gth, oa->port, oa->parm, config); spin_unlock(>h->gth_lock); + pm_runtime_put(dev); + return count; } @@ -451,7 +465,7 @@ static int intel_th_output_attributes(struct gth_device *gth) } /** - * intel_th_gth_disable() - enable tracing to an output device + * intel_th_gth_disable() - disable tracing to an output device * @thdev: GTH device * @output: output device's descriptor * diff --git a/drivers/hwtracing/intel_th/intel_th.h b/drivers/hwtracing/intel_th/intel_th.h index 0df22e30673d..4c195786bf1f 100644 --- a/drivers/hwtracing/intel_th/intel_th.h +++ b/drivers/hwtracing/intel_th/intel_th.h @@ -114,6 +114,9 @@ intel_th_output_assigned(struct intel_th_device *thdev) * @unassign: deassociate an output type device from an output port * @enable: enable tracing for a given output device * @disable: disable tracing for a given output device + * @irq: interrupt callback + * @activate: enable tracing on the output's side + * @deactivate: disable tracing on the output's side * @fops: file operations for device nodes * @attr_group: attributes provided by the driver * @@ -205,6 +208,9 @@ struct intel_th { int id; int major; +#ifdef CONFIG_MODULES + struct work_struct request_module_work; +#endif /* CONFIG_MODULES */ #ifdef CONFIG_INTEL_TH_DEBUG struct dentry *dbg; #endif diff --git a/drivers/hwtracing/intel_th/pci.c b/drivers/hwtracing/intel_th/pci.c index 5e25c7eb31d3..0bba3842336e 100644 --- a/drivers/hwtracing/intel_th/pci.c +++ b/drivers/hwtracing/intel_th/pci.c @@ -80,6 +80,11 @@ static const struct pci_device_id intel_th_pci_id_table[] = { PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1a8e), .driver_data = (kernel_ulong_t)0, }, + { + /* Kaby Lake PCH-H */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xa2a6), + .driver_data = (kernel_ulong_t)0, + }, { 0 }, }; diff --git a/drivers/hwtracing/stm/core.c b/drivers/hwtracing/stm/core.c index ff31108b066f..51f81d64ca37 100644 --- a/drivers/hwtracing/stm/core.c +++ b/drivers/hwtracing/stm/core.c @@ -15,6 +15,7 @@ * as defined in MIPI STPv2 specification. */ +#include #include #include #include @@ -482,14 +483,40 @@ static ssize_t stm_char_write(struct file *file, const char __user *buf, return -EFAULT; } + pm_runtime_get_sync(&stm->dev); + count = stm_write(stm->data, stmf->output.master, stmf->output.channel, kbuf, count); + pm_runtime_mark_last_busy(&stm->dev); + pm_runtime_put_autosuspend(&stm->dev); kfree(kbuf); return count; } +static void stm_mmap_open(struct vm_area_struct *vma) +{ + struct stm_file *stmf = vma->vm_file->private_data; + struct stm_device *stm = stmf->stm; + + pm_runtime_get(&stm->dev); +} + +static void stm_mmap_close(struct vm_area_struct *vma) +{ + struct stm_file *stmf = vma->vm_file->private_data; + struct stm_device *stm = stmf->stm; + + pm_runtime_mark_last_busy(&stm->dev); + pm_runtime_put_autosuspend(&stm->dev); +} + +static const struct vm_operations_struct stm_mmap_vmops = { + .open = stm_mmap_open, + .close = stm_mmap_close, +}; + static int stm_char_mmap(struct file *file, struct vm_area_struct *vma) { struct stm_file *stmf = file->private_data; @@ -514,8 +541,11 @@ static int stm_char_mmap(struct file *file, struct vm_area_struct *vma) if (!phys) return -EINVAL; + pm_runtime_get_sync(&stm->dev); + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; + vma->vm_ops = &stm_mmap_vmops; vm_iomap_memory(vma, phys, size); return 0; @@ -701,6 +731,17 @@ int stm_register_device(struct device *parent, struct stm_data *stm_data, if (err) goto err_device; + /* + * Use delayed autosuspend to avoid bouncing back and forth + * on recurring character device writes, with the initial + * delay time of 2 seconds. + */ + pm_runtime_no_callbacks(&stm->dev); + pm_runtime_use_autosuspend(&stm->dev); + pm_runtime_set_autosuspend_delay(&stm->dev, 2000); + pm_runtime_set_suspended(&stm->dev); + pm_runtime_enable(&stm->dev); + return 0; err_device: @@ -724,6 +765,9 @@ void stm_unregister_device(struct stm_data *stm_data) struct stm_source_device *src, *iter; int i, ret; + pm_runtime_dont_use_autosuspend(&stm->dev); + pm_runtime_disable(&stm->dev); + mutex_lock(&stm->link_mutex); list_for_each_entry_safe(src, iter, &stm->link_list, link_entry) { ret = __stm_source_link_drop(src, stm); @@ -878,6 +922,8 @@ static int __stm_source_link_drop(struct stm_source_device *src, stm_output_free(link, &src->output); list_del_init(&src->link_entry); + pm_runtime_mark_last_busy(&link->dev); + pm_runtime_put_autosuspend(&link->dev); /* matches stm_find_device() from stm_source_link_store() */ stm_put_device(link); rcu_assign_pointer(src->link, NULL); @@ -971,8 +1017,11 @@ static ssize_t stm_source_link_store(struct device *dev, if (!link) return -EINVAL; + pm_runtime_get(&link->dev); + err = stm_source_link_add(src, link); if (err) { + pm_runtime_put_autosuspend(&link->dev); /* matches the stm_find_device() above */ stm_put_device(link); } @@ -1033,6 +1082,9 @@ int stm_source_register_device(struct device *parent, if (err) goto err; + pm_runtime_no_callbacks(&src->dev); + pm_runtime_forbid(&src->dev); + err = device_add(&src->dev); if (err) goto err; diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c index cc6439ab3f71..041050edd809 100644 --- a/drivers/i2c/busses/i2c-qup.c +++ b/drivers/i2c/busses/i2c-qup.c @@ -1268,6 +1268,8 @@ static int qup_i2c_xfer_v2(struct i2c_adapter *adap, } } + idx = 0; + do { if (msgs[idx].len == 0) { ret = -EINVAL; diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index 445398c314a3..b126dbaa47e3 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -912,7 +912,7 @@ static int tegra_i2c_probe(struct platform_device *pdev) ret = tegra_i2c_init(i2c_dev); if (ret) { dev_err(&pdev->dev, "Failed to initialize i2c controller"); - goto unprepare_div_clk; + goto disable_div_clk; } ret = devm_request_irq(&pdev->dev, i2c_dev->irq, diff --git a/drivers/i2c/i2c-boardinfo.c b/drivers/i2c/i2c-boardinfo.c index e33022e2d459..6e5fac6a5262 100644 --- a/drivers/i2c/i2c-boardinfo.c +++ b/drivers/i2c/i2c-boardinfo.c @@ -56,9 +56,7 @@ EXPORT_SYMBOL_GPL(__i2c_first_dynamic_bus_num); * The board info passed can safely be __initdata, but be careful of embedded * pointers (for platform_data, functions, etc) since that won't be copied. */ -int __init -i2c_register_board_info(int busnum, - struct i2c_board_info const *info, unsigned len) +int i2c_register_board_info(int busnum, struct i2c_board_info const *info, unsigned len) { int status; diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c index af11b658984d..74e5aeaf84f9 100644 --- a/drivers/i2c/i2c-core.c +++ b/drivers/i2c/i2c-core.c @@ -107,12 +107,11 @@ struct acpi_i2c_lookup { acpi_handle device_handle; }; -static int acpi_i2c_find_address(struct acpi_resource *ares, void *data) +static int acpi_i2c_fill_info(struct acpi_resource *ares, void *data) { struct acpi_i2c_lookup *lookup = data; struct i2c_board_info *info = lookup->info; struct acpi_resource_i2c_serialbus *sb; - acpi_handle adapter_handle; acpi_status status; if (info->addr || ares->type != ACPI_RESOURCE_TYPE_SERIAL_BUS) @@ -122,80 +121,102 @@ static int acpi_i2c_find_address(struct acpi_resource *ares, void *data) if (sb->type != ACPI_RESOURCE_SERIAL_TYPE_I2C) return 1; - /* - * Extract the ResourceSource and make sure that the handle matches - * with the I2C adapter handle. - */ status = acpi_get_handle(lookup->device_handle, sb->resource_source.string_ptr, - &adapter_handle); - if (ACPI_SUCCESS(status) && adapter_handle == lookup->adapter_handle) { - info->addr = sb->slave_address; - if (sb->access_mode == ACPI_I2C_10BIT_MODE) - info->flags |= I2C_CLIENT_TEN; - } + &lookup->adapter_handle); + if (!ACPI_SUCCESS(status)) + return 1; + + info->addr = sb->slave_address; + if (sb->access_mode == ACPI_I2C_10BIT_MODE) + info->flags |= I2C_CLIENT_TEN; return 1; } -static acpi_status acpi_i2c_add_device(acpi_handle handle, u32 level, - void *data, void **return_value) +static int acpi_i2c_get_info(struct acpi_device *adev, + struct i2c_board_info *info, + acpi_handle *adapter_handle) { - struct i2c_adapter *adapter = data; struct list_head resource_list; - struct acpi_i2c_lookup lookup; struct resource_entry *entry; - struct i2c_board_info info; - struct acpi_device *adev; + struct acpi_i2c_lookup lookup; int ret; - if (acpi_bus_get_device(handle, &adev)) - return AE_OK; - if (acpi_bus_get_status(adev) || !adev->status.present) - return AE_OK; + if (acpi_bus_get_status(adev) || !adev->status.present || + acpi_device_enumerated(adev)) + return -EINVAL; - memset(&info, 0, sizeof(info)); - info.fwnode = acpi_fwnode_handle(adev); + memset(info, 0, sizeof(*info)); + info->fwnode = acpi_fwnode_handle(adev); memset(&lookup, 0, sizeof(lookup)); - lookup.adapter_handle = ACPI_HANDLE(&adapter->dev); - lookup.device_handle = handle; - lookup.info = &info; + lookup.device_handle = acpi_device_handle(adev); + lookup.info = info; - /* - * Look up for I2cSerialBus resource with ResourceSource that - * matches with this adapter. - */ + /* Look up for I2cSerialBus resource */ INIT_LIST_HEAD(&resource_list); ret = acpi_dev_get_resources(adev, &resource_list, - acpi_i2c_find_address, &lookup); + acpi_i2c_fill_info, &lookup); acpi_dev_free_resource_list(&resource_list); - if (ret < 0 || !info.addr) - return AE_OK; + if (ret < 0 || !info->addr) + return -EINVAL; + + *adapter_handle = lookup.adapter_handle; /* Then fill IRQ number if any */ ret = acpi_dev_get_resources(adev, &resource_list, NULL, NULL); if (ret < 0) - return AE_OK; + return -EINVAL; resource_list_for_each_entry(entry, &resource_list) { if (resource_type(entry->res) == IORESOURCE_IRQ) { - info.irq = entry->res->start; + info->irq = entry->res->start; break; } } acpi_dev_free_resource_list(&resource_list); + strlcpy(info->type, dev_name(&adev->dev), sizeof(info->type)); + + return 0; +} + +static void acpi_i2c_register_device(struct i2c_adapter *adapter, + struct acpi_device *adev, + struct i2c_board_info *info) +{ adev->power.flags.ignore_parent = true; - strlcpy(info.type, dev_name(&adev->dev), sizeof(info.type)); - if (!i2c_new_device(adapter, &info)) { + acpi_device_set_enumerated(adev); + + if (!i2c_new_device(adapter, info)) { adev->power.flags.ignore_parent = false; dev_err(&adapter->dev, "failed to add I2C device %s from ACPI\n", dev_name(&adev->dev)); } +} + +static acpi_status acpi_i2c_add_device(acpi_handle handle, u32 level, + void *data, void **return_value) +{ + struct i2c_adapter *adapter = data; + struct acpi_device *adev; + acpi_handle adapter_handle; + struct i2c_board_info info; + + if (acpi_bus_get_device(handle, &adev)) + return AE_OK; + + if (acpi_i2c_get_info(adev, &info, &adapter_handle)) + return AE_OK; + + if (adapter_handle != ACPI_HANDLE(&adapter->dev)) + return AE_OK; + + acpi_i2c_register_device(adapter, adev, &info); return AE_OK; } @@ -225,8 +246,80 @@ static void acpi_i2c_register_devices(struct i2c_adapter *adap) dev_warn(&adap->dev, "failed to enumerate I2C slaves\n"); } +static int acpi_i2c_match_adapter(struct device *dev, void *data) +{ + struct i2c_adapter *adapter = i2c_verify_adapter(dev); + + if (!adapter) + return 0; + + return ACPI_HANDLE(dev) == (acpi_handle)data; +} + +static int acpi_i2c_match_device(struct device *dev, void *data) +{ + return ACPI_COMPANION(dev) == data; +} + +static struct i2c_adapter *acpi_i2c_find_adapter_by_handle(acpi_handle handle) +{ + struct device *dev; + + dev = bus_find_device(&i2c_bus_type, NULL, handle, + acpi_i2c_match_adapter); + return dev ? i2c_verify_adapter(dev) : NULL; +} + +static struct i2c_client *acpi_i2c_find_client_by_adev(struct acpi_device *adev) +{ + struct device *dev; + + dev = bus_find_device(&i2c_bus_type, NULL, adev, acpi_i2c_match_device); + return dev ? i2c_verify_client(dev) : NULL; +} + +static int acpi_i2c_notify(struct notifier_block *nb, unsigned long value, + void *arg) +{ + struct acpi_device *adev = arg; + struct i2c_board_info info; + acpi_handle adapter_handle; + struct i2c_adapter *adapter; + struct i2c_client *client; + + switch (value) { + case ACPI_RECONFIG_DEVICE_ADD: + if (acpi_i2c_get_info(adev, &info, &adapter_handle)) + break; + + adapter = acpi_i2c_find_adapter_by_handle(adapter_handle); + if (!adapter) + break; + + acpi_i2c_register_device(adapter, adev, &info); + break; + case ACPI_RECONFIG_DEVICE_REMOVE: + if (!acpi_device_enumerated(adev)) + break; + + client = acpi_i2c_find_client_by_adev(adev); + if (!client) + break; + + i2c_unregister_device(client); + put_device(&client->dev); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block i2c_acpi_notifier = { + .notifier_call = acpi_i2c_notify, +}; #else /* CONFIG_ACPI */ static inline void acpi_i2c_register_devices(struct i2c_adapter *adap) { } +extern struct notifier_block i2c_acpi_notifier; #endif /* CONFIG_ACPI */ #ifdef CONFIG_ACPI_I2C_OPREGION @@ -1089,6 +1182,8 @@ void i2c_unregister_device(struct i2c_client *client) { if (client->dev.of_node) of_node_clear_flag(client->dev.of_node, OF_POPULATED); + if (ACPI_COMPANION(&client->dev)) + acpi_device_clear_enumerated(ACPI_COMPANION(&client->dev)); device_unregister(&client->dev); } EXPORT_SYMBOL_GPL(i2c_unregister_device); @@ -2117,6 +2212,8 @@ static int __init i2c_init(void) if (IS_ENABLED(CONFIG_OF_DYNAMIC)) WARN_ON(of_reconfig_notifier_register(&i2c_of_notifier)); + if (IS_ENABLED(CONFIG_ACPI)) + WARN_ON(acpi_reconfig_notifier_register(&i2c_acpi_notifier)); return 0; @@ -2132,6 +2229,8 @@ bus_err: static void __exit i2c_exit(void) { + if (IS_ENABLED(CONFIG_ACPI)) + WARN_ON(acpi_reconfig_notifier_unregister(&i2c_acpi_notifier)); if (IS_ENABLED(CONFIG_OF_DYNAMIC)) WARN_ON(of_reconfig_notifier_unregister(&i2c_of_notifier)); i2c_del_driver(&dummy_driver); diff --git a/drivers/i2c/muxes/i2c-mux-reg.c b/drivers/i2c/muxes/i2c-mux-reg.c index 26e7c5187a58..c6a90b4a9c62 100644 --- a/drivers/i2c/muxes/i2c-mux-reg.c +++ b/drivers/i2c/muxes/i2c-mux-reg.c @@ -145,7 +145,7 @@ static int i2c_mux_reg_probe_dt(struct regmux *mux, mux->data.idle_in_use = true; /* map address from "reg" if exists */ - if (of_address_to_resource(np, 0, &res)) { + if (of_address_to_resource(np, 0, &res) == 0) { mux->data.reg_size = resource_size(&res); mux->data.reg = devm_ioremap_resource(&pdev->dev, &res); if (IS_ERR(mux->data.reg)) diff --git a/drivers/ide/ide-cd.c b/drivers/ide/ide-cd.c index ef907fd5ba98..bf9a2ad296ed 100644 --- a/drivers/ide/ide-cd.c +++ b/drivers/ide/ide-cd.c @@ -1770,7 +1770,6 @@ static int ide_cd_probe(ide_drive_t *drive) drive->driver_data = info; g->minors = 1; - g->driverfs_dev = &drive->gendev; g->flags = GENHD_FL_CD | GENHD_FL_REMOVABLE; if (ide_cdrom_setup(drive)) { put_device(&info->dev); @@ -1780,7 +1779,7 @@ static int ide_cd_probe(ide_drive_t *drive) ide_cd_read_toc(drive, &sense); g->fops = &idecd_ops; g->flags |= GENHD_FL_REMOVABLE | GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE; - add_disk(g); + device_add_disk(&drive->gendev, g); return 0; out_free_disk: diff --git a/drivers/ide/ide-cd_ioctl.c b/drivers/ide/ide-cd_ioctl.c index 474173eb31bb..5887a7a09e37 100644 --- a/drivers/ide/ide-cd_ioctl.c +++ b/drivers/ide/ide-cd_ioctl.c @@ -459,9 +459,6 @@ int ide_cdrom_packet(struct cdrom_device_info *cdi, layer. the packet must be complete, as we do not touch it at all. */ - if (cgc->data_direction == CGC_DATA_WRITE) - flags |= REQ_WRITE; - if (cgc->sense) memset(cgc->sense, 0, sizeof(struct request_sense)); diff --git a/drivers/ide/ide-disk.c b/drivers/ide/ide-disk.c index 5ceb176dc148..83679da0c3f0 100644 --- a/drivers/ide/ide-disk.c +++ b/drivers/ide/ide-disk.c @@ -431,7 +431,7 @@ static int idedisk_prep_fn(struct request_queue *q, struct request *rq) ide_drive_t *drive = q->queuedata; struct ide_cmd *cmd; - if (!(rq->cmd_flags & REQ_FLUSH)) + if (req_op(rq) != REQ_OP_FLUSH) return BLKPREP_OK; if (rq->special) { diff --git a/drivers/ide/ide-floppy.c b/drivers/ide/ide-floppy.c index 2fb5350c5410..f079d8d1d856 100644 --- a/drivers/ide/ide-floppy.c +++ b/drivers/ide/ide-floppy.c @@ -206,7 +206,7 @@ static void idefloppy_create_rw_cmd(ide_drive_t *drive, memcpy(rq->cmd, pc->c, 12); pc->rq = rq; - if (rq->cmd_flags & REQ_WRITE) + if (cmd == WRITE) pc->flags |= PC_FLAG_WRITING; pc->flags |= PC_FLAG_DMA_OK; diff --git a/drivers/ide/ide-gd.c b/drivers/ide/ide-gd.c index 838996a0039e..e823394ed543 100644 --- a/drivers/ide/ide-gd.c +++ b/drivers/ide/ide-gd.c @@ -412,12 +412,11 @@ static int ide_gd_probe(ide_drive_t *drive) set_capacity(g, ide_gd_capacity(drive)); g->minors = IDE_DISK_MINORS; - g->driverfs_dev = &drive->gendev; g->flags |= GENHD_FL_EXT_DEVT; if (drive->dev_flags & IDE_DFLAG_REMOVABLE) g->flags = GENHD_FL_REMOVABLE; g->fops = &ide_gd_ops; - add_disk(g); + device_add_disk(&drive->gendev, g); return 0; out_free_disk: diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index c96649292b55..9b2ef248788d 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -46,8 +46,6 @@ * to avoid complications with the lapic timer workaround. * Have not seen issues with suspend, but may need same workaround here. * - * There is currently no kernel-based automatic probing/loading mechanism - * if the driver is built as a module. */ /* un-comment DEBUG to enable pr_debug() statements */ @@ -60,8 +58,9 @@ #include #include #include -#include +#include #include +#include #include #include @@ -827,6 +826,35 @@ static struct cpuidle_state bxt_cstates[] = { .enter = NULL } }; +static struct cpuidle_state dnv_cstates[] = { + { + .name = "C1-DNV", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 2, + .target_residency = 2, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C1E-DNV", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01), + .exit_latency = 10, + .target_residency = 20, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6-DNV", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 50, + .target_residency = 500, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; + /** * intel_idle * @dev: cpuidle_device @@ -1016,45 +1044,50 @@ static const struct idle_cpu idle_cpu_bxt = { .disable_promotion_to_c1e = true, }; +static const struct idle_cpu idle_cpu_dnv = { + .state_table = dnv_cstates, + .disable_promotion_to_c1e = true, +}; + #define ICPU(model, cpu) \ { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } static const struct x86_cpu_id intel_idle_ids[] __initconst = { - ICPU(0x1a, idle_cpu_nehalem), - ICPU(0x1e, idle_cpu_nehalem), - ICPU(0x1f, idle_cpu_nehalem), - ICPU(0x25, idle_cpu_nehalem), - ICPU(0x2c, idle_cpu_nehalem), - ICPU(0x2e, idle_cpu_nehalem), - ICPU(0x1c, idle_cpu_atom), - ICPU(0x26, idle_cpu_lincroft), - ICPU(0x2f, idle_cpu_nehalem), - ICPU(0x2a, idle_cpu_snb), - ICPU(0x2d, idle_cpu_snb), - ICPU(0x36, idle_cpu_atom), - ICPU(0x37, idle_cpu_byt), - ICPU(0x4c, idle_cpu_cht), - ICPU(0x3a, idle_cpu_ivb), - ICPU(0x3e, idle_cpu_ivt), - ICPU(0x3c, idle_cpu_hsw), - ICPU(0x3f, idle_cpu_hsw), - ICPU(0x45, idle_cpu_hsw), - ICPU(0x46, idle_cpu_hsw), - ICPU(0x4d, idle_cpu_avn), - ICPU(0x3d, idle_cpu_bdw), - ICPU(0x47, idle_cpu_bdw), - ICPU(0x4f, idle_cpu_bdw), - ICPU(0x56, idle_cpu_bdw), - ICPU(0x4e, idle_cpu_skl), - ICPU(0x5e, idle_cpu_skl), - ICPU(0x8e, idle_cpu_skl), - ICPU(0x9e, idle_cpu_skl), - ICPU(0x55, idle_cpu_skx), - ICPU(0x57, idle_cpu_knl), - ICPU(0x5c, idle_cpu_bxt), + ICPU(INTEL_FAM6_NEHALEM_EP, idle_cpu_nehalem), + ICPU(INTEL_FAM6_NEHALEM, idle_cpu_nehalem), + ICPU(INTEL_FAM6_WESTMERE2, idle_cpu_nehalem), + ICPU(INTEL_FAM6_WESTMERE, idle_cpu_nehalem), + ICPU(INTEL_FAM6_WESTMERE_EP, idle_cpu_nehalem), + ICPU(INTEL_FAM6_NEHALEM_EX, idle_cpu_nehalem), + ICPU(INTEL_FAM6_ATOM_PINEVIEW, idle_cpu_atom), + ICPU(INTEL_FAM6_ATOM_LINCROFT, idle_cpu_lincroft), + ICPU(INTEL_FAM6_WESTMERE_EX, idle_cpu_nehalem), + ICPU(INTEL_FAM6_SANDYBRIDGE, idle_cpu_snb), + ICPU(INTEL_FAM6_SANDYBRIDGE_X, idle_cpu_snb), + ICPU(INTEL_FAM6_ATOM_CEDARVIEW, idle_cpu_atom), + ICPU(INTEL_FAM6_ATOM_SILVERMONT1, idle_cpu_byt), + ICPU(INTEL_FAM6_ATOM_AIRMONT, idle_cpu_cht), + ICPU(INTEL_FAM6_IVYBRIDGE, idle_cpu_ivb), + ICPU(INTEL_FAM6_IVYBRIDGE_X, idle_cpu_ivt), + ICPU(INTEL_FAM6_HASWELL_CORE, idle_cpu_hsw), + ICPU(INTEL_FAM6_HASWELL_X, idle_cpu_hsw), + ICPU(INTEL_FAM6_HASWELL_ULT, idle_cpu_hsw), + ICPU(INTEL_FAM6_HASWELL_GT3E, idle_cpu_hsw), + ICPU(INTEL_FAM6_ATOM_SILVERMONT2, idle_cpu_avn), + ICPU(INTEL_FAM6_BROADWELL_CORE, idle_cpu_bdw), + ICPU(INTEL_FAM6_BROADWELL_GT3E, idle_cpu_bdw), + ICPU(INTEL_FAM6_BROADWELL_X, idle_cpu_bdw), + ICPU(INTEL_FAM6_BROADWELL_XEON_D, idle_cpu_bdw), + ICPU(INTEL_FAM6_SKYLAKE_MOBILE, idle_cpu_skl), + ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, idle_cpu_skl), + ICPU(INTEL_FAM6_KABYLAKE_MOBILE, idle_cpu_skl), + ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, idle_cpu_skl), + ICPU(INTEL_FAM6_SKYLAKE_X, idle_cpu_skx), + ICPU(INTEL_FAM6_XEON_PHI_KNL, idle_cpu_knl), + ICPU(INTEL_FAM6_ATOM_GOLDMONT, idle_cpu_bxt), + ICPU(INTEL_FAM6_ATOM_DENVERTON, idle_cpu_dnv), {} }; -MODULE_DEVICE_TABLE(x86cpu, intel_idle_ids); /* * intel_idle_probe() @@ -1154,7 +1187,10 @@ static unsigned long long irtl_2_usec(unsigned long long irtl) { unsigned long long ns; - ns = irtl_ns_units[(irtl >> 10) & 0x3]; + if (!irtl) + return 0; + + ns = irtl_ns_units[(irtl >> 10) & 0x7]; return div64_u64((irtl & 0x3FF) * ns, 1000); } @@ -1167,43 +1203,39 @@ static unsigned long long irtl_2_usec(unsigned long long irtl) static void bxt_idle_state_table_update(void) { unsigned long long msr; + unsigned int usec; rdmsrl(MSR_PKGC6_IRTL, msr); - if (msr) { - unsigned int usec = irtl_2_usec(msr); - + usec = irtl_2_usec(msr); + if (usec) { bxt_cstates[2].exit_latency = usec; bxt_cstates[2].target_residency = usec; } rdmsrl(MSR_PKGC7_IRTL, msr); - if (msr) { - unsigned int usec = irtl_2_usec(msr); - + usec = irtl_2_usec(msr); + if (usec) { bxt_cstates[3].exit_latency = usec; bxt_cstates[3].target_residency = usec; } rdmsrl(MSR_PKGC8_IRTL, msr); - if (msr) { - unsigned int usec = irtl_2_usec(msr); - + usec = irtl_2_usec(msr); + if (usec) { bxt_cstates[4].exit_latency = usec; bxt_cstates[4].target_residency = usec; } rdmsrl(MSR_PKGC9_IRTL, msr); - if (msr) { - unsigned int usec = irtl_2_usec(msr); - + usec = irtl_2_usec(msr); + if (usec) { bxt_cstates[5].exit_latency = usec; bxt_cstates[5].target_residency = usec; } rdmsrl(MSR_PKGC10_IRTL, msr); - if (msr) { - unsigned int usec = irtl_2_usec(msr); - + usec = irtl_2_usec(msr); + if (usec) { bxt_cstates[6].exit_latency = usec; bxt_cstates[6].target_residency = usec; } @@ -1261,13 +1293,13 @@ static void intel_idle_state_table_update(void) { switch (boot_cpu_data.x86_model) { - case 0x3e: /* IVT */ + case INTEL_FAM6_IVYBRIDGE_X: ivt_idle_state_table_update(); break; - case 0x5c: /* BXT */ + case INTEL_FAM6_ATOM_GOLDMONT: bxt_idle_state_table_update(); break; - case 0x5e: /* SKL-H */ + case INTEL_FAM6_SKYLAKE_DESKTOP: sklh_idle_state_table_update(); break; } @@ -1415,34 +1447,12 @@ static int __init intel_idle_init(void) return 0; } +device_initcall(intel_idle_init); -static void __exit intel_idle_exit(void) -{ - struct cpuidle_device *dev; - int i; - - cpu_notifier_register_begin(); - - if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) - on_each_cpu(__setup_broadcast_timer, (void *)false, 1); - __unregister_cpu_notifier(&cpu_hotplug_notifier); - - for_each_possible_cpu(i) { - dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); - cpuidle_unregister_device(dev); - } - - cpu_notifier_register_done(); - - cpuidle_unregister_driver(&intel_idle_driver); - free_percpu(intel_idle_cpuidle_devices); -} - -module_init(intel_idle_init); -module_exit(intel_idle_exit); - +/* + * We are not really modular, but we used to support that. Meaning we also + * support "intel_idle.max_cstate=..." at boot and also a read-only export of + * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param + * is the easiest way (currently) to continue doing that. + */ module_param(max_cstate, int, 0444); - -MODULE_AUTHOR("Len Brown "); -MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION); -MODULE_LICENSE("GPL"); diff --git a/drivers/iio/Kconfig b/drivers/iio/Kconfig index 505e921f0b19..6743b18194fb 100644 --- a/drivers/iio/Kconfig +++ b/drivers/iio/Kconfig @@ -46,6 +46,14 @@ config IIO_CONSUMERS_PER_TRIGGER This value controls the maximum number of consumers that a given trigger may handle. Default is 2. +config IIO_SW_DEVICE + tristate "Enable software IIO device support" + select IIO_CONFIGFS + help + Provides IIO core support for software devices. A software + device can be created via configfs or directly by a driver + using the API provided. + config IIO_SW_TRIGGER tristate "Enable software triggers support" select IIO_CONFIGFS diff --git a/drivers/iio/Makefile b/drivers/iio/Makefile index 20f649073462..87e4c4369e2f 100644 --- a/drivers/iio/Makefile +++ b/drivers/iio/Makefile @@ -8,6 +8,7 @@ industrialio-$(CONFIG_IIO_BUFFER) += industrialio-buffer.o industrialio-$(CONFIG_IIO_TRIGGER) += industrialio-trigger.o obj-$(CONFIG_IIO_CONFIGFS) += industrialio-configfs.o +obj-$(CONFIG_IIO_SW_DEVICE) += industrialio-sw-device.o obj-$(CONFIG_IIO_SW_TRIGGER) += industrialio-sw-trigger.o obj-$(CONFIG_IIO_TRIGGERED_EVENT) += industrialio-triggered-event.o diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig index e4a758cd7d35..89d78208de3f 100644 --- a/drivers/iio/accel/Kconfig +++ b/drivers/iio/accel/Kconfig @@ -17,6 +17,16 @@ config BMA180 To compile this driver as a module, choose M here: the module will be called bma180. +config BMA220 + tristate "Bosch BMA220 3-Axis Accelerometer Driver" + depends on SPI + help + Say yes here to add support for the Bosch BMA220 triaxial + acceleration sensor. + + To compile this driver as a module, choose M here: the + module will be called bma220_spi. + config BMC150_ACCEL tristate "Bosch BMC150 Accelerometer Driver" select IIO_BUFFER @@ -136,13 +146,23 @@ config MMA7455_SPI To compile this driver as a module, choose M here: the module will be called mma7455_spi. +config MMA7660 + tristate "Freescale MMA7660FC 3-Axis Accelerometer Driver" + depends on I2C + help + Say yes here to get support for the Freescale MMA7660FC 3-Axis + accelerometer. + + Choosing M will build the driver as a module. If so, the module + will be called mma7660. + config MMA8452 - tristate "Freescale MMA8452Q and similar Accelerometers Driver" + tristate "Freescale / NXP MMA8452Q and similar Accelerometers Driver" depends on I2C select IIO_BUFFER select IIO_TRIGGERED_BUFFER help - Say yes here to build support for the following Freescale 3-axis + Say yes here to build support for the following Freescale / NXP 3-axis accelerometers: MMA8451Q, MMA8452Q, MMA8453Q, MMA8652FC, MMA8653FC, FXLS8471Q. diff --git a/drivers/iio/accel/Makefile b/drivers/iio/accel/Makefile index 71b6794de885..6cedbecca2ee 100644 --- a/drivers/iio/accel/Makefile +++ b/drivers/iio/accel/Makefile @@ -4,6 +4,7 @@ # When adding new entries keep the list in alphabetical order obj-$(CONFIG_BMA180) += bma180.o +obj-$(CONFIG_BMA220) += bma220_spi.o obj-$(CONFIG_BMC150_ACCEL) += bmc150-accel-core.o obj-$(CONFIG_BMC150_ACCEL_I2C) += bmc150-accel-i2c.o obj-$(CONFIG_BMC150_ACCEL_SPI) += bmc150-accel-spi.o @@ -15,6 +16,8 @@ obj-$(CONFIG_MMA7455) += mma7455_core.o obj-$(CONFIG_MMA7455_I2C) += mma7455_i2c.o obj-$(CONFIG_MMA7455_SPI) += mma7455_spi.o +obj-$(CONFIG_MMA7660) += mma7660.o + obj-$(CONFIG_MMA8452) += mma8452.o obj-$(CONFIG_MMA9551_CORE) += mma9551_core.o diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c index f04b88406995..e3f88ba5faf3 100644 --- a/drivers/iio/accel/bma180.c +++ b/drivers/iio/accel/bma180.c @@ -654,7 +654,7 @@ static irqreturn_t bma180_trigger_handler(int irq, void *p) struct iio_poll_func *pf = p; struct iio_dev *indio_dev = pf->indio_dev; struct bma180_data *data = iio_priv(indio_dev); - int64_t time_ns = iio_get_time_ns(); + s64 time_ns = iio_get_time_ns(indio_dev); int bit, ret, i = 0; mutex_lock(&data->mutex); diff --git a/drivers/iio/accel/bma220_spi.c b/drivers/iio/accel/bma220_spi.c new file mode 100644 index 000000000000..1098d10df8e8 --- /dev/null +++ b/drivers/iio/accel/bma220_spi.c @@ -0,0 +1,338 @@ +/** + * BMA220 Digital triaxial acceleration sensor driver + * + * Copyright (c) 2016, Intel Corporation. + * + * This file is subject to the terms and conditions of version 2 of + * the GNU General Public License. See the file COPYING in the main + * directory of this archive for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BMA220_REG_ID 0x00 +#define BMA220_REG_ACCEL_X 0x02 +#define BMA220_REG_ACCEL_Y 0x03 +#define BMA220_REG_ACCEL_Z 0x04 +#define BMA220_REG_RANGE 0x11 +#define BMA220_REG_SUSPEND 0x18 + +#define BMA220_CHIP_ID 0xDD +#define BMA220_READ_MASK 0x80 +#define BMA220_RANGE_MASK 0x03 +#define BMA220_DATA_SHIFT 2 +#define BMA220_SUSPEND_SLEEP 0xFF +#define BMA220_SUSPEND_WAKE 0x00 + +#define BMA220_DEVICE_NAME "bma220" +#define BMA220_SCALE_AVAILABLE "0.623 1.248 2.491 4.983" + +#define BMA220_ACCEL_CHANNEL(index, reg, axis) { \ + .type = IIO_ACCEL, \ + .address = reg, \ + .modified = 1, \ + .channel2 = IIO_MOD_##axis, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ + .scan_index = index, \ + .scan_type = { \ + .sign = 's', \ + .realbits = 6, \ + .storagebits = 8, \ + .shift = BMA220_DATA_SHIFT, \ + .endianness = IIO_CPU, \ + }, \ +} + +enum bma220_axis { + AXIS_X, + AXIS_Y, + AXIS_Z, +}; + +static IIO_CONST_ATTR(in_accel_scale_available, BMA220_SCALE_AVAILABLE); + +static struct attribute *bma220_attributes[] = { + &iio_const_attr_in_accel_scale_available.dev_attr.attr, + NULL, +}; + +static const struct attribute_group bma220_attribute_group = { + .attrs = bma220_attributes, +}; + +static const int bma220_scale_table[][4] = { + {0, 623000}, {1, 248000}, {2, 491000}, {4, 983000} +}; + +struct bma220_data { + struct spi_device *spi_device; + struct mutex lock; + s8 buffer[16]; /* 3x8-bit channels + 5x8 padding + 8x8 timestamp */ + u8 tx_buf[2] ____cacheline_aligned; +}; + +static const struct iio_chan_spec bma220_channels[] = { + BMA220_ACCEL_CHANNEL(0, BMA220_REG_ACCEL_X, X), + BMA220_ACCEL_CHANNEL(1, BMA220_REG_ACCEL_Y, Y), + BMA220_ACCEL_CHANNEL(2, BMA220_REG_ACCEL_Z, Z), + IIO_CHAN_SOFT_TIMESTAMP(3), +}; + +static inline int bma220_read_reg(struct spi_device *spi, u8 reg) +{ + return spi_w8r8(spi, reg | BMA220_READ_MASK); +} + +static const unsigned long bma220_accel_scan_masks[] = { + BIT(AXIS_X) | BIT(AXIS_Y) | BIT(AXIS_Z), + 0 +}; + +static irqreturn_t bma220_trigger_handler(int irq, void *p) +{ + int ret; + struct iio_poll_func *pf = p; + struct iio_dev *indio_dev = pf->indio_dev; + struct bma220_data *data = iio_priv(indio_dev); + struct spi_device *spi = data->spi_device; + + mutex_lock(&data->lock); + data->tx_buf[0] = BMA220_REG_ACCEL_X | BMA220_READ_MASK; + ret = spi_write_then_read(spi, data->tx_buf, 1, data->buffer, + ARRAY_SIZE(bma220_channels) - 1); + if (ret < 0) + goto err; + + iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, + pf->timestamp); +err: + mutex_unlock(&data->lock); + iio_trigger_notify_done(indio_dev->trig); + + return IRQ_HANDLED; +} + +static int bma220_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + int ret; + u8 range_idx; + struct bma220_data *data = iio_priv(indio_dev); + + switch (mask) { + case IIO_CHAN_INFO_RAW: + ret = bma220_read_reg(data->spi_device, chan->address); + if (ret < 0) + return -EINVAL; + *val = sign_extend32(ret >> BMA220_DATA_SHIFT, 5); + return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + ret = bma220_read_reg(data->spi_device, BMA220_REG_RANGE); + if (ret < 0) + return ret; + range_idx = ret & BMA220_RANGE_MASK; + *val = bma220_scale_table[range_idx][0]; + *val2 = bma220_scale_table[range_idx][1]; + return IIO_VAL_INT_PLUS_MICRO; + } + + return -EINVAL; +} + +static int bma220_write_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int val, int val2, long mask) +{ + int i; + int ret; + int index = -1; + struct bma220_data *data = iio_priv(indio_dev); + + switch (mask) { + case IIO_CHAN_INFO_SCALE: + for (i = 0; i < ARRAY_SIZE(bma220_scale_table); i++) + if (val == bma220_scale_table[i][0] && + val2 == bma220_scale_table[i][1]) { + index = i; + break; + } + if (index < 0) + return -EINVAL; + + mutex_lock(&data->lock); + data->tx_buf[0] = BMA220_REG_RANGE; + data->tx_buf[1] = index; + ret = spi_write(data->spi_device, data->tx_buf, + sizeof(data->tx_buf)); + if (ret < 0) + dev_err(&data->spi_device->dev, + "failed to set measurement range\n"); + mutex_unlock(&data->lock); + + return 0; + } + + return -EINVAL; +} + +static const struct iio_info bma220_info = { + .driver_module = THIS_MODULE, + .read_raw = bma220_read_raw, + .write_raw = bma220_write_raw, + .attrs = &bma220_attribute_group, +}; + +static int bma220_init(struct spi_device *spi) +{ + int ret; + + ret = bma220_read_reg(spi, BMA220_REG_ID); + if (ret != BMA220_CHIP_ID) + return -ENODEV; + + /* Make sure the chip is powered on */ + ret = bma220_read_reg(spi, BMA220_REG_SUSPEND); + if (ret < 0) + return ret; + else if (ret == BMA220_SUSPEND_WAKE) + return bma220_read_reg(spi, BMA220_REG_SUSPEND); + + return 0; +} + +static int bma220_deinit(struct spi_device *spi) +{ + int ret; + + /* Make sure the chip is powered off */ + ret = bma220_read_reg(spi, BMA220_REG_SUSPEND); + if (ret < 0) + return ret; + else if (ret == BMA220_SUSPEND_SLEEP) + return bma220_read_reg(spi, BMA220_REG_SUSPEND); + + return 0; +} + +static int bma220_probe(struct spi_device *spi) +{ + int ret; + struct iio_dev *indio_dev; + struct bma220_data *data; + + indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*data)); + if (!indio_dev) { + dev_err(&spi->dev, "iio allocation failed!\n"); + return -ENOMEM; + } + + data = iio_priv(indio_dev); + data->spi_device = spi; + spi_set_drvdata(spi, indio_dev); + mutex_init(&data->lock); + + indio_dev->dev.parent = &spi->dev; + indio_dev->info = &bma220_info; + indio_dev->name = BMA220_DEVICE_NAME; + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->channels = bma220_channels; + indio_dev->num_channels = ARRAY_SIZE(bma220_channels); + indio_dev->available_scan_masks = bma220_accel_scan_masks; + + ret = bma220_init(data->spi_device); + if (ret < 0) + return ret; + + ret = iio_triggered_buffer_setup(indio_dev, NULL, + bma220_trigger_handler, NULL); + if (ret < 0) { + dev_err(&spi->dev, "iio triggered buffer setup failed\n"); + goto err_suspend; + } + + ret = iio_device_register(indio_dev); + if (ret < 0) { + dev_err(&spi->dev, "iio_device_register failed\n"); + iio_triggered_buffer_cleanup(indio_dev); + goto err_suspend; + } + + return 0; + +err_suspend: + return bma220_deinit(spi); +} + +static int bma220_remove(struct spi_device *spi) +{ + struct iio_dev *indio_dev = spi_get_drvdata(spi); + + iio_device_unregister(indio_dev); + iio_triggered_buffer_cleanup(indio_dev); + + return bma220_deinit(spi); +} + +#ifdef CONFIG_PM_SLEEP +static int bma220_suspend(struct device *dev) +{ + struct bma220_data *data = + iio_priv(spi_get_drvdata(to_spi_device(dev))); + + /* The chip can be suspended/woken up by a simple register read. */ + return bma220_read_reg(data->spi_device, BMA220_REG_SUSPEND); +} + +static int bma220_resume(struct device *dev) +{ + struct bma220_data *data = + iio_priv(spi_get_drvdata(to_spi_device(dev))); + + return bma220_read_reg(data->spi_device, BMA220_REG_SUSPEND); +} + +static SIMPLE_DEV_PM_OPS(bma220_pm_ops, bma220_suspend, bma220_resume); + +#define BMA220_PM_OPS (&bma220_pm_ops) +#else +#define BMA220_PM_OPS NULL +#endif + +static const struct spi_device_id bma220_spi_id[] = { + {"bma220", 0}, + {} +}; + +static const struct acpi_device_id bma220_acpi_id[] = { + {"BMA0220", 0}, + {} +}; + +MODULE_DEVICE_TABLE(spi, bma220_spi_id); + +static struct spi_driver bma220_driver = { + .driver = { + .name = "bma220_spi", + .pm = BMA220_PM_OPS, + .acpi_match_table = ACPI_PTR(bma220_acpi_id), + }, + .probe = bma220_probe, + .remove = bma220_remove, + .id_table = bma220_spi_id, +}; + +module_spi_driver(bma220_driver); + +MODULE_AUTHOR("Tiberiu Breana "); +MODULE_DESCRIPTION("BMA220 acceleration sensor driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/accel/bmc150-accel-core.c b/drivers/iio/accel/bmc150-accel-core.c index 197e693e7e7b..bf17aae66145 100644 --- a/drivers/iio/accel/bmc150-accel-core.c +++ b/drivers/iio/accel/bmc150-accel-core.c @@ -901,7 +901,7 @@ static int __bmc150_accel_fifo_flush(struct iio_dev *indio_dev, */ if (!irq) { data->old_timestamp = data->timestamp; - data->timestamp = iio_get_time_ns(); + data->timestamp = iio_get_time_ns(indio_dev); } /* @@ -1303,7 +1303,7 @@ static irqreturn_t bmc150_accel_irq_handler(int irq, void *private) int i; data->old_timestamp = data->timestamp; - data->timestamp = iio_get_time_ns(); + data->timestamp = iio_get_time_ns(indio_dev); for (i = 0; i < BMC150_ACCEL_TRIGGERS; i++) { if (data->triggers[i].enabled) { diff --git a/drivers/iio/accel/kxcjk-1013.c b/drivers/iio/accel/kxcjk-1013.c index bfe219a8bea2..765a72362dc6 100644 --- a/drivers/iio/accel/kxcjk-1013.c +++ b/drivers/iio/accel/kxcjk-1013.c @@ -1129,7 +1129,7 @@ static irqreturn_t kxcjk1013_data_rdy_trig_poll(int irq, void *private) struct iio_dev *indio_dev = private; struct kxcjk1013_data *data = iio_priv(indio_dev); - data->timestamp = iio_get_time_ns(); + data->timestamp = iio_get_time_ns(indio_dev); if (data->dready_trigger_on) iio_trigger_poll(data->dready_trig); diff --git a/drivers/iio/accel/kxsd9.c b/drivers/iio/accel/kxsd9.c index 923f56598d4b..3a9f106787d2 100644 --- a/drivers/iio/accel/kxsd9.c +++ b/drivers/iio/accel/kxsd9.c @@ -81,7 +81,7 @@ static int kxsd9_write_scale(struct iio_dev *indio_dev, int micro) mutex_lock(&st->buf_lock); ret = spi_w8r8(st->us, KXSD9_READ(KXSD9_REG_CTRL_C)); - if (ret) + if (ret < 0) goto error_ret; st->tx[0] = KXSD9_WRITE(KXSD9_REG_CTRL_C); st->tx[1] = (ret & ~KXSD9_FS_MASK) | i; @@ -163,7 +163,7 @@ static int kxsd9_read_raw(struct iio_dev *indio_dev, break; case IIO_CHAN_INFO_SCALE: ret = spi_w8r8(st->us, KXSD9_READ(KXSD9_REG_CTRL_C)); - if (ret) + if (ret < 0) goto error_ret; *val2 = kxsd9_micro_scales[ret & KXSD9_FS_MASK]; ret = IIO_VAL_INT_PLUS_MICRO; diff --git a/drivers/iio/accel/mma7455_core.c b/drivers/iio/accel/mma7455_core.c index c902f54c23f5..6551085bedd7 100644 --- a/drivers/iio/accel/mma7455_core.c +++ b/drivers/iio/accel/mma7455_core.c @@ -97,7 +97,8 @@ static irqreturn_t mma7455_trigger_handler(int irq, void *p) if (ret) goto done; - iio_push_to_buffers_with_timestamp(indio_dev, buf, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, buf, + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); diff --git a/drivers/iio/accel/mma7660.c b/drivers/iio/accel/mma7660.c new file mode 100644 index 000000000000..0acdee516973 --- /dev/null +++ b/drivers/iio/accel/mma7660.c @@ -0,0 +1,277 @@ +/** + * Freescale MMA7660FC 3-Axis Accelerometer + * + * Copyright (c) 2016, Intel Corporation. + * + * This file is subject to the terms and conditions of version 2 of + * the GNU General Public License. See the file COPYING in the main + * directory of this archive for more details. + * + * IIO driver for Freescale MMA7660FC; 7-bit I2C address: 0x4c. + */ + +#include +#include +#include +#include +#include + +#define MMA7660_DRIVER_NAME "mma7660" + +#define MMA7660_REG_XOUT 0x00 +#define MMA7660_REG_YOUT 0x01 +#define MMA7660_REG_ZOUT 0x02 +#define MMA7660_REG_OUT_BIT_ALERT BIT(6) + +#define MMA7660_REG_MODE 0x07 +#define MMA7660_REG_MODE_BIT_MODE BIT(0) +#define MMA7660_REG_MODE_BIT_TON BIT(2) + +#define MMA7660_I2C_READ_RETRIES 5 + +/* + * The accelerometer has one measurement range: + * + * -1.5g - +1.5g (6-bit, signed) + * + * scale = (1.5 + 1.5) * 9.81 / (2^6 - 1) = 0.467142857 + */ + +#define MMA7660_SCALE_AVAIL "0.467142857" + +const int mma7660_nscale = 467142857; + +#define MMA7660_CHANNEL(reg, axis) { \ + .type = IIO_ACCEL, \ + .address = reg, \ + .modified = 1, \ + .channel2 = IIO_MOD_##axis, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ +} + +static const struct iio_chan_spec mma7660_channels[] = { + MMA7660_CHANNEL(MMA7660_REG_XOUT, X), + MMA7660_CHANNEL(MMA7660_REG_YOUT, Y), + MMA7660_CHANNEL(MMA7660_REG_ZOUT, Z), +}; + +enum mma7660_mode { + MMA7660_MODE_STANDBY, + MMA7660_MODE_ACTIVE +}; + +struct mma7660_data { + struct i2c_client *client; + struct mutex lock; + enum mma7660_mode mode; +}; + +static IIO_CONST_ATTR(in_accel_scale_available, MMA7660_SCALE_AVAIL); + +static struct attribute *mma7660_attributes[] = { + &iio_const_attr_in_accel_scale_available.dev_attr.attr, + NULL, +}; + +static const struct attribute_group mma7660_attribute_group = { + .attrs = mma7660_attributes +}; + +static int mma7660_set_mode(struct mma7660_data *data, + enum mma7660_mode mode) +{ + int ret; + struct i2c_client *client = data->client; + + if (mode == data->mode) + return 0; + + ret = i2c_smbus_read_byte_data(client, MMA7660_REG_MODE); + if (ret < 0) { + dev_err(&client->dev, "failed to read sensor mode\n"); + return ret; + } + + if (mode == MMA7660_MODE_ACTIVE) { + ret &= ~MMA7660_REG_MODE_BIT_TON; + ret |= MMA7660_REG_MODE_BIT_MODE; + } else { + ret &= ~MMA7660_REG_MODE_BIT_TON; + ret &= ~MMA7660_REG_MODE_BIT_MODE; + } + + ret = i2c_smbus_write_byte_data(client, MMA7660_REG_MODE, ret); + if (ret < 0) { + dev_err(&client->dev, "failed to change sensor mode\n"); + return ret; + } + + data->mode = mode; + + return ret; +} + +static int mma7660_read_accel(struct mma7660_data *data, u8 address) +{ + int ret, retries = MMA7660_I2C_READ_RETRIES; + struct i2c_client *client = data->client; + + /* + * Read data. If the Alert bit is set, the register was read at + * the same time as the device was attempting to update the content. + * The solution is to read the register again. Do this only + * MMA7660_I2C_READ_RETRIES times to avoid spending too much time + * in the kernel. + */ + do { + ret = i2c_smbus_read_byte_data(client, address); + if (ret < 0) { + dev_err(&client->dev, "register read failed\n"); + return ret; + } + } while (retries-- > 0 && ret & MMA7660_REG_OUT_BIT_ALERT); + + if (ret & MMA7660_REG_OUT_BIT_ALERT) { + dev_err(&client->dev, "all register read retries failed\n"); + return -ETIMEDOUT; + } + + return ret; +} + +static int mma7660_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct mma7660_data *data = iio_priv(indio_dev); + int ret; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + mutex_lock(&data->lock); + ret = mma7660_read_accel(data, chan->address); + mutex_unlock(&data->lock); + if (ret < 0) + return ret; + *val = sign_extend32(ret, 5); + return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + *val = 0; + *val2 = mma7660_nscale; + return IIO_VAL_INT_PLUS_NANO; + default: + return -EINVAL; + } + + return -EINVAL; +} + +static const struct iio_info mma7660_info = { + .driver_module = THIS_MODULE, + .read_raw = mma7660_read_raw, + .attrs = &mma7660_attribute_group, +}; + +static int mma7660_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + int ret; + struct iio_dev *indio_dev; + struct mma7660_data *data; + + indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data)); + if (!indio_dev) { + dev_err(&client->dev, "iio allocation failed!\n"); + return -ENOMEM; + } + + data = iio_priv(indio_dev); + data->client = client; + i2c_set_clientdata(client, indio_dev); + mutex_init(&data->lock); + data->mode = MMA7660_MODE_STANDBY; + + indio_dev->dev.parent = &client->dev; + indio_dev->info = &mma7660_info; + indio_dev->name = MMA7660_DRIVER_NAME; + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->channels = mma7660_channels; + indio_dev->num_channels = ARRAY_SIZE(mma7660_channels); + + ret = mma7660_set_mode(data, MMA7660_MODE_ACTIVE); + if (ret < 0) + return ret; + + ret = iio_device_register(indio_dev); + if (ret < 0) { + dev_err(&client->dev, "device_register failed\n"); + mma7660_set_mode(data, MMA7660_MODE_STANDBY); + } + + return ret; +} + +static int mma7660_remove(struct i2c_client *client) +{ + struct iio_dev *indio_dev = i2c_get_clientdata(client); + + iio_device_unregister(indio_dev); + + return mma7660_set_mode(iio_priv(indio_dev), MMA7660_MODE_STANDBY); +} + +#ifdef CONFIG_PM_SLEEP +static int mma7660_suspend(struct device *dev) +{ + struct mma7660_data *data; + + data = iio_priv(i2c_get_clientdata(to_i2c_client(dev))); + + return mma7660_set_mode(data, MMA7660_MODE_STANDBY); +} + +static int mma7660_resume(struct device *dev) +{ + struct mma7660_data *data; + + data = iio_priv(i2c_get_clientdata(to_i2c_client(dev))); + + return mma7660_set_mode(data, MMA7660_MODE_ACTIVE); +} + +static SIMPLE_DEV_PM_OPS(mma7660_pm_ops, mma7660_suspend, mma7660_resume); + +#define MMA7660_PM_OPS (&mma7660_pm_ops) +#else +#define MMA7660_PM_OPS NULL +#endif + +static const struct i2c_device_id mma7660_i2c_id[] = { + {"mma7660", 0}, + {} +}; + +static const struct acpi_device_id mma7660_acpi_id[] = { + {"MMA7660", 0}, + {} +}; + +MODULE_DEVICE_TABLE(acpi, mma7660_acpi_id); + +static struct i2c_driver mma7660_driver = { + .driver = { + .name = "mma7660", + .pm = MMA7660_PM_OPS, + .acpi_match_table = ACPI_PTR(mma7660_acpi_id), + }, + .probe = mma7660_probe, + .remove = mma7660_remove, + .id_table = mma7660_i2c_id, +}; + +module_i2c_driver(mma7660_driver); + +MODULE_AUTHOR("Constantin Musca "); +MODULE_DESCRIPTION("Freescale MMA7660FC 3-Axis Accelerometer driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c index e225d3c53bd5..d41e1b588e68 100644 --- a/drivers/iio/accel/mma8452.c +++ b/drivers/iio/accel/mma8452.c @@ -1,22 +1,22 @@ /* - * mma8452.c - Support for following Freescale 3-axis accelerometers: + * mma8452.c - Support for following Freescale / NXP 3-axis accelerometers: * - * MMA8451Q (14 bit) - * MMA8452Q (12 bit) - * MMA8453Q (10 bit) - * MMA8652FC (12 bit) - * MMA8653FC (10 bit) - * FXLS8471Q (14 bit) + * device name digital output 7-bit I2C slave address (pin selectable) + * --------------------------------------------------------------------- + * MMA8451Q 14 bit 0x1c / 0x1d + * MMA8452Q 12 bit 0x1c / 0x1d + * MMA8453Q 10 bit 0x1c / 0x1d + * MMA8652FC 12 bit 0x1d + * MMA8653FC 10 bit 0x1d + * FXLS8471Q 14 bit 0x1e / 0x1d / 0x1c / 0x1f * - * Copyright 2015 Martin Kepplinger + * Copyright 2015 Martin Kepplinger * Copyright 2014 Peter Meerwald * * This file is subject to the terms and conditions of version 2 of * the GNU General Public License. See the file COPYING in the main * directory of this archive for more details. * - * 7-bit I2C slave address 0x1c/0x1d (pin selectable) - * * TODO: orientation events */ @@ -76,6 +76,8 @@ #define MMA8452_CTRL_DR_DEFAULT 0x4 /* 50 Hz sample frequency */ #define MMA8452_CTRL_REG2 0x2b #define MMA8452_CTRL_REG2_RST BIT(6) +#define MMA8452_CTRL_REG2_MODS_SHIFT 3 +#define MMA8452_CTRL_REG2_MODS_MASK 0x1b #define MMA8452_CTRL_REG4 0x2d #define MMA8452_CTRL_REG5 0x2e #define MMA8452_OFF_X 0x2f @@ -106,7 +108,7 @@ struct mma8452_data { }; /** - * struct mma_chip_info - chip specific data for Freescale's accelerometers + * struct mma_chip_info - chip specific data * @chip_id: WHO_AM_I register's value * @channels: struct iio_chan_spec matching the device's * capabilities @@ -257,20 +259,17 @@ static const int mma8452_samp_freq[8][2] = { {6, 250000}, {1, 560000} }; -/* Datasheet table 35 (step time vs sample frequency) */ -static const int mma8452_transient_time_step_us[8] = { - 1250, - 2500, - 5000, - 10000, - 20000, - 20000, - 20000, - 20000 +/* Datasheet table: step time "Relationship with the ODR" (sample frequency) */ +static const int mma8452_transient_time_step_us[4][8] = { + { 1250, 2500, 5000, 10000, 20000, 20000, 20000, 20000 }, /* normal */ + { 1250, 2500, 5000, 10000, 20000, 80000, 80000, 80000 }, /* l p l n */ + { 1250, 2500, 2500, 2500, 2500, 2500, 2500, 2500 }, /* high res*/ + { 1250, 2500, 5000, 10000, 20000, 80000, 160000, 160000 } /* l p */ }; -/* Datasheet table 18 (normal mode) */ -static const int mma8452_hp_filter_cutoff[8][4][2] = { +/* Datasheet table "High-Pass Filter Cutoff Options" */ +static const int mma8452_hp_filter_cutoff[4][8][4][2] = { + { /* normal */ { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, /* 800 Hz sample */ { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, /* 400 Hz sample */ { {8, 0}, {4, 0}, {2, 0}, {1, 0} }, /* 200 Hz sample */ @@ -279,8 +278,61 @@ static const int mma8452_hp_filter_cutoff[8][4][2] = { { {2, 0}, {1, 0}, {0, 500000}, {0, 250000} }, /* 12.5 Hz sample */ { {2, 0}, {1, 0}, {0, 500000}, {0, 250000} }, /* 6.25 Hz sample */ { {2, 0}, {1, 0}, {0, 500000}, {0, 250000} } /* 1.56 Hz sample */ + }, + { /* low noise low power */ + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {8, 0}, {4, 0}, {2, 0}, {1, 0} }, + { {4, 0}, {2, 0}, {1, 0}, {0, 500000} }, + { {2, 0}, {1, 0}, {0, 500000}, {0, 250000} }, + { {0, 500000}, {0, 250000}, {0, 125000}, {0, 063000} }, + { {0, 500000}, {0, 250000}, {0, 125000}, {0, 063000} }, + { {0, 500000}, {0, 250000}, {0, 125000}, {0, 063000} } + }, + { /* high resolution */ + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {16, 0}, {8, 0}, {4, 0}, {2, 0} } + }, + { /* low power */ + { {16, 0}, {8, 0}, {4, 0}, {2, 0} }, + { {8, 0}, {4, 0}, {2, 0}, {1, 0} }, + { {4, 0}, {2, 0}, {1, 0}, {0, 500000} }, + { {2, 0}, {1, 0}, {0, 500000}, {0, 250000} }, + { {1, 0}, {0, 500000}, {0, 250000}, {0, 125000} }, + { {0, 250000}, {0, 125000}, {0, 063000}, {0, 031000} }, + { {0, 250000}, {0, 125000}, {0, 063000}, {0, 031000} }, + { {0, 250000}, {0, 125000}, {0, 063000}, {0, 031000} } + } }; +/* Datasheet table "MODS Oversampling modes averaging values at each ODR" */ +static const u16 mma8452_os_ratio[4][8] = { + /* 800 Hz, 400 Hz, ... , 1.56 Hz */ + { 2, 4, 4, 4, 4, 16, 32, 128 }, /* normal */ + { 2, 4, 4, 4, 4, 4, 8, 32 }, /* low power low noise */ + { 2, 4, 8, 16, 32, 128, 256, 1024 }, /* high resolution */ + { 2, 2, 2, 2, 2, 2, 4, 16 } /* low power */ +}; + +static int mma8452_get_power_mode(struct mma8452_data *data) +{ + int reg; + + reg = i2c_smbus_read_byte_data(data->client, + MMA8452_CTRL_REG2); + if (reg < 0) + return reg; + + return ((reg & MMA8452_CTRL_REG2_MODS_MASK) >> + MMA8452_CTRL_REG2_MODS_SHIFT); +} + static ssize_t mma8452_show_samp_freq_avail(struct device *dev, struct device_attribute *attr, char *buf) @@ -303,13 +355,42 @@ static ssize_t mma8452_show_scale_avail(struct device *dev, static ssize_t mma8452_show_hp_cutoff_avail(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct mma8452_data *data = iio_priv(indio_dev); + int i, j; + + i = mma8452_get_odr_index(data); + j = mma8452_get_power_mode(data); + if (j < 0) + return j; + + return mma8452_show_int_plus_micros(buf, mma8452_hp_filter_cutoff[j][i], + ARRAY_SIZE(mma8452_hp_filter_cutoff[0][0])); +} + +static ssize_t mma8452_show_os_ratio_avail(struct device *dev, + struct device_attribute *attr, + char *buf) { struct iio_dev *indio_dev = dev_to_iio_dev(dev); struct mma8452_data *data = iio_priv(indio_dev); int i = mma8452_get_odr_index(data); + int j; + u16 val = 0; + size_t len = 0; - return mma8452_show_int_plus_micros(buf, mma8452_hp_filter_cutoff[i], - ARRAY_SIZE(mma8452_hp_filter_cutoff[0])); + for (j = 0; j < ARRAY_SIZE(mma8452_os_ratio); j++) { + if (val == mma8452_os_ratio[j][i]) + continue; + + val = mma8452_os_ratio[j][i]; + + len += scnprintf(buf + len, PAGE_SIZE - len, "%d ", val); + } + buf[len - 1] = '\n'; + + return len; } static IIO_DEV_ATTR_SAMP_FREQ_AVAIL(mma8452_show_samp_freq_avail); @@ -317,6 +398,8 @@ static IIO_DEVICE_ATTR(in_accel_scale_available, S_IRUGO, mma8452_show_scale_avail, NULL, 0); static IIO_DEVICE_ATTR(in_accel_filter_high_pass_3db_frequency_available, S_IRUGO, mma8452_show_hp_cutoff_avail, NULL, 0); +static IIO_DEVICE_ATTR(in_accel_oversampling_ratio_available, S_IRUGO, + mma8452_show_os_ratio_avail, NULL, 0); static int mma8452_get_samp_freq_index(struct mma8452_data *data, int val, int val2) @@ -335,24 +418,33 @@ static int mma8452_get_scale_index(struct mma8452_data *data, int val, int val2) static int mma8452_get_hp_filter_index(struct mma8452_data *data, int val, int val2) { - int i = mma8452_get_odr_index(data); + int i, j; + + i = mma8452_get_odr_index(data); + j = mma8452_get_power_mode(data); + if (j < 0) + return j; - return mma8452_get_int_plus_micros_index(mma8452_hp_filter_cutoff[i], - ARRAY_SIZE(mma8452_hp_filter_cutoff[0]), val, val2); + return mma8452_get_int_plus_micros_index(mma8452_hp_filter_cutoff[j][i], + ARRAY_SIZE(mma8452_hp_filter_cutoff[0][0]), val, val2); } static int mma8452_read_hp_filter(struct mma8452_data *data, int *hz, int *uHz) { - int i, ret; + int j, i, ret; ret = i2c_smbus_read_byte_data(data->client, MMA8452_HP_FILTER_CUTOFF); if (ret < 0) return ret; i = mma8452_get_odr_index(data); + j = mma8452_get_power_mode(data); + if (j < 0) + return j; + ret &= MMA8452_HP_FILTER_CUTOFF_SEL_MASK; - *hz = mma8452_hp_filter_cutoff[i][ret][0]; - *uHz = mma8452_hp_filter_cutoff[i][ret][1]; + *hz = mma8452_hp_filter_cutoff[j][i][ret][0]; + *uHz = mma8452_hp_filter_cutoff[j][i][ret][1]; return 0; } @@ -414,6 +506,15 @@ static int mma8452_read_raw(struct iio_dev *indio_dev, } return IIO_VAL_INT_PLUS_MICRO; + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + ret = mma8452_get_power_mode(data); + if (ret < 0) + return ret; + + i = mma8452_get_odr_index(data); + + *val = mma8452_os_ratio[ret][i]; + return IIO_VAL_INT; } return -EINVAL; @@ -480,6 +581,21 @@ fail: return ret; } +static int mma8452_set_power_mode(struct mma8452_data *data, u8 mode) +{ + int reg; + + reg = i2c_smbus_read_byte_data(data->client, + MMA8452_CTRL_REG2); + if (reg < 0) + return reg; + + reg &= ~MMA8452_CTRL_REG2_MODS_MASK; + reg |= mode << MMA8452_CTRL_REG2_MODS_SHIFT; + + return mma8452_change_config(data, MMA8452_CTRL_REG2, reg); +} + /* returns >0 if in freefall mode, 0 if not or <0 if an error occurred */ static int mma8452_freefall_mode_enabled(struct mma8452_data *data) { @@ -518,11 +634,7 @@ static int mma8452_set_freefall_mode(struct mma8452_data *data, bool state) val |= MMA8452_FF_MT_CFG_OAE; } - val = mma8452_change_config(data, chip->ev_cfg, val); - if (val) - return val; - - return 0; + return mma8452_change_config(data, chip->ev_cfg, val); } static int mma8452_set_hp_filter_frequency(struct mma8452_data *data, @@ -597,6 +709,14 @@ static int mma8452_write_raw(struct iio_dev *indio_dev, return mma8452_change_config(data, MMA8452_DATA_CFG, data->data_cfg); + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + ret = mma8452_get_odr_index(data); + + for (i = 0; i < ARRAY_SIZE(mma8452_os_ratio); i++) { + if (mma8452_os_ratio[i][ret] == val) + return mma8452_set_power_mode(data, i); + } + default: return -EINVAL; } @@ -610,7 +730,7 @@ static int mma8452_read_thresh(struct iio_dev *indio_dev, int *val, int *val2) { struct mma8452_data *data = iio_priv(indio_dev); - int ret, us; + int ret, us, power_mode; switch (info) { case IIO_EV_INFO_VALUE: @@ -629,7 +749,11 @@ static int mma8452_read_thresh(struct iio_dev *indio_dev, if (ret < 0) return ret; - us = ret * mma8452_transient_time_step_us[ + power_mode = mma8452_get_power_mode(data); + if (power_mode < 0) + return power_mode; + + us = ret * mma8452_transient_time_step_us[power_mode][ mma8452_get_odr_index(data)]; *val = us / USEC_PER_SEC; *val2 = us % USEC_PER_SEC; @@ -677,8 +801,12 @@ static int mma8452_write_thresh(struct iio_dev *indio_dev, val); case IIO_EV_INFO_PERIOD: + ret = mma8452_get_power_mode(data); + if (ret < 0) + return ret; + steps = (val * USEC_PER_SEC + val2) / - mma8452_transient_time_step_us[ + mma8452_transient_time_step_us[ret][ mma8452_get_odr_index(data)]; if (steps < 0 || steps > 0xff) @@ -785,7 +913,7 @@ static int mma8452_write_event_config(struct iio_dev *indio_dev, static void mma8452_transient_interrupt(struct iio_dev *indio_dev) { struct mma8452_data *data = iio_priv(indio_dev); - s64 ts = iio_get_time_ns(); + s64 ts = iio_get_time_ns(indio_dev); int src; src = i2c_smbus_read_byte_data(data->client, data->chip_info->ev_src); @@ -865,7 +993,7 @@ static irqreturn_t mma8452_trigger_handler(int irq, void *p) goto done; iio_push_to_buffers_with_timestamp(indio_dev, buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -978,7 +1106,8 @@ static struct attribute_group mma8452_event_attribute_group = { BIT(IIO_CHAN_INFO_CALIBBIAS), \ .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SAMP_FREQ) | \ BIT(IIO_CHAN_INFO_SCALE) | \ - BIT(IIO_CHAN_INFO_HIGH_PASS_FILTER_3DB_FREQUENCY), \ + BIT(IIO_CHAN_INFO_HIGH_PASS_FILTER_3DB_FREQUENCY) | \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ .scan_index = idx, \ .scan_type = { \ .sign = 's', \ @@ -998,7 +1127,8 @@ static struct attribute_group mma8452_event_attribute_group = { .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \ BIT(IIO_CHAN_INFO_CALIBBIAS), \ .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SAMP_FREQ) | \ - BIT(IIO_CHAN_INFO_SCALE), \ + BIT(IIO_CHAN_INFO_SCALE) | \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ .scan_index = idx, \ .scan_type = { \ .sign = 's', \ @@ -1171,6 +1301,7 @@ static struct attribute *mma8452_attributes[] = { &iio_dev_attr_sampling_frequency_available.dev_attr.attr, &iio_dev_attr_in_accel_scale_available.dev_attr.attr, &iio_dev_attr_in_accel_filter_high_pass_3db_frequency_available.dev_attr.attr, + &iio_dev_attr_in_accel_oversampling_ratio_available.dev_attr.attr, NULL }; @@ -1444,8 +1575,8 @@ static int mma8452_probe(struct i2c_client *client, goto buffer_cleanup; ret = mma8452_set_freefall_mode(data, false); - if (ret) - return ret; + if (ret < 0) + goto buffer_cleanup; return 0; @@ -1558,5 +1689,5 @@ static struct i2c_driver mma8452_driver = { module_i2c_driver(mma8452_driver); MODULE_AUTHOR("Peter Meerwald "); -MODULE_DESCRIPTION("Freescale MMA8452 accelerometer driver"); +MODULE_DESCRIPTION("Freescale / NXP MMA8452 accelerometer driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/iio/accel/mma9551.c b/drivers/iio/accel/mma9551.c index d899a4d4307f..bf2704435629 100644 --- a/drivers/iio/accel/mma9551.c +++ b/drivers/iio/accel/mma9551.c @@ -391,7 +391,7 @@ static irqreturn_t mma9551_event_handler(int irq, void *private) iio_push_event(indio_dev, IIO_MOD_EVENT_CODE(IIO_INCLI, 0, (mma_axis + 1), IIO_EV_TYPE_ROC, IIO_EV_DIR_RISING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); out: mutex_unlock(&data->mutex); diff --git a/drivers/iio/accel/mma9553.c b/drivers/iio/accel/mma9553.c index bb05f3efddca..36bf19733be0 100644 --- a/drivers/iio/accel/mma9553.c +++ b/drivers/iio/accel/mma9553.c @@ -1001,7 +1001,7 @@ static irqreturn_t mma9553_irq_handler(int irq, void *private) struct iio_dev *indio_dev = private; struct mma9553_data *data = iio_priv(indio_dev); - data->timestamp = iio_get_time_ns(); + data->timestamp = iio_get_time_ns(indio_dev); /* * Since we only configure the interrupt pin when an * event is enabled, we are sure we have at least diff --git a/drivers/iio/accel/st_accel.h b/drivers/iio/accel/st_accel.h index 57f83a67948c..f8dfdb690563 100644 --- a/drivers/iio/accel/st_accel.h +++ b/drivers/iio/accel/st_accel.h @@ -29,6 +29,7 @@ #define LSM330_ACCEL_DEV_NAME "lsm330_accel" #define LSM303AGR_ACCEL_DEV_NAME "lsm303agr_accel" #define LIS2DH12_ACCEL_DEV_NAME "lis2dh12_accel" +#define LIS3L02DQ_ACCEL_DEV_NAME "lis3l02dq" /** * struct st_sensors_platform_data - default accel platform data diff --git a/drivers/iio/accel/st_accel_core.c b/drivers/iio/accel/st_accel_core.c index 4d95bfc4786c..da3fb069ec5c 100644 --- a/drivers/iio/accel/st_accel_core.c +++ b/drivers/iio/accel/st_accel_core.c @@ -215,6 +215,22 @@ #define ST_ACCEL_6_IHL_IRQ_MASK 0x80 #define ST_ACCEL_6_MULTIREAD_BIT true +/* CUSTOM VALUES FOR SENSOR 7 */ +#define ST_ACCEL_7_ODR_ADDR 0x20 +#define ST_ACCEL_7_ODR_MASK 0x30 +#define ST_ACCEL_7_ODR_AVL_280HZ_VAL 0x00 +#define ST_ACCEL_7_ODR_AVL_560HZ_VAL 0x01 +#define ST_ACCEL_7_ODR_AVL_1120HZ_VAL 0x02 +#define ST_ACCEL_7_ODR_AVL_4480HZ_VAL 0x03 +#define ST_ACCEL_7_PW_ADDR 0x20 +#define ST_ACCEL_7_PW_MASK 0xc0 +#define ST_ACCEL_7_FS_AVL_2_GAIN IIO_G_TO_M_S_2(488) +#define ST_ACCEL_7_BDU_ADDR 0x21 +#define ST_ACCEL_7_BDU_MASK 0x40 +#define ST_ACCEL_7_DRDY_IRQ_ADDR 0x21 +#define ST_ACCEL_7_DRDY_IRQ_INT1_MASK 0x04 +#define ST_ACCEL_7_MULTIREAD_BIT false + static const struct iio_chan_spec st_accel_8bit_channels[] = { ST_SENSORS_LSM_CHANNELS(IIO_ACCEL, BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), @@ -662,6 +678,54 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = { .multi_read_bit = ST_ACCEL_6_MULTIREAD_BIT, .bootime = 2, }, + { + /* No WAI register present */ + .sensors_supported = { + [0] = LIS3L02DQ_ACCEL_DEV_NAME, + }, + .ch = (struct iio_chan_spec *)st_accel_12bit_channels, + .odr = { + .addr = ST_ACCEL_7_ODR_ADDR, + .mask = ST_ACCEL_7_ODR_MASK, + .odr_avl = { + { 280, ST_ACCEL_7_ODR_AVL_280HZ_VAL, }, + { 560, ST_ACCEL_7_ODR_AVL_560HZ_VAL, }, + { 1120, ST_ACCEL_7_ODR_AVL_1120HZ_VAL, }, + { 4480, ST_ACCEL_7_ODR_AVL_4480HZ_VAL, }, + }, + }, + .pw = { + .addr = ST_ACCEL_7_PW_ADDR, + .mask = ST_ACCEL_7_PW_MASK, + .value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE, + .value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE, + }, + .enable_axis = { + .addr = ST_SENSORS_DEFAULT_AXIS_ADDR, + .mask = ST_SENSORS_DEFAULT_AXIS_MASK, + }, + .fs = { + .fs_avl = { + [0] = { + .num = ST_ACCEL_FS_AVL_2G, + .gain = ST_ACCEL_7_FS_AVL_2_GAIN, + }, + }, + }, + /* + * The part has a BDU bit but if set the data is never + * updated so don't set it. + */ + .bdu = { + }, + .drdy_irq = { + .addr = ST_ACCEL_7_DRDY_IRQ_ADDR, + .mask_int1 = ST_ACCEL_7_DRDY_IRQ_INT1_MASK, + .addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR, + }, + .multi_read_bit = ST_ACCEL_7_MULTIREAD_BIT, + .bootime = 2, + }, }; static int st_accel_read_raw(struct iio_dev *indio_dev, @@ -758,13 +822,15 @@ int st_accel_common_probe(struct iio_dev *indio_dev) indio_dev->info = &accel_info; mutex_init(&adata->tb.buf_lock); - st_sensors_power_enable(indio_dev); + err = st_sensors_power_enable(indio_dev); + if (err) + return err; err = st_sensors_check_device_support(indio_dev, ARRAY_SIZE(st_accel_sensors_settings), st_accel_sensors_settings); if (err < 0) - return err; + goto st_accel_power_off; adata->num_data_channels = ST_ACCEL_NUMBER_DATA_CHANNELS; adata->multiread_bit = adata->sensor_settings->multi_read_bit; @@ -781,11 +847,11 @@ int st_accel_common_probe(struct iio_dev *indio_dev) err = st_sensors_init_sensor(indio_dev, adata->dev->platform_data); if (err < 0) - return err; + goto st_accel_power_off; err = st_accel_allocate_ring(indio_dev); if (err < 0) - return err; + goto st_accel_power_off; if (irq > 0) { err = st_sensors_allocate_trigger(indio_dev, @@ -808,6 +874,8 @@ st_accel_device_register_error: st_sensors_deallocate_trigger(indio_dev); st_accel_probe_trigger_error: st_accel_deallocate_ring(indio_dev); +st_accel_power_off: + st_sensors_power_disable(indio_dev); return err; } diff --git a/drivers/iio/accel/st_accel_i2c.c b/drivers/iio/accel/st_accel_i2c.c index 7333ee9fb11b..e9d427a5df7c 100644 --- a/drivers/iio/accel/st_accel_i2c.c +++ b/drivers/iio/accel/st_accel_i2c.c @@ -80,6 +80,10 @@ static const struct of_device_id st_accel_of_match[] = { .compatible = "st,h3lis331dl-accel", .data = H3LIS331DL_DRIVER_NAME, }, + { + .compatible = "st,lis3l02dq", + .data = LIS3L02DQ_ACCEL_DEV_NAME, + }, {}, }; MODULE_DEVICE_TABLE(of, st_accel_of_match); @@ -130,6 +134,7 @@ static const struct i2c_device_id st_accel_id_table[] = { { LSM330_ACCEL_DEV_NAME }, { LSM303AGR_ACCEL_DEV_NAME }, { LIS2DH12_ACCEL_DEV_NAME }, + { LIS3L02DQ_ACCEL_DEV_NAME }, {}, }; MODULE_DEVICE_TABLE(i2c, st_accel_id_table); diff --git a/drivers/iio/accel/st_accel_spi.c b/drivers/iio/accel/st_accel_spi.c index fcd5847a3fd3..efd43941d45d 100644 --- a/drivers/iio/accel/st_accel_spi.c +++ b/drivers/iio/accel/st_accel_spi.c @@ -59,6 +59,7 @@ static const struct spi_device_id st_accel_id_table[] = { { LSM330_ACCEL_DEV_NAME }, { LSM303AGR_ACCEL_DEV_NAME }, { LIS2DH12_ACCEL_DEV_NAME }, + { LIS3L02DQ_ACCEL_DEV_NAME }, {}, }; MODULE_DEVICE_TABLE(spi, st_accel_id_table); diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index 25378c5882e2..1de31bdd4ce4 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -153,6 +153,18 @@ config AXP288_ADC To compile this driver as a module, choose M here: the module will be called axp288_adc. +config BCM_IPROC_ADC + tristate "Broadcom IPROC ADC driver" + depends on ARCH_BCM_IPROC || COMPILE_TEST + depends on MFD_SYSCON + default ARCH_BCM_CYGNUS + help + Say Y here if you want to add support for the Broadcom static + ADC driver. + + Broadcom iProc ADC driver. Broadcom iProc ADC controller has 8 + channels. The driver allows the user to read voltage values. + config BERLIN2_ADC tristate "Marvell Berlin2 ADC driver" depends on ARCH_BERLIN diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile index 38638d46f972..0ba0d500eedb 100644 --- a/drivers/iio/adc/Makefile +++ b/drivers/iio/adc/Makefile @@ -16,6 +16,7 @@ obj-$(CONFIG_AD799X) += ad799x.o obj-$(CONFIG_AT91_ADC) += at91_adc.o obj-$(CONFIG_AT91_SAMA5D2_ADC) += at91-sama5d2_adc.o obj-$(CONFIG_AXP288_ADC) += axp288_adc.o +obj-$(CONFIG_BCM_IPROC_ADC) += bcm_iproc_adc.o obj-$(CONFIG_BERLIN2_ADC) += berlin2-adc.o obj-$(CONFIG_CC10001_ADC) += cc10001_adc.o obj-$(CONFIG_DA9150_GPADC) += da9150-gpadc.o diff --git a/drivers/iio/adc/ad7266.c b/drivers/iio/adc/ad7266.c index 21e19b60e2b9..c0f6a98fd9bd 100644 --- a/drivers/iio/adc/ad7266.c +++ b/drivers/iio/adc/ad7266.c @@ -154,12 +154,11 @@ static int ad7266_read_raw(struct iio_dev *indio_dev, switch (m) { case IIO_CHAN_INFO_RAW: - if (iio_buffer_enabled(indio_dev)) - return -EBUSY; - - ret = ad7266_read_single(st, val, chan->address); + ret = iio_device_claim_direct_mode(indio_dev); if (ret) return ret; + ret = ad7266_read_single(st, val, chan->address); + iio_device_release_direct_mode(indio_dev); *val = (*val >> 2) & 0xfff; if (chan->scan_type.sign == 's') @@ -396,8 +395,8 @@ static int ad7266_probe(struct spi_device *spi) st = iio_priv(indio_dev); - st->reg = devm_regulator_get(&spi->dev, "vref"); - if (!IS_ERR_OR_NULL(st->reg)) { + st->reg = devm_regulator_get_optional(&spi->dev, "vref"); + if (!IS_ERR(st->reg)) { ret = regulator_enable(st->reg); if (ret) return ret; @@ -408,6 +407,9 @@ static int ad7266_probe(struct spi_device *spi) st->vref_mv = ret / 1000; } else { + /* Any other error indicates that the regulator does exist */ + if (PTR_ERR(st->reg) != -ENODEV) + return PTR_ERR(st->reg); /* Use internal reference */ st->vref_mv = 2500; } @@ -438,6 +440,7 @@ static int ad7266_probe(struct spi_device *spi) st->spi = spi; indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->name = spi_get_device_id(spi)->name; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->info = &ad7266_info; diff --git a/drivers/iio/adc/ad7291.c b/drivers/iio/adc/ad7291.c index c0eabf156702..1d90b02732bb 100644 --- a/drivers/iio/adc/ad7291.c +++ b/drivers/iio/adc/ad7291.c @@ -115,7 +115,7 @@ static irqreturn_t ad7291_event_handler(int irq, void *private) u16 t_status, v_status; u16 command; int i; - s64 timestamp = iio_get_time_ns(); + s64 timestamp = iio_get_time_ns(indio_dev); if (ad7291_i2c_read(chip, AD7291_T_ALERT_STATUS, &t_status)) return IRQ_HANDLED; @@ -505,6 +505,7 @@ static int ad7291_probe(struct i2c_client *client, indio_dev->num_channels = ARRAY_SIZE(ad7291_channels); indio_dev->dev.parent = &client->dev; + indio_dev->dev.of_node = client->dev.of_node; indio_dev->info = &ad7291_info; indio_dev->modes = INDIO_DIRECT_MODE; diff --git a/drivers/iio/adc/ad7298.c b/drivers/iio/adc/ad7298.c index 62bb8f7ce4a0..10ec8fce395f 100644 --- a/drivers/iio/adc/ad7298.c +++ b/drivers/iio/adc/ad7298.c @@ -163,7 +163,7 @@ static irqreturn_t ad7298_trigger_handler(int irq, void *p) goto done; iio_push_to_buffers_with_timestamp(indio_dev, st->rx_buf, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -315,6 +315,7 @@ static int ad7298_probe(struct spi_device *spi) indio_dev->name = spi_get_device_id(spi)->name; indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = ad7298_channels; indio_dev->num_channels = ARRAY_SIZE(ad7298_channels); diff --git a/drivers/iio/adc/ad7476.c b/drivers/iio/adc/ad7476.c index be85c2a0ad97..b7ecf9aab90f 100644 --- a/drivers/iio/adc/ad7476.c +++ b/drivers/iio/adc/ad7476.c @@ -70,7 +70,7 @@ static irqreturn_t ad7476_trigger_handler(int irq, void *p) goto done; iio_push_to_buffers_with_timestamp(indio_dev, st->data, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -106,12 +106,11 @@ static int ad7476_read_raw(struct iio_dev *indio_dev, switch (m) { case IIO_CHAN_INFO_RAW: - mutex_lock(&indio_dev->mlock); - if (iio_buffer_enabled(indio_dev)) - ret = -EBUSY; - else - ret = ad7476_scan_direct(st); - mutex_unlock(&indio_dev->mlock); + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + ret = ad7476_scan_direct(st); + iio_device_release_direct_mode(indio_dev); if (ret < 0) return ret; @@ -228,6 +227,7 @@ static int ad7476_probe(struct spi_device *spi) /* Establish that the iio_dev is a child of the spi device */ indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->name = spi_get_device_id(spi)->name; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = st->chip_info->channel; diff --git a/drivers/iio/adc/ad7791.c b/drivers/iio/adc/ad7791.c index cf172d58cd44..1817ebf5ad84 100644 --- a/drivers/iio/adc/ad7791.c +++ b/drivers/iio/adc/ad7791.c @@ -272,30 +272,22 @@ static ssize_t ad7791_write_frequency(struct device *dev, struct ad7791_state *st = iio_priv(indio_dev); int i, ret; - mutex_lock(&indio_dev->mlock); - if (iio_buffer_enabled(indio_dev)) { - mutex_unlock(&indio_dev->mlock); - return -EBUSY; - } - mutex_unlock(&indio_dev->mlock); - - ret = -EINVAL; - - for (i = 0; i < ARRAY_SIZE(ad7791_sample_freq_avail); i++) { - if (sysfs_streq(ad7791_sample_freq_avail[i], buf)) { - - mutex_lock(&indio_dev->mlock); - st->filter &= ~AD7791_FILTER_RATE_MASK; - st->filter |= i; - ad_sd_write_reg(&st->sd, AD7791_REG_FILTER, - sizeof(st->filter), st->filter); - mutex_unlock(&indio_dev->mlock); - ret = 0; + for (i = 0; i < ARRAY_SIZE(ad7791_sample_freq_avail); i++) + if (sysfs_streq(ad7791_sample_freq_avail[i], buf)) break; - } - } + if (i == ARRAY_SIZE(ad7791_sample_freq_avail)) + return -EINVAL; + + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + st->filter &= ~AD7791_FILTER_RATE_MASK; + st->filter |= i; + ad_sd_write_reg(&st->sd, AD7791_REG_FILTER, sizeof(st->filter), + st->filter); + iio_device_release_direct_mode(indio_dev); - return ret ? ret : len; + return len; } static IIO_DEV_ATTR_SAMP_FREQ(S_IWUSR | S_IRUGO, @@ -383,6 +375,7 @@ static int ad7791_probe(struct spi_device *spi) spi_set_drvdata(spi, indio_dev); indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->name = spi_get_device_id(spi)->name; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = st->info->channels; diff --git a/drivers/iio/adc/ad7793.c b/drivers/iio/adc/ad7793.c index 7b07bb651671..847789bae821 100644 --- a/drivers/iio/adc/ad7793.c +++ b/drivers/iio/adc/ad7793.c @@ -369,13 +369,6 @@ static ssize_t ad7793_write_frequency(struct device *dev, long lval; int i, ret; - mutex_lock(&indio_dev->mlock); - if (iio_buffer_enabled(indio_dev)) { - mutex_unlock(&indio_dev->mlock); - return -EBUSY; - } - mutex_unlock(&indio_dev->mlock); - ret = kstrtol(buf, 10, &lval); if (ret) return ret; @@ -383,20 +376,21 @@ static ssize_t ad7793_write_frequency(struct device *dev, if (lval == 0) return -EINVAL; - ret = -EINVAL; - for (i = 0; i < 16; i++) - if (lval == st->chip_info->sample_freq_avail[i]) { - mutex_lock(&indio_dev->mlock); - st->mode &= ~AD7793_MODE_RATE(-1); - st->mode |= AD7793_MODE_RATE(i); - ad_sd_write_reg(&st->sd, AD7793_REG_MODE, - sizeof(st->mode), st->mode); - mutex_unlock(&indio_dev->mlock); - ret = 0; - } + if (lval == st->chip_info->sample_freq_avail[i]) + break; + if (i == 16) + return -EINVAL; - return ret ? ret : len; + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + st->mode &= ~AD7793_MODE_RATE(-1); + st->mode |= AD7793_MODE_RATE(i); + ad_sd_write_reg(&st->sd, AD7793_REG_MODE, sizeof(st->mode), st->mode); + iio_device_release_direct_mode(indio_dev); + + return len; } static IIO_DEV_ATTR_SAMP_FREQ(S_IWUSR | S_IRUGO, @@ -790,6 +784,7 @@ static int ad7793_probe(struct spi_device *spi) spi_set_drvdata(spi, indio_dev); indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->name = spi_get_device_id(spi)->name; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = st->chip_info->channels; diff --git a/drivers/iio/adc/ad7887.c b/drivers/iio/adc/ad7887.c index 2d3c397e66ad..7a483bfbd70c 100644 --- a/drivers/iio/adc/ad7887.c +++ b/drivers/iio/adc/ad7887.c @@ -122,7 +122,7 @@ static irqreturn_t ad7887_trigger_handler(int irq, void *p) goto done; iio_push_to_buffers_with_timestamp(indio_dev, st->data, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -156,12 +156,11 @@ static int ad7887_read_raw(struct iio_dev *indio_dev, switch (m) { case IIO_CHAN_INFO_RAW: - mutex_lock(&indio_dev->mlock); - if (iio_buffer_enabled(indio_dev)) - ret = -EBUSY; - else - ret = ad7887_scan_direct(st, chan->address); - mutex_unlock(&indio_dev->mlock); + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + ret = ad7887_scan_direct(st, chan->address); + iio_device_release_direct_mode(indio_dev); if (ret < 0) return ret; @@ -265,6 +264,7 @@ static int ad7887_probe(struct spi_device *spi) /* Estabilish that the iio_dev is a child of the spi device */ indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->name = spi_get_device_id(spi)->name; indio_dev->info = &ad7887_info; indio_dev->modes = INDIO_DIRECT_MODE; diff --git a/drivers/iio/adc/ad7923.c b/drivers/iio/adc/ad7923.c index 45e29ccd824f..77a675e11ebb 100644 --- a/drivers/iio/adc/ad7923.c +++ b/drivers/iio/adc/ad7923.c @@ -181,7 +181,7 @@ static irqreturn_t ad7923_trigger_handler(int irq, void *p) goto done; iio_push_to_buffers_with_timestamp(indio_dev, st->rx_buf, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -233,12 +233,11 @@ static int ad7923_read_raw(struct iio_dev *indio_dev, switch (m) { case IIO_CHAN_INFO_RAW: - mutex_lock(&indio_dev->mlock); - if (iio_buffer_enabled(indio_dev)) - ret = -EBUSY; - else - ret = ad7923_scan_direct(st, chan->address); - mutex_unlock(&indio_dev->mlock); + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + ret = ad7923_scan_direct(st, chan->address); + iio_device_release_direct_mode(indio_dev); if (ret < 0) return ret; @@ -289,6 +288,7 @@ static int ad7923_probe(struct spi_device *spi) indio_dev->name = spi_get_device_id(spi)->name; indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = info->channels; indio_dev->num_channels = info->num_channels; diff --git a/drivers/iio/adc/ad799x.c b/drivers/iio/adc/ad799x.c index a3f5254f4e51..b6163764489c 100644 --- a/drivers/iio/adc/ad799x.c +++ b/drivers/iio/adc/ad799x.c @@ -212,7 +212,7 @@ static irqreturn_t ad799x_trigger_handler(int irq, void *p) goto out; iio_push_to_buffers_with_timestamp(indio_dev, st->rx_buf, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); out: iio_trigger_notify_done(indio_dev->trig); @@ -282,12 +282,11 @@ static int ad799x_read_raw(struct iio_dev *indio_dev, switch (m) { case IIO_CHAN_INFO_RAW: - mutex_lock(&indio_dev->mlock); - if (iio_buffer_enabled(indio_dev)) - ret = -EBUSY; - else - ret = ad799x_scan_direct(st, chan->scan_index); - mutex_unlock(&indio_dev->mlock); + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + ret = ad799x_scan_direct(st, chan->scan_index); + iio_device_release_direct_mode(indio_dev); if (ret < 0) return ret; @@ -395,11 +394,9 @@ static int ad799x_write_event_config(struct iio_dev *indio_dev, struct ad799x_state *st = iio_priv(indio_dev); int ret; - mutex_lock(&indio_dev->mlock); - if (iio_buffer_enabled(indio_dev)) { - ret = -EBUSY; - goto done; - } + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; if (state) st->config |= BIT(chan->scan_index) << AD799X_CHANNEL_SHIFT; @@ -412,10 +409,7 @@ static int ad799x_write_event_config(struct iio_dev *indio_dev, st->config &= ~AD7998_ALERT_EN; ret = ad799x_write_config(st, st->config); - -done: - mutex_unlock(&indio_dev->mlock); - + iio_device_release_direct_mode(indio_dev); return ret; } @@ -508,7 +502,7 @@ static irqreturn_t ad799x_event_handler(int irq, void *private) (i >> 1), IIO_EV_TYPE_THRESH, IIO_EV_DIR_FALLING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } done: @@ -812,6 +806,7 @@ static int ad799x_probe(struct i2c_client *client, st->client = client; indio_dev->dev.parent = &client->dev; + indio_dev->dev.of_node = client->dev.of_node; indio_dev->name = id->name; indio_dev->info = st->chip_config->info; diff --git a/drivers/iio/adc/bcm_iproc_adc.c b/drivers/iio/adc/bcm_iproc_adc.c new file mode 100644 index 000000000000..21d38c8af21e --- /dev/null +++ b/drivers/iio/adc/bcm_iproc_adc.c @@ -0,0 +1,644 @@ +/* + * Copyright 2016 Broadcom + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation (the "GPL"). + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 (GPLv2) for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 (GPLv2) along with this source code. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* Below Register's are common to IPROC ADC and Touchscreen IP */ +#define IPROC_REGCTL1 0x00 +#define IPROC_REGCTL2 0x04 +#define IPROC_INTERRUPT_THRES 0x08 +#define IPROC_INTERRUPT_MASK 0x0c +#define IPROC_INTERRUPT_STATUS 0x10 +#define IPROC_ANALOG_CONTROL 0x1c +#define IPROC_CONTROLLER_STATUS 0x14 +#define IPROC_AUX_DATA 0x20 +#define IPROC_SOFT_BYPASS_CONTROL 0x38 +#define IPROC_SOFT_BYPASS_DATA 0x3C + +/* IPROC ADC Channel register offsets */ +#define IPROC_ADC_CHANNEL_REGCTL1 0x800 +#define IPROC_ADC_CHANNEL_REGCTL2 0x804 +#define IPROC_ADC_CHANNEL_STATUS 0x808 +#define IPROC_ADC_CHANNEL_INTERRUPT_STATUS 0x80c +#define IPROC_ADC_CHANNEL_INTERRUPT_MASK 0x810 +#define IPROC_ADC_CHANNEL_DATA 0x814 +#define IPROC_ADC_CHANNEL_OFFSET 0x20 + +/* Bit definitions for IPROC_REGCTL2 */ +#define IPROC_ADC_AUXIN_SCAN_ENA BIT(0) +#define IPROC_ADC_PWR_LDO BIT(5) +#define IPROC_ADC_PWR_ADC BIT(4) +#define IPROC_ADC_PWR_BG BIT(3) +#define IPROC_ADC_CONTROLLER_EN BIT(17) + +/* Bit definitions for IPROC_INTERRUPT_MASK and IPROC_INTERRUPT_STATUS */ +#define IPROC_ADC_AUXDATA_RDY_INTR BIT(3) +#define IPROC_ADC_INTR 9 +#define IPROC_ADC_INTR_MASK (0xFF << IPROC_ADC_INTR) + +/* Bit definitions for IPROC_ANALOG_CONTROL */ +#define IPROC_ADC_CHANNEL_SEL 11 +#define IPROC_ADC_CHANNEL_SEL_MASK (0x7 << IPROC_ADC_CHANNEL_SEL) + +/* Bit definitions for IPROC_ADC_CHANNEL_REGCTL1 */ +#define IPROC_ADC_CHANNEL_ROUNDS 0x2 +#define IPROC_ADC_CHANNEL_ROUNDS_MASK (0x3F << IPROC_ADC_CHANNEL_ROUNDS) +#define IPROC_ADC_CHANNEL_MODE 0x1 +#define IPROC_ADC_CHANNEL_MODE_MASK (0x1 << IPROC_ADC_CHANNEL_MODE) +#define IPROC_ADC_CHANNEL_MODE_TDM 0x1 +#define IPROC_ADC_CHANNEL_MODE_SNAPSHOT 0x0 +#define IPROC_ADC_CHANNEL_ENABLE 0x0 +#define IPROC_ADC_CHANNEL_ENABLE_MASK 0x1 + +/* Bit definitions for IPROC_ADC_CHANNEL_REGCTL2 */ +#define IPROC_ADC_CHANNEL_WATERMARK 0x0 +#define IPROC_ADC_CHANNEL_WATERMARK_MASK \ + (0x3F << IPROC_ADC_CHANNEL_WATERMARK) + +#define IPROC_ADC_WATER_MARK_LEVEL 0x1 + +/* Bit definitions for IPROC_ADC_CHANNEL_STATUS */ +#define IPROC_ADC_CHANNEL_DATA_LOST 0x0 +#define IPROC_ADC_CHANNEL_DATA_LOST_MASK \ + (0x0 << IPROC_ADC_CHANNEL_DATA_LOST) +#define IPROC_ADC_CHANNEL_VALID_ENTERIES 0x1 +#define IPROC_ADC_CHANNEL_VALID_ENTERIES_MASK \ + (0xFF << IPROC_ADC_CHANNEL_VALID_ENTERIES) +#define IPROC_ADC_CHANNEL_TOTAL_ENTERIES 0x9 +#define IPROC_ADC_CHANNEL_TOTAL_ENTERIES_MASK \ + (0xFF << IPROC_ADC_CHANNEL_TOTAL_ENTERIES) + +/* Bit definitions for IPROC_ADC_CHANNEL_INTERRUPT_MASK */ +#define IPROC_ADC_CHANNEL_WTRMRK_INTR 0x0 +#define IPROC_ADC_CHANNEL_WTRMRK_INTR_MASK \ + (0x1 << IPROC_ADC_CHANNEL_WTRMRK_INTR) +#define IPROC_ADC_CHANNEL_FULL_INTR 0x1 +#define IPROC_ADC_CHANNEL_FULL_INTR_MASK \ + (0x1 << IPROC_ADC_IPROC_ADC_CHANNEL_FULL_INTR) +#define IPROC_ADC_CHANNEL_EMPTY_INTR 0x2 +#define IPROC_ADC_CHANNEL_EMPTY_INTR_MASK \ + (0x1 << IPROC_ADC_CHANNEL_EMPTY_INTR) + +#define IPROC_ADC_WATER_MARK_INTR_ENABLE 0x1 + +/* Number of time to retry a set of the interrupt mask reg */ +#define IPROC_ADC_INTMASK_RETRY_ATTEMPTS 10 + +#define IPROC_ADC_READ_TIMEOUT (HZ*2) + +#define iproc_adc_dbg_reg(dev, priv, reg) \ +do { \ + u32 val; \ + regmap_read(priv->regmap, reg, &val); \ + dev_dbg(dev, "%20s= 0x%08x\n", #reg, val); \ +} while (0) + +struct iproc_adc_priv { + struct regmap *regmap; + struct clk *adc_clk; + struct mutex mutex; + int irqno; + int chan_val; + int chan_id; + struct completion completion; +}; + +static void iproc_adc_reg_dump(struct iio_dev *indio_dev) +{ + struct device *dev = &indio_dev->dev; + struct iproc_adc_priv *adc_priv = iio_priv(indio_dev); + + iproc_adc_dbg_reg(dev, adc_priv, IPROC_REGCTL1); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_REGCTL2); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_INTERRUPT_THRES); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_INTERRUPT_MASK); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_INTERRUPT_STATUS); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_CONTROLLER_STATUS); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_ANALOG_CONTROL); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_AUX_DATA); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_SOFT_BYPASS_CONTROL); + iproc_adc_dbg_reg(dev, adc_priv, IPROC_SOFT_BYPASS_DATA); +} + +static irqreturn_t iproc_adc_interrupt_handler(int irq, void *data) +{ + u32 channel_intr_status; + u32 intr_status; + u32 intr_mask; + struct iio_dev *indio_dev = data; + struct iproc_adc_priv *adc_priv = iio_priv(indio_dev); + + /* + * This interrupt is shared with the touchscreen driver. + * Make sure this interrupt is intended for us. + * Handle only ADC channel specific interrupts. + */ + regmap_read(adc_priv->regmap, IPROC_INTERRUPT_STATUS, &intr_status); + regmap_read(adc_priv->regmap, IPROC_INTERRUPT_MASK, &intr_mask); + intr_status = intr_status & intr_mask; + channel_intr_status = (intr_status & IPROC_ADC_INTR_MASK) >> + IPROC_ADC_INTR; + if (channel_intr_status) + return IRQ_WAKE_THREAD; + + return IRQ_NONE; +} + +static irqreturn_t iproc_adc_interrupt_thread(int irq, void *data) +{ + irqreturn_t retval = IRQ_NONE; + struct iproc_adc_priv *adc_priv; + struct iio_dev *indio_dev = data; + unsigned int valid_entries; + u32 intr_status; + u32 intr_channels; + u32 channel_status; + u32 ch_intr_status; + + adc_priv = iio_priv(indio_dev); + + regmap_read(adc_priv->regmap, IPROC_INTERRUPT_STATUS, &intr_status); + dev_dbg(&indio_dev->dev, "iproc_adc_interrupt_thread(),INTRPT_STS:%x\n", + intr_status); + + intr_channels = (intr_status & IPROC_ADC_INTR_MASK) >> IPROC_ADC_INTR; + if (intr_channels) { + regmap_read(adc_priv->regmap, + IPROC_ADC_CHANNEL_INTERRUPT_STATUS + + IPROC_ADC_CHANNEL_OFFSET * adc_priv->chan_id, + &ch_intr_status); + + if (ch_intr_status & IPROC_ADC_CHANNEL_WTRMRK_INTR_MASK) { + regmap_read(adc_priv->regmap, + IPROC_ADC_CHANNEL_STATUS + + IPROC_ADC_CHANNEL_OFFSET * + adc_priv->chan_id, + &channel_status); + + valid_entries = ((channel_status & + IPROC_ADC_CHANNEL_VALID_ENTERIES_MASK) >> + IPROC_ADC_CHANNEL_VALID_ENTERIES); + if (valid_entries >= 1) { + regmap_read(adc_priv->regmap, + IPROC_ADC_CHANNEL_DATA + + IPROC_ADC_CHANNEL_OFFSET * + adc_priv->chan_id, + &adc_priv->chan_val); + complete(&adc_priv->completion); + } else { + dev_err(&indio_dev->dev, + "No data rcvd on channel %d\n", + adc_priv->chan_id); + } + regmap_write(adc_priv->regmap, + IPROC_ADC_CHANNEL_INTERRUPT_MASK + + IPROC_ADC_CHANNEL_OFFSET * + adc_priv->chan_id, + (ch_intr_status & + ~(IPROC_ADC_CHANNEL_WTRMRK_INTR_MASK))); + } + regmap_write(adc_priv->regmap, + IPROC_ADC_CHANNEL_INTERRUPT_STATUS + + IPROC_ADC_CHANNEL_OFFSET * adc_priv->chan_id, + ch_intr_status); + regmap_write(adc_priv->regmap, IPROC_INTERRUPT_STATUS, + intr_channels); + retval = IRQ_HANDLED; + } + + return retval; +} + +static int iproc_adc_do_read(struct iio_dev *indio_dev, + int channel, + u16 *p_adc_data) +{ + int read_len = 0; + u32 val; + u32 mask; + u32 val_check; + int failed_cnt = 0; + struct iproc_adc_priv *adc_priv = iio_priv(indio_dev); + + mutex_lock(&adc_priv->mutex); + + /* + * After a read is complete the ADC interrupts will be disabled so + * we can assume this section of code is safe from interrupts. + */ + adc_priv->chan_val = -1; + adc_priv->chan_id = channel; + + reinit_completion(&adc_priv->completion); + /* Clear any pending interrupt */ + regmap_update_bits(adc_priv->regmap, IPROC_INTERRUPT_STATUS, + IPROC_ADC_INTR_MASK | IPROC_ADC_AUXDATA_RDY_INTR, + ((0x0 << channel) << IPROC_ADC_INTR) | + IPROC_ADC_AUXDATA_RDY_INTR); + + /* Configure channel for snapshot mode and enable */ + val = (BIT(IPROC_ADC_CHANNEL_ROUNDS) | + (IPROC_ADC_CHANNEL_MODE_SNAPSHOT << IPROC_ADC_CHANNEL_MODE) | + (0x1 << IPROC_ADC_CHANNEL_ENABLE)); + + mask = IPROC_ADC_CHANNEL_ROUNDS_MASK | IPROC_ADC_CHANNEL_MODE_MASK | + IPROC_ADC_CHANNEL_ENABLE_MASK; + regmap_update_bits(adc_priv->regmap, (IPROC_ADC_CHANNEL_REGCTL1 + + IPROC_ADC_CHANNEL_OFFSET * channel), + mask, val); + + /* Set the Watermark for a channel */ + regmap_update_bits(adc_priv->regmap, (IPROC_ADC_CHANNEL_REGCTL2 + + IPROC_ADC_CHANNEL_OFFSET * channel), + IPROC_ADC_CHANNEL_WATERMARK_MASK, + 0x1); + + /* Enable water mark interrupt */ + regmap_update_bits(adc_priv->regmap, (IPROC_ADC_CHANNEL_INTERRUPT_MASK + + IPROC_ADC_CHANNEL_OFFSET * + channel), + IPROC_ADC_CHANNEL_WTRMRK_INTR_MASK, + IPROC_ADC_WATER_MARK_INTR_ENABLE); + regmap_read(adc_priv->regmap, IPROC_INTERRUPT_MASK, &val); + + /* Enable ADC interrupt for a channel */ + val |= (BIT(channel) << IPROC_ADC_INTR); + regmap_write(adc_priv->regmap, IPROC_INTERRUPT_MASK, val); + + /* + * There seems to be a very rare issue where writing to this register + * does not take effect. To work around the issue we will try multiple + * writes. In total we will spend about 10*10 = 100 us attempting this. + * Testing has shown that this may loop a few time, but we have never + * hit the full count. + */ + regmap_read(adc_priv->regmap, IPROC_INTERRUPT_MASK, &val_check); + while (val_check != val) { + failed_cnt++; + + if (failed_cnt > IPROC_ADC_INTMASK_RETRY_ATTEMPTS) + break; + + udelay(10); + regmap_update_bits(adc_priv->regmap, IPROC_INTERRUPT_MASK, + IPROC_ADC_INTR_MASK, + ((0x1 << channel) << + IPROC_ADC_INTR)); + + regmap_read(adc_priv->regmap, IPROC_INTERRUPT_MASK, &val_check); + } + + if (failed_cnt) { + dev_dbg(&indio_dev->dev, + "IntMask failed (%d times)", failed_cnt); + if (failed_cnt > IPROC_ADC_INTMASK_RETRY_ATTEMPTS) { + dev_err(&indio_dev->dev, + "IntMask set failed. Read will likely fail."); + read_len = -EIO; + goto adc_err; + }; + } + regmap_read(adc_priv->regmap, IPROC_INTERRUPT_MASK, &val_check); + + if (wait_for_completion_timeout(&adc_priv->completion, + IPROC_ADC_READ_TIMEOUT) > 0) { + + /* Only the lower 16 bits are relevant */ + *p_adc_data = adc_priv->chan_val & 0xFFFF; + read_len = sizeof(*p_adc_data); + + } else { + /* + * We never got the interrupt, something went wrong. + * Perhaps the interrupt may still be coming, we do not want + * that now. Lets disable the ADC interrupt, and clear the + * status to put it back in to normal state. + */ + read_len = -ETIMEDOUT; + goto adc_err; + } + mutex_unlock(&adc_priv->mutex); + + return read_len; + +adc_err: + regmap_update_bits(adc_priv->regmap, IPROC_INTERRUPT_MASK, + IPROC_ADC_INTR_MASK, + ((0x0 << channel) << IPROC_ADC_INTR)); + + regmap_update_bits(adc_priv->regmap, IPROC_INTERRUPT_STATUS, + IPROC_ADC_INTR_MASK, + ((0x0 << channel) << IPROC_ADC_INTR)); + + dev_err(&indio_dev->dev, "Timed out waiting for ADC data!\n"); + iproc_adc_reg_dump(indio_dev); + mutex_unlock(&adc_priv->mutex); + + return read_len; +} + +static int iproc_adc_enable(struct iio_dev *indio_dev) +{ + u32 val; + u32 channel_id; + struct iproc_adc_priv *adc_priv = iio_priv(indio_dev); + int ret; + + /* Set i_amux = 3b'000, select channel 0 */ + ret = regmap_update_bits(adc_priv->regmap, IPROC_ANALOG_CONTROL, + IPROC_ADC_CHANNEL_SEL_MASK, 0); + if (ret) { + dev_err(&indio_dev->dev, + "failed to write IPROC_ANALOG_CONTROL %d\n", ret); + return ret; + } + adc_priv->chan_val = -1; + + /* + * PWR up LDO, ADC, and Band Gap (0 to enable) + * Also enable ADC controller (set high) + */ + ret = regmap_read(adc_priv->regmap, IPROC_REGCTL2, &val); + if (ret) { + dev_err(&indio_dev->dev, + "failed to read IPROC_REGCTL2 %d\n", ret); + return ret; + } + + val &= ~(IPROC_ADC_PWR_LDO | IPROC_ADC_PWR_ADC | IPROC_ADC_PWR_BG); + + ret = regmap_write(adc_priv->regmap, IPROC_REGCTL2, val); + if (ret) { + dev_err(&indio_dev->dev, + "failed to write IPROC_REGCTL2 %d\n", ret); + return ret; + } + + ret = regmap_read(adc_priv->regmap, IPROC_REGCTL2, &val); + if (ret) { + dev_err(&indio_dev->dev, + "failed to read IPROC_REGCTL2 %d\n", ret); + return ret; + } + + val |= IPROC_ADC_CONTROLLER_EN; + ret = regmap_write(adc_priv->regmap, IPROC_REGCTL2, val); + if (ret) { + dev_err(&indio_dev->dev, + "failed to write IPROC_REGCTL2 %d\n", ret); + return ret; + } + + for (channel_id = 0; channel_id < indio_dev->num_channels; + channel_id++) { + ret = regmap_write(adc_priv->regmap, + IPROC_ADC_CHANNEL_INTERRUPT_MASK + + IPROC_ADC_CHANNEL_OFFSET * channel_id, 0); + if (ret) { + dev_err(&indio_dev->dev, + "failed to write ADC_CHANNEL_INTERRUPT_MASK %d\n", + ret); + return ret; + } + + ret = regmap_write(adc_priv->regmap, + IPROC_ADC_CHANNEL_INTERRUPT_STATUS + + IPROC_ADC_CHANNEL_OFFSET * channel_id, 0); + if (ret) { + dev_err(&indio_dev->dev, + "failed to write ADC_CHANNEL_INTERRUPT_STATUS %d\n", + ret); + return ret; + } + } + + return 0; +} + +static void iproc_adc_disable(struct iio_dev *indio_dev) +{ + u32 val; + int ret; + struct iproc_adc_priv *adc_priv = iio_priv(indio_dev); + + ret = regmap_read(adc_priv->regmap, IPROC_REGCTL2, &val); + if (ret) { + dev_err(&indio_dev->dev, + "failed to read IPROC_REGCTL2 %d\n", ret); + return; + } + + val &= ~IPROC_ADC_CONTROLLER_EN; + ret = regmap_write(adc_priv->regmap, IPROC_REGCTL2, val); + if (ret) { + dev_err(&indio_dev->dev, + "failed to write IPROC_REGCTL2 %d\n", ret); + return; + } +} + +static int iproc_adc_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, + int *val2, + long mask) +{ + u16 adc_data; + int err; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + err = iproc_adc_do_read(indio_dev, chan->channel, &adc_data); + if (err < 0) + return err; + *val = adc_data; + return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + switch (chan->type) { + case IIO_VOLTAGE: + *val = 1800; + *val2 = 10; + return IIO_VAL_FRACTIONAL_LOG2; + default: + return -EINVAL; + } + default: + return -EINVAL; + } +} + +static const struct iio_info iproc_adc_iio_info = { + .read_raw = &iproc_adc_read_raw, + .driver_module = THIS_MODULE, +}; + +#define IPROC_ADC_CHANNEL(_index, _id) { \ + .type = IIO_VOLTAGE, \ + .indexed = 1, \ + .channel = _index, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ + .datasheet_name = _id, \ +} + +static const struct iio_chan_spec iproc_adc_iio_channels[] = { + IPROC_ADC_CHANNEL(0, "adc0"), + IPROC_ADC_CHANNEL(1, "adc1"), + IPROC_ADC_CHANNEL(2, "adc2"), + IPROC_ADC_CHANNEL(3, "adc3"), + IPROC_ADC_CHANNEL(4, "adc4"), + IPROC_ADC_CHANNEL(5, "adc5"), + IPROC_ADC_CHANNEL(6, "adc6"), + IPROC_ADC_CHANNEL(7, "adc7"), +}; + +static int iproc_adc_probe(struct platform_device *pdev) +{ + struct iproc_adc_priv *adc_priv; + struct iio_dev *indio_dev = NULL; + int ret; + + indio_dev = devm_iio_device_alloc(&pdev->dev, + sizeof(*adc_priv)); + if (!indio_dev) { + dev_err(&pdev->dev, "failed to allocate iio device\n"); + return -ENOMEM; + } + + adc_priv = iio_priv(indio_dev); + platform_set_drvdata(pdev, indio_dev); + + mutex_init(&adc_priv->mutex); + + init_completion(&adc_priv->completion); + + adc_priv->regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, + "adc-syscon"); + if (IS_ERR(adc_priv->regmap)) { + dev_err(&pdev->dev, "failed to get handle for tsc syscon\n"); + ret = PTR_ERR(adc_priv->regmap); + return ret; + } + + adc_priv->adc_clk = devm_clk_get(&pdev->dev, "tsc_clk"); + if (IS_ERR(adc_priv->adc_clk)) { + dev_err(&pdev->dev, + "failed getting clock tsc_clk\n"); + ret = PTR_ERR(adc_priv->adc_clk); + return ret; + } + + adc_priv->irqno = platform_get_irq(pdev, 0); + if (adc_priv->irqno <= 0) { + dev_err(&pdev->dev, "platform_get_irq failed\n"); + ret = -ENODEV; + return ret; + } + + ret = regmap_update_bits(adc_priv->regmap, IPROC_REGCTL2, + IPROC_ADC_AUXIN_SCAN_ENA, 0); + if (ret) { + dev_err(&pdev->dev, "failed to write IPROC_REGCTL2 %d\n", ret); + return ret; + } + + ret = devm_request_threaded_irq(&pdev->dev, adc_priv->irqno, + iproc_adc_interrupt_thread, + iproc_adc_interrupt_handler, + IRQF_SHARED, "iproc-adc", indio_dev); + if (ret) { + dev_err(&pdev->dev, "request_irq error %d\n", ret); + return ret; + } + + ret = clk_prepare_enable(adc_priv->adc_clk); + if (ret) { + dev_err(&pdev->dev, + "clk_prepare_enable failed %d\n", ret); + return ret; + } + + ret = iproc_adc_enable(indio_dev); + if (ret) { + dev_err(&pdev->dev, "failed to enable adc %d\n", ret); + goto err_adc_enable; + } + + indio_dev->name = "iproc-static-adc"; + indio_dev->dev.parent = &pdev->dev; + indio_dev->dev.of_node = pdev->dev.of_node; + indio_dev->info = &iproc_adc_iio_info; + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->channels = iproc_adc_iio_channels; + indio_dev->num_channels = ARRAY_SIZE(iproc_adc_iio_channels); + + ret = iio_device_register(indio_dev); + if (ret) { + dev_err(&pdev->dev, "iio_device_register failed:err %d\n", ret); + goto err_clk; + } + + return 0; + +err_clk: + iproc_adc_disable(indio_dev); +err_adc_enable: + clk_disable_unprepare(adc_priv->adc_clk); + + return ret; +} + +static int iproc_adc_remove(struct platform_device *pdev) +{ + struct iio_dev *indio_dev = platform_get_drvdata(pdev); + struct iproc_adc_priv *adc_priv = iio_priv(indio_dev); + + iio_device_unregister(indio_dev); + iproc_adc_disable(indio_dev); + clk_disable_unprepare(adc_priv->adc_clk); + + return 0; +} + +static const struct of_device_id iproc_adc_of_match[] = { + {.compatible = "brcm,iproc-static-adc", }, + { }, +}; +MODULE_DEVICE_TABLE(of, iproc_adc_of_match); + +static struct platform_driver iproc_adc_driver = { + .probe = iproc_adc_probe, + .remove = iproc_adc_remove, + .driver = { + .name = "iproc-static-adc", + .of_match_table = of_match_ptr(iproc_adc_of_match), + }, +}; +module_platform_driver(iproc_adc_driver); + +MODULE_DESCRIPTION("Broadcom iProc ADC controller driver"); +MODULE_AUTHOR("Raveendra Padasalagi "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/adc/cc10001_adc.c b/drivers/iio/adc/cc10001_adc.c index 8254f529b2a9..91636c0ba5b5 100644 --- a/drivers/iio/adc/cc10001_adc.c +++ b/drivers/iio/adc/cc10001_adc.c @@ -186,7 +186,7 @@ done: if (!sample_invalid) iio_push_to_buffers_with_timestamp(indio_dev, data, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); iio_trigger_notify_done(indio_dev->trig); return IRQ_HANDLED; diff --git a/drivers/iio/adc/hi8435.c b/drivers/iio/adc/hi8435.c index c73c6c62a6ac..678e8c7ea763 100644 --- a/drivers/iio/adc/hi8435.c +++ b/drivers/iio/adc/hi8435.c @@ -400,7 +400,7 @@ static void hi8435_iio_push_event(struct iio_dev *idev, unsigned int val) iio_push_event(idev, IIO_UNMOD_EVENT_CODE(IIO_VOLTAGE, i, IIO_EV_TYPE_THRESH, dir), - iio_get_time_ns()); + iio_get_time_ns(idev)); } } @@ -455,6 +455,7 @@ static int hi8435_probe(struct spi_device *spi) mutex_init(&priv->lock); idev->dev.parent = &spi->dev; + idev->dev.of_node = spi->dev.of_node; idev->name = spi_get_device_id(spi)->name; idev->modes = INDIO_DIRECT_MODE; idev->info = &hi8435_info; diff --git a/drivers/iio/adc/ina2xx-adc.c b/drivers/iio/adc/ina2xx-adc.c index 502f2fbe8aef..955f3fdaf519 100644 --- a/drivers/iio/adc/ina2xx-adc.c +++ b/drivers/iio/adc/ina2xx-adc.c @@ -465,7 +465,7 @@ static int ina2xx_work_buffer(struct iio_dev *indio_dev) s64 time_a, time_b; unsigned int alert; - time_a = iio_get_time_ns(); + time_a = iio_get_time_ns(indio_dev); /* * Because the timer thread and the chip conversion clock @@ -504,7 +504,7 @@ static int ina2xx_work_buffer(struct iio_dev *indio_dev) data[i++] = val; } - time_b = iio_get_time_ns(); + time_b = iio_get_time_ns(indio_dev); iio_push_to_buffers_with_timestamp(indio_dev, (unsigned int *)data, time_a); @@ -554,7 +554,7 @@ static int ina2xx_buffer_enable(struct iio_dev *indio_dev) dev_dbg(&indio_dev->dev, "Async readout mode: %d\n", chip->allow_async_readout); - chip->prev_ns = iio_get_time_ns(); + chip->prev_ns = iio_get_time_ns(indio_dev); chip->task = kthread_run(ina2xx_capture_thread, (void *)indio_dev, "%s:%d-%uus", indio_dev->name, indio_dev->id, @@ -691,6 +691,7 @@ static int ina2xx_probe(struct i2c_client *client, indio_dev->modes = INDIO_DIRECT_MODE | INDIO_BUFFER_SOFTWARE; indio_dev->dev.parent = &client->dev; + indio_dev->dev.of_node = client->dev.of_node; indio_dev->channels = ina2xx_channels; indio_dev->num_channels = ARRAY_SIZE(ina2xx_channels); indio_dev->name = id->name; diff --git a/drivers/iio/adc/max1027.c b/drivers/iio/adc/max1027.c index 41d495c6035e..712fbd2b1f16 100644 --- a/drivers/iio/adc/max1027.c +++ b/drivers/iio/adc/max1027.c @@ -426,6 +426,7 @@ static int max1027_probe(struct spi_device *spi) indio_dev->name = spi_get_device_id(spi)->name; indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->info = &max1027_info; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = st->info->channels; diff --git a/drivers/iio/adc/max1363.c b/drivers/iio/adc/max1363.c index 998dc3caad4c..841a13c9b6ea 100644 --- a/drivers/iio/adc/max1363.c +++ b/drivers/iio/adc/max1363.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include @@ -788,7 +790,7 @@ static irqreturn_t max1363_event_handler(int irq, void *private) { struct iio_dev *indio_dev = private; struct max1363_state *st = iio_priv(indio_dev); - s64 timestamp = iio_get_time_ns(); + s64 timestamp = iio_get_time_ns(indio_dev); unsigned long mask, loc; u8 rx; u8 tx[2] = { st->setupbyte, @@ -1506,7 +1508,8 @@ static irqreturn_t max1363_trigger_handler(int irq, void *p) if (b_sent < 0) goto done_free; - iio_push_to_buffers_with_timestamp(indio_dev, rxbuf, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, rxbuf, + iio_get_time_ns(indio_dev)); done_free: kfree(rxbuf); @@ -1516,6 +1519,56 @@ done: return IRQ_HANDLED; } +#ifdef CONFIG_OF + +#define MAX1363_COMPATIBLE(of_compatible, cfg) { \ + .compatible = of_compatible, \ + .data = &max1363_chip_info_tbl[cfg], \ +} + +static const struct of_device_id max1363_of_match[] = { + MAX1363_COMPATIBLE("maxim,max1361", max1361), + MAX1363_COMPATIBLE("maxim,max1362", max1362), + MAX1363_COMPATIBLE("maxim,max1363", max1363), + MAX1363_COMPATIBLE("maxim,max1364", max1364), + MAX1363_COMPATIBLE("maxim,max1036", max1036), + MAX1363_COMPATIBLE("maxim,max1037", max1037), + MAX1363_COMPATIBLE("maxim,max1038", max1038), + MAX1363_COMPATIBLE("maxim,max1039", max1039), + MAX1363_COMPATIBLE("maxim,max1136", max1136), + MAX1363_COMPATIBLE("maxim,max1137", max1137), + MAX1363_COMPATIBLE("maxim,max1138", max1138), + MAX1363_COMPATIBLE("maxim,max1139", max1139), + MAX1363_COMPATIBLE("maxim,max1236", max1236), + MAX1363_COMPATIBLE("maxim,max1237", max1237), + MAX1363_COMPATIBLE("maxim,max1238", max1238), + MAX1363_COMPATIBLE("maxim,max1239", max1239), + MAX1363_COMPATIBLE("maxim,max11600", max11600), + MAX1363_COMPATIBLE("maxim,max11601", max11601), + MAX1363_COMPATIBLE("maxim,max11602", max11602), + MAX1363_COMPATIBLE("maxim,max11603", max11603), + MAX1363_COMPATIBLE("maxim,max11604", max11604), + MAX1363_COMPATIBLE("maxim,max11605", max11605), + MAX1363_COMPATIBLE("maxim,max11606", max11606), + MAX1363_COMPATIBLE("maxim,max11607", max11607), + MAX1363_COMPATIBLE("maxim,max11608", max11608), + MAX1363_COMPATIBLE("maxim,max11609", max11609), + MAX1363_COMPATIBLE("maxim,max11610", max11610), + MAX1363_COMPATIBLE("maxim,max11611", max11611), + MAX1363_COMPATIBLE("maxim,max11612", max11612), + MAX1363_COMPATIBLE("maxim,max11613", max11613), + MAX1363_COMPATIBLE("maxim,max11614", max11614), + MAX1363_COMPATIBLE("maxim,max11615", max11615), + MAX1363_COMPATIBLE("maxim,max11616", max11616), + MAX1363_COMPATIBLE("maxim,max11617", max11617), + MAX1363_COMPATIBLE("maxim,max11644", max11644), + MAX1363_COMPATIBLE("maxim,max11645", max11645), + MAX1363_COMPATIBLE("maxim,max11646", max11646), + MAX1363_COMPATIBLE("maxim,max11647", max11647), + { /* sentinel */ } +}; +#endif + static int max1363_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -1523,6 +1576,7 @@ static int max1363_probe(struct i2c_client *client, struct max1363_state *st; struct iio_dev *indio_dev; struct regulator *vref; + const struct of_device_id *match; indio_dev = devm_iio_device_alloc(&client->dev, sizeof(struct max1363_state)); @@ -1549,7 +1603,12 @@ static int max1363_probe(struct i2c_client *client, /* this is only used for device removal purposes */ i2c_set_clientdata(client, indio_dev); - st->chip_info = &max1363_chip_info_tbl[id->driver_data]; + match = of_match_device(of_match_ptr(max1363_of_match), + &client->dev); + if (match) + st->chip_info = of_device_get_match_data(&client->dev); + else + st->chip_info = &max1363_chip_info_tbl[id->driver_data]; st->client = client; st->vref_uv = st->chip_info->int_vref_mv * 1000; @@ -1587,6 +1646,7 @@ static int max1363_probe(struct i2c_client *client, /* Establish that the iio_dev is a child of the i2c device */ indio_dev->dev.parent = &client->dev; + indio_dev->dev.of_node = client->dev.of_node; indio_dev->name = id->name; indio_dev->channels = st->chip_info->channels; indio_dev->num_channels = st->chip_info->num_channels; @@ -1692,6 +1752,7 @@ MODULE_DEVICE_TABLE(i2c, max1363_id); static struct i2c_driver max1363_driver = { .driver = { .name = "max1363", + .of_match_table = of_match_ptr(max1363_of_match), }, .probe = max1363_probe, .remove = max1363_remove, diff --git a/drivers/iio/adc/mcp320x.c b/drivers/iio/adc/mcp320x.c index a850ca7d1eda..634717ae12f3 100644 --- a/drivers/iio/adc/mcp320x.c +++ b/drivers/iio/adc/mcp320x.c @@ -308,6 +308,7 @@ static int mcp320x_probe(struct spi_device *spi) adc->spi = spi; indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->name = spi_get_device_id(spi)->name; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->info = &mcp320x_info; diff --git a/drivers/iio/adc/mcp3422.c b/drivers/iio/adc/mcp3422.c index d1172dc1e8e2..254135e07792 100644 --- a/drivers/iio/adc/mcp3422.c +++ b/drivers/iio/adc/mcp3422.c @@ -352,6 +352,7 @@ static int mcp3422_probe(struct i2c_client *client, mutex_init(&adc->lock); indio_dev->dev.parent = &client->dev; + indio_dev->dev.of_node = client->dev.of_node; indio_dev->name = dev_name(&client->dev); indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->info = &mcp3422_info; diff --git a/drivers/iio/adc/mxs-lradc.c b/drivers/iio/adc/mxs-lradc.c index ad26da1edbee..b84d37c80a94 100644 --- a/drivers/iio/adc/mxs-lradc.c +++ b/drivers/iio/adc/mxs-lradc.c @@ -373,13 +373,6 @@ static u32 mxs_lradc_plate_mask(struct mxs_lradc *lradc) return LRADC_CTRL0_MX28_PLATE_MASK; } -static u32 mxs_lradc_irq_en_mask(struct mxs_lradc *lradc) -{ - if (lradc->soc == IMX23_LRADC) - return LRADC_CTRL1_MX23_LRADC_IRQ_EN_MASK; - return LRADC_CTRL1_MX28_LRADC_IRQ_EN_MASK; -} - static u32 mxs_lradc_irq_mask(struct mxs_lradc *lradc) { if (lradc->soc == IMX23_LRADC) @@ -1120,18 +1113,16 @@ static int mxs_lradc_ts_register(struct mxs_lradc *lradc) { struct input_dev *input; struct device *dev = lradc->dev; - int ret; if (!lradc->use_touchscreen) return 0; - input = input_allocate_device(); + input = devm_input_allocate_device(dev); if (!input) return -ENOMEM; input->name = DRIVER_NAME; input->id.bustype = BUS_HOST; - input->dev.parent = dev; input->open = mxs_lradc_ts_open; input->close = mxs_lradc_ts_close; @@ -1146,20 +1137,8 @@ static int mxs_lradc_ts_register(struct mxs_lradc *lradc) lradc->ts_input = input; input_set_drvdata(input, lradc); - ret = input_register_device(input); - if (ret) - input_free_device(lradc->ts_input); - - return ret; -} - -static void mxs_lradc_ts_unregister(struct mxs_lradc *lradc) -{ - if (!lradc->use_touchscreen) - return; - mxs_lradc_disable_ts(lradc); - input_unregister_device(lradc->ts_input); + return input_register_device(input); } /* @@ -1510,7 +1489,9 @@ static void mxs_lradc_hw_stop(struct mxs_lradc *lradc) { int i; - mxs_lradc_reg_clear(lradc, mxs_lradc_irq_en_mask(lradc), LRADC_CTRL1); + mxs_lradc_reg_clear(lradc, + lradc->buffer_vchans << LRADC_CTRL1_LRADC_IRQ_EN_OFFSET, + LRADC_CTRL1); for (i = 0; i < LRADC_MAX_DELAY_CHANS; i++) mxs_lradc_reg_wrt(lradc, 0, LRADC_DELAY(i)); @@ -1721,13 +1702,11 @@ static int mxs_lradc_probe(struct platform_device *pdev) ret = iio_device_register(iio); if (ret) { dev_err(dev, "Failed to register IIO device\n"); - goto err_ts; + return ret; } return 0; -err_ts: - mxs_lradc_ts_unregister(lradc); err_ts_register: mxs_lradc_hw_stop(lradc); err_dev: @@ -1745,7 +1724,6 @@ static int mxs_lradc_remove(struct platform_device *pdev) struct mxs_lradc *lradc = iio_priv(iio); iio_device_unregister(iio); - mxs_lradc_ts_unregister(lradc); mxs_lradc_hw_stop(lradc); mxs_lradc_trigger_remove(iio); iio_triggered_buffer_cleanup(iio); diff --git a/drivers/iio/adc/nau7802.c b/drivers/iio/adc/nau7802.c index e525aa6475c4..db9b829ccf0d 100644 --- a/drivers/iio/adc/nau7802.c +++ b/drivers/iio/adc/nau7802.c @@ -79,10 +79,29 @@ static const struct iio_chan_spec nau7802_chan_array[] = { static const u16 nau7802_sample_freq_avail[] = {10, 20, 40, 80, 10, 10, 10, 320}; +static ssize_t nau7802_show_scales(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nau7802_state *st = iio_priv(dev_to_iio_dev(dev)); + int i, len = 0; + + for (i = 0; i < ARRAY_SIZE(st->scale_avail); i++) + len += scnprintf(buf + len, PAGE_SIZE - len, "0.%09d ", + st->scale_avail[i]); + + buf[len-1] = '\n'; + + return len; +} + static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("10 40 80 320"); +static IIO_DEVICE_ATTR(in_voltage_scale_available, S_IRUGO, nau7802_show_scales, + NULL, 0); + static struct attribute *nau7802_attributes[] = { &iio_const_attr_sampling_frequency_available.dev_attr.attr, + &iio_dev_attr_in_voltage_scale_available.dev_attr.attr, NULL }; @@ -414,6 +433,7 @@ static int nau7802_probe(struct i2c_client *client, i2c_set_clientdata(client, indio_dev); indio_dev->dev.parent = &client->dev; + indio_dev->dev.of_node = client->dev.of_node; indio_dev->name = dev_name(&client->dev); indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->info = &nau7802_info; diff --git a/drivers/iio/adc/ti-adc081c.c b/drivers/iio/adc/ti-adc081c.c index 9fd032d9f402..319172cf7da8 100644 --- a/drivers/iio/adc/ti-adc081c.c +++ b/drivers/iio/adc/ti-adc081c.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -138,7 +139,8 @@ static irqreturn_t adc081c_trigger_handler(int irq, void *p) if (ret < 0) goto out; buf[0] = ret; - iio_push_to_buffers_with_timestamp(indio_dev, buf, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, buf, + iio_get_time_ns(indio_dev)); out: iio_trigger_notify_done(indio_dev->trig); return IRQ_HANDLED; @@ -149,12 +151,24 @@ static int adc081c_probe(struct i2c_client *client, { struct iio_dev *iio; struct adc081c *adc; - struct adcxx1c_model *model = &adcxx1c_models[id->driver_data]; + struct adcxx1c_model *model; int err; if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WORD_DATA)) return -EOPNOTSUPP; + if (ACPI_COMPANION(&client->dev)) { + const struct acpi_device_id *ad_id; + + ad_id = acpi_match_device(client->dev.driver->acpi_match_table, + &client->dev); + if (!ad_id) + return -ENODEV; + model = &adcxx1c_models[ad_id->driver_data]; + } else { + model = &adcxx1c_models[id->driver_data]; + } + iio = devm_iio_device_alloc(&client->dev, sizeof(*adc)); if (!iio) return -ENOMEM; @@ -172,6 +186,7 @@ static int adc081c_probe(struct i2c_client *client, return err; iio->dev.parent = &client->dev; + iio->dev.of_node = client->dev.of_node; iio->name = dev_name(&client->dev); iio->modes = INDIO_DIRECT_MODE; iio->info = &adc081c_info; @@ -231,10 +246,21 @@ static const struct of_device_id adc081c_of_match[] = { MODULE_DEVICE_TABLE(of, adc081c_of_match); #endif +#ifdef CONFIG_ACPI +static const struct acpi_device_id adc081c_acpi_match[] = { + { "ADC081C", ADC081C }, + { "ADC101C", ADC101C }, + { "ADC121C", ADC121C }, + { } +}; +MODULE_DEVICE_TABLE(acpi, adc081c_acpi_match); +#endif + static struct i2c_driver adc081c_driver = { .driver = { .name = "adc081c", .of_match_table = of_match_ptr(adc081c_of_match), + .acpi_match_table = ACPI_PTR(adc081c_acpi_match), }, .probe = adc081c_probe, .remove = adc081c_remove, diff --git a/drivers/iio/adc/ti-adc0832.c b/drivers/iio/adc/ti-adc0832.c index 0afeac0c9bad..f4ba23effe9a 100644 --- a/drivers/iio/adc/ti-adc0832.c +++ b/drivers/iio/adc/ti-adc0832.c @@ -194,6 +194,7 @@ static int adc0832_probe(struct spi_device *spi) indio_dev->name = spi_get_device_id(spi)->name; indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->info = &adc0832_info; indio_dev->modes = INDIO_DIRECT_MODE; diff --git a/drivers/iio/adc/ti-adc128s052.c b/drivers/iio/adc/ti-adc128s052.c index bc58867d6e8d..89dfbd31be5c 100644 --- a/drivers/iio/adc/ti-adc128s052.c +++ b/drivers/iio/adc/ti-adc128s052.c @@ -150,6 +150,7 @@ static int adc128_probe(struct spi_device *spi) spi_set_drvdata(spi, indio_dev); indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->name = spi_get_device_id(spi)->name; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->info = &adc128_info; diff --git a/drivers/iio/adc/ti-ads1015.c b/drivers/iio/adc/ti-ads1015.c index 73cbf0b54e54..1ef398770a1f 100644 --- a/drivers/iio/adc/ti-ads1015.c +++ b/drivers/iio/adc/ti-ads1015.c @@ -55,6 +55,11 @@ #define ADS1015_DEFAULT_DATA_RATE 4 #define ADS1015_DEFAULT_CHAN 0 +enum { + ADS1015, + ADS1115, +}; + enum ads1015_channels { ADS1015_AIN0_AIN1 = 0, ADS1015_AIN0_AIN3, @@ -71,6 +76,10 @@ static const unsigned int ads1015_data_rate[] = { 128, 250, 490, 920, 1600, 2400, 3300, 3300 }; +static const unsigned int ads1115_data_rate[] = { + 8, 16, 32, 64, 128, 250, 475, 860 +}; + static const struct { int scale; int uscale; @@ -101,6 +110,7 @@ static const struct { .shift = 4, \ .endianness = IIO_CPU, \ }, \ + .datasheet_name = "AIN"#_chan, \ } #define ADS1015_V_DIFF_CHAN(_chan, _chan2, _addr) { \ @@ -121,6 +131,45 @@ static const struct { .shift = 4, \ .endianness = IIO_CPU, \ }, \ + .datasheet_name = "AIN"#_chan"-AIN"#_chan2, \ +} + +#define ADS1115_V_CHAN(_chan, _addr) { \ + .type = IIO_VOLTAGE, \ + .indexed = 1, \ + .address = _addr, \ + .channel = _chan, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \ + BIT(IIO_CHAN_INFO_SCALE) | \ + BIT(IIO_CHAN_INFO_SAMP_FREQ), \ + .scan_index = _addr, \ + .scan_type = { \ + .sign = 's', \ + .realbits = 16, \ + .storagebits = 16, \ + .endianness = IIO_CPU, \ + }, \ + .datasheet_name = "AIN"#_chan, \ +} + +#define ADS1115_V_DIFF_CHAN(_chan, _chan2, _addr) { \ + .type = IIO_VOLTAGE, \ + .differential = 1, \ + .indexed = 1, \ + .address = _addr, \ + .channel = _chan, \ + .channel2 = _chan2, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \ + BIT(IIO_CHAN_INFO_SCALE) | \ + BIT(IIO_CHAN_INFO_SAMP_FREQ), \ + .scan_index = _addr, \ + .scan_type = { \ + .sign = 's', \ + .realbits = 16, \ + .storagebits = 16, \ + .endianness = IIO_CPU, \ + }, \ + .datasheet_name = "AIN"#_chan"-AIN"#_chan2, \ } struct ads1015_data { @@ -131,6 +180,8 @@ struct ads1015_data { */ struct mutex lock; struct ads1015_channel_data channel_data[ADS1015_CHANNELS]; + + unsigned int *data_rate; }; static bool ads1015_is_writeable_reg(struct device *dev, unsigned int reg) @@ -157,6 +208,18 @@ static const struct iio_chan_spec ads1015_channels[] = { IIO_CHAN_SOFT_TIMESTAMP(ADS1015_TIMESTAMP), }; +static const struct iio_chan_spec ads1115_channels[] = { + ADS1115_V_DIFF_CHAN(0, 1, ADS1015_AIN0_AIN1), + ADS1115_V_DIFF_CHAN(0, 3, ADS1015_AIN0_AIN3), + ADS1115_V_DIFF_CHAN(1, 3, ADS1015_AIN1_AIN3), + ADS1115_V_DIFF_CHAN(2, 3, ADS1015_AIN2_AIN3), + ADS1115_V_CHAN(0, ADS1015_AIN0), + ADS1115_V_CHAN(1, ADS1015_AIN1), + ADS1115_V_CHAN(2, ADS1015_AIN2), + ADS1115_V_CHAN(3, ADS1015_AIN3), + IIO_CHAN_SOFT_TIMESTAMP(ADS1015_TIMESTAMP), +}; + static int ads1015_set_power_state(struct ads1015_data *data, bool on) { int ret; @@ -196,7 +259,7 @@ int ads1015_get_adc_result(struct ads1015_data *data, int chan, int *val) return ret; if (change) { - conv_time = DIV_ROUND_UP(USEC_PER_SEC, ads1015_data_rate[dr]); + conv_time = DIV_ROUND_UP(USEC_PER_SEC, data->data_rate[dr]); usleep_range(conv_time, conv_time + 1); } @@ -225,7 +288,8 @@ static irqreturn_t ads1015_trigger_handler(int irq, void *p) buf[0] = res; mutex_unlock(&data->lock); - iio_push_to_buffers_with_timestamp(indio_dev, buf, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, buf, + iio_get_time_ns(indio_dev)); err: iio_trigger_notify_done(indio_dev->trig); @@ -263,7 +327,7 @@ static int ads1015_set_data_rate(struct ads1015_data *data, int chan, int rate) int i, ret, rindex = -1; for (i = 0; i < ARRAY_SIZE(ads1015_data_rate); i++) - if (ads1015_data_rate[i] == rate) { + if (data->data_rate[i] == rate) { rindex = i; break; } @@ -291,7 +355,9 @@ static int ads1015_read_raw(struct iio_dev *indio_dev, mutex_lock(&indio_dev->mlock); mutex_lock(&data->lock); switch (mask) { - case IIO_CHAN_INFO_RAW: + case IIO_CHAN_INFO_RAW: { + int shift = chan->scan_type.shift; + if (iio_buffer_enabled(indio_dev)) { ret = -EBUSY; break; @@ -307,8 +373,7 @@ static int ads1015_read_raw(struct iio_dev *indio_dev, break; } - /* 12 bit res, D0 is bit 4 in conversion register */ - *val = sign_extend32(*val >> 4, 11); + *val = sign_extend32(*val >> shift, 15 - shift); ret = ads1015_set_power_state(data, false); if (ret < 0) @@ -316,6 +381,7 @@ static int ads1015_read_raw(struct iio_dev *indio_dev, ret = IIO_VAL_INT; break; + } case IIO_CHAN_INFO_SCALE: idx = data->channel_data[chan->address].pga; *val = ads1015_scale[idx].scale; @@ -324,7 +390,7 @@ static int ads1015_read_raw(struct iio_dev *indio_dev, break; case IIO_CHAN_INFO_SAMP_FREQ: idx = data->channel_data[chan->address].data_rate; - *val = ads1015_data_rate[idx]; + *val = data->data_rate[idx]; ret = IIO_VAL_INT; break; default: @@ -380,12 +446,15 @@ static const struct iio_buffer_setup_ops ads1015_buffer_setup_ops = { }; static IIO_CONST_ATTR(scale_available, "3 2 1 0.5 0.25 0.125"); -static IIO_CONST_ATTR(sampling_frequency_available, - "128 250 490 920 1600 2400 3300"); + +static IIO_CONST_ATTR_NAMED(ads1015_sampling_frequency_available, + sampling_frequency_available, "128 250 490 920 1600 2400 3300"); +static IIO_CONST_ATTR_NAMED(ads1115_sampling_frequency_available, + sampling_frequency_available, "8 16 32 64 128 250 475 860"); static struct attribute *ads1015_attributes[] = { &iio_const_attr_scale_available.dev_attr.attr, - &iio_const_attr_sampling_frequency_available.dev_attr.attr, + &iio_const_attr_ads1015_sampling_frequency_available.dev_attr.attr, NULL, }; @@ -393,11 +462,28 @@ static const struct attribute_group ads1015_attribute_group = { .attrs = ads1015_attributes, }; -static const struct iio_info ads1015_info = { +static struct attribute *ads1115_attributes[] = { + &iio_const_attr_scale_available.dev_attr.attr, + &iio_const_attr_ads1115_sampling_frequency_available.dev_attr.attr, + NULL, +}; + +static const struct attribute_group ads1115_attribute_group = { + .attrs = ads1115_attributes, +}; + +static struct iio_info ads1015_info = { + .driver_module = THIS_MODULE, + .read_raw = ads1015_read_raw, + .write_raw = ads1015_write_raw, + .attrs = &ads1015_attribute_group, +}; + +static struct iio_info ads1115_info = { .driver_module = THIS_MODULE, .read_raw = ads1015_read_raw, .write_raw = ads1015_write_raw, - .attrs = &ads1015_attribute_group, + .attrs = &ads1115_attribute_group, }; #ifdef CONFIG_OF @@ -500,12 +586,25 @@ static int ads1015_probe(struct i2c_client *client, mutex_init(&data->lock); indio_dev->dev.parent = &client->dev; - indio_dev->info = &ads1015_info; + indio_dev->dev.of_node = client->dev.of_node; indio_dev->name = ADS1015_DRV_NAME; - indio_dev->channels = ads1015_channels; - indio_dev->num_channels = ARRAY_SIZE(ads1015_channels); indio_dev->modes = INDIO_DIRECT_MODE; + switch (id->driver_data) { + case ADS1015: + indio_dev->channels = ads1015_channels; + indio_dev->num_channels = ARRAY_SIZE(ads1015_channels); + indio_dev->info = &ads1015_info; + data->data_rate = (unsigned int *) &ads1015_data_rate; + break; + case ADS1115: + indio_dev->channels = ads1115_channels; + indio_dev->num_channels = ARRAY_SIZE(ads1115_channels); + indio_dev->info = &ads1115_info; + data->data_rate = (unsigned int *) &ads1115_data_rate; + break; + } + /* we need to keep this ABI the same as used by hwmon ADS1015 driver */ ads1015_get_channels_config(client); @@ -590,7 +689,8 @@ static const struct dev_pm_ops ads1015_pm_ops = { }; static const struct i2c_device_id ads1015_id[] = { - {"ads1015", 0}, + {"ads1015", ADS1015}, + {"ads1115", ADS1115}, {} }; MODULE_DEVICE_TABLE(i2c, ads1015_id); diff --git a/drivers/iio/adc/ti-ads8688.c b/drivers/iio/adc/ti-ads8688.c index 03e907028cb6..c400439900af 100644 --- a/drivers/iio/adc/ti-ads8688.c +++ b/drivers/iio/adc/ti-ads8688.c @@ -421,6 +421,7 @@ static int ads8688_probe(struct spi_device *spi) indio_dev->name = spi_get_device_id(spi)->name; indio_dev->dev.parent = &spi->dev; + indio_dev->dev.of_node = spi->dev.of_node; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = st->chip_info->channels; indio_dev->num_channels = st->chip_info->num_channels; diff --git a/drivers/iio/adc/ti_am335x_adc.c b/drivers/iio/adc/ti_am335x_adc.c index c1e05532d437..8a368756881b 100644 --- a/drivers/iio/adc/ti_am335x_adc.c +++ b/drivers/iio/adc/ti_am335x_adc.c @@ -326,8 +326,7 @@ static int tiadc_channel_init(struct iio_dev *indio_dev, int channels) int i; indio_dev->num_channels = channels; - chan_array = kcalloc(channels, - sizeof(struct iio_chan_spec), GFP_KERNEL); + chan_array = kcalloc(channels, sizeof(*chan_array), GFP_KERNEL); if (chan_array == NULL) return -ENOMEM; @@ -467,8 +466,7 @@ static int tiadc_probe(struct platform_device *pdev) return -EINVAL; } - indio_dev = devm_iio_device_alloc(&pdev->dev, - sizeof(struct tiadc_device)); + indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*indio_dev)); if (indio_dev == NULL) { dev_err(&pdev->dev, "failed to allocate iio device\n"); return -ENOMEM; @@ -531,8 +529,7 @@ static int tiadc_remove(struct platform_device *pdev) return 0; } -#ifdef CONFIG_PM -static int tiadc_suspend(struct device *dev) +static int __maybe_unused tiadc_suspend(struct device *dev) { struct iio_dev *indio_dev = dev_get_drvdata(dev); struct tiadc_device *adc_dev = iio_priv(indio_dev); @@ -550,7 +547,7 @@ static int tiadc_suspend(struct device *dev) return 0; } -static int tiadc_resume(struct device *dev) +static int __maybe_unused tiadc_resume(struct device *dev) { struct iio_dev *indio_dev = dev_get_drvdata(dev); struct tiadc_device *adc_dev = iio_priv(indio_dev); @@ -567,14 +564,7 @@ static int tiadc_resume(struct device *dev) return 0; } -static const struct dev_pm_ops tiadc_pm_ops = { - .suspend = tiadc_suspend, - .resume = tiadc_resume, -}; -#define TIADC_PM_OPS (&tiadc_pm_ops) -#else -#define TIADC_PM_OPS NULL -#endif +static SIMPLE_DEV_PM_OPS(tiadc_pm_ops, tiadc_suspend, tiadc_resume); static const struct of_device_id ti_adc_dt_ids[] = { { .compatible = "ti,am3359-adc", }, @@ -585,7 +575,7 @@ MODULE_DEVICE_TABLE(of, ti_adc_dt_ids); static struct platform_driver tiadc_driver = { .driver = { .name = "TI-am335x-adc", - .pm = TIADC_PM_OPS, + .pm = &tiadc_pm_ops, .of_match_table = ti_adc_dt_ids, }, .probe = tiadc_probe, diff --git a/drivers/iio/adc/vf610_adc.c b/drivers/iio/adc/vf610_adc.c index 653bf1379d2e..228a003adeed 100644 --- a/drivers/iio/adc/vf610_adc.c +++ b/drivers/iio/adc/vf610_adc.c @@ -594,7 +594,8 @@ static irqreturn_t vf610_adc_isr(int irq, void *dev_id) if (iio_buffer_enabled(indio_dev)) { info->buffer[0] = info->value; iio_push_to_buffers_with_timestamp(indio_dev, - info->buffer, iio_get_time_ns()); + info->buffer, + iio_get_time_ns(indio_dev)); iio_trigger_notify_done(indio_dev->trig); } else complete(&info->completion); diff --git a/drivers/iio/adc/xilinx-xadc-events.c b/drivers/iio/adc/xilinx-xadc-events.c index edcf3aabd70d..6d5c2a6f4e6e 100644 --- a/drivers/iio/adc/xilinx-xadc-events.c +++ b/drivers/iio/adc/xilinx-xadc-events.c @@ -46,7 +46,7 @@ static void xadc_handle_event(struct iio_dev *indio_dev, unsigned int event) iio_push_event(indio_dev, IIO_UNMOD_EVENT_CODE(chan->type, chan->channel, IIO_EV_TYPE_THRESH, IIO_EV_DIR_RISING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } else { /* * For other channels we don't know whether it is a upper or @@ -56,7 +56,7 @@ static void xadc_handle_event(struct iio_dev *indio_dev, unsigned int event) iio_push_event(indio_dev, IIO_UNMOD_EVENT_CODE(chan->type, chan->channel, IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } } diff --git a/drivers/iio/buffer/industrialio-buffer-dma.c b/drivers/iio/buffer/industrialio-buffer-dma.c index 212cbedc7abb..dd99d273bae9 100644 --- a/drivers/iio/buffer/industrialio-buffer-dma.c +++ b/drivers/iio/buffer/industrialio-buffer-dma.c @@ -305,7 +305,7 @@ int iio_dma_buffer_request_update(struct iio_buffer *buffer) queue->fileio.active_block = NULL; spin_lock_irq(&queue->list_lock); - for (i = 0; i < 2; i++) { + for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) { block = queue->fileio.blocks[i]; /* If we can't re-use it free it */ @@ -323,7 +323,7 @@ int iio_dma_buffer_request_update(struct iio_buffer *buffer) INIT_LIST_HEAD(&queue->incoming); - for (i = 0; i < 2; i++) { + for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) { if (queue->fileio.blocks[i]) { block = queue->fileio.blocks[i]; if (block->state == IIO_BLOCK_STATE_DEAD) { diff --git a/drivers/iio/chemical/Kconfig b/drivers/iio/chemical/Kconfig index f73290f84c90..4bcc025e8c8a 100644 --- a/drivers/iio/chemical/Kconfig +++ b/drivers/iio/chemical/Kconfig @@ -5,15 +5,17 @@ menu "Chemical Sensors" config ATLAS_PH_SENSOR - tristate "Atlas Scientific OEM pH-SM sensor" + tristate "Atlas Scientific OEM SM sensors" depends on I2C select REGMAP_I2C select IIO_BUFFER select IIO_TRIGGERED_BUFFER select IRQ_WORK help - Say Y here to build I2C interface support for the Atlas - Scientific OEM pH-SM sensor. + Say Y here to build I2C interface support for the following + Atlas Scientific OEM SM sensors: + * pH SM sensor + * EC SM sensor To compile this driver as module, choose M here: the module will be called atlas-ph-sensor. diff --git a/drivers/iio/chemical/atlas-ph-sensor.c b/drivers/iio/chemical/atlas-ph-sensor.c index 62b37cd8fb56..ae038a59d256 100644 --- a/drivers/iio/chemical/atlas-ph-sensor.c +++ b/drivers/iio/chemical/atlas-ph-sensor.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -43,29 +44,50 @@ #define ATLAS_REG_PWR_CONTROL 0x06 -#define ATLAS_REG_CALIB_STATUS 0x0d -#define ATLAS_REG_CALIB_STATUS_MASK 0x07 -#define ATLAS_REG_CALIB_STATUS_LOW BIT(0) -#define ATLAS_REG_CALIB_STATUS_MID BIT(1) -#define ATLAS_REG_CALIB_STATUS_HIGH BIT(2) +#define ATLAS_REG_PH_CALIB_STATUS 0x0d +#define ATLAS_REG_PH_CALIB_STATUS_MASK 0x07 +#define ATLAS_REG_PH_CALIB_STATUS_LOW BIT(0) +#define ATLAS_REG_PH_CALIB_STATUS_MID BIT(1) +#define ATLAS_REG_PH_CALIB_STATUS_HIGH BIT(2) -#define ATLAS_REG_TEMP_DATA 0x0e +#define ATLAS_REG_EC_CALIB_STATUS 0x0f +#define ATLAS_REG_EC_CALIB_STATUS_MASK 0x0f +#define ATLAS_REG_EC_CALIB_STATUS_DRY BIT(0) +#define ATLAS_REG_EC_CALIB_STATUS_SINGLE BIT(1) +#define ATLAS_REG_EC_CALIB_STATUS_LOW BIT(2) +#define ATLAS_REG_EC_CALIB_STATUS_HIGH BIT(3) + +#define ATLAS_REG_PH_TEMP_DATA 0x0e #define ATLAS_REG_PH_DATA 0x16 +#define ATLAS_REG_EC_PROBE 0x08 +#define ATLAS_REG_EC_TEMP_DATA 0x10 +#define ATLAS_REG_EC_DATA 0x18 +#define ATLAS_REG_TDS_DATA 0x1c +#define ATLAS_REG_PSS_DATA 0x20 + #define ATLAS_PH_INT_TIME_IN_US 450000 +#define ATLAS_EC_INT_TIME_IN_US 650000 + +enum { + ATLAS_PH_SM, + ATLAS_EC_SM, +}; struct atlas_data { struct i2c_client *client; struct iio_trigger *trig; + struct atlas_device *chip; struct regmap *regmap; struct irq_work work; - __be32 buffer[4]; /* 32-bit pH data + 32-bit pad + 64-bit timestamp */ + __be32 buffer[6]; /* 96-bit data + 32-bit pad + 64-bit timestamp */ }; static const struct regmap_range atlas_volatile_ranges[] = { regmap_reg_range(ATLAS_REG_INT_CONTROL, ATLAS_REG_INT_CONTROL), regmap_reg_range(ATLAS_REG_PH_DATA, ATLAS_REG_PH_DATA + 4), + regmap_reg_range(ATLAS_REG_EC_DATA, ATLAS_REG_PSS_DATA + 4), }; static const struct regmap_access_table atlas_volatile_table = { @@ -80,13 +102,14 @@ static const struct regmap_config atlas_regmap_config = { .val_bits = 8, .volatile_table = &atlas_volatile_table, - .max_register = ATLAS_REG_PH_DATA + 4, + .max_register = ATLAS_REG_PSS_DATA + 4, .cache_type = REGCACHE_RBTREE, }; -static const struct iio_chan_spec atlas_channels[] = { +static const struct iio_chan_spec atlas_ph_channels[] = { { .type = IIO_PH, + .address = ATLAS_REG_PH_DATA, .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), .scan_index = 0, @@ -100,7 +123,7 @@ static const struct iio_chan_spec atlas_channels[] = { IIO_CHAN_SOFT_TIMESTAMP(1), { .type = IIO_TEMP, - .address = ATLAS_REG_TEMP_DATA, + .address = ATLAS_REG_PH_TEMP_DATA, .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), .output = 1, @@ -108,6 +131,142 @@ static const struct iio_chan_spec atlas_channels[] = { }, }; +#define ATLAS_EC_CHANNEL(_idx, _addr) \ + {\ + .type = IIO_CONCENTRATION, \ + .indexed = 1, \ + .channel = _idx, \ + .address = _addr, \ + .info_mask_separate = \ + BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), \ + .scan_index = _idx + 1, \ + .scan_type = { \ + .sign = 'u', \ + .realbits = 32, \ + .storagebits = 32, \ + .endianness = IIO_BE, \ + }, \ + } + +static const struct iio_chan_spec atlas_ec_channels[] = { + { + .type = IIO_ELECTRICALCONDUCTIVITY, + .address = ATLAS_REG_EC_DATA, + .info_mask_separate = + BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), + .scan_index = 0, + .scan_type = { + .sign = 'u', + .realbits = 32, + .storagebits = 32, + .endianness = IIO_BE, + }, + }, + ATLAS_EC_CHANNEL(0, ATLAS_REG_TDS_DATA), + ATLAS_EC_CHANNEL(1, ATLAS_REG_PSS_DATA), + IIO_CHAN_SOFT_TIMESTAMP(3), + { + .type = IIO_TEMP, + .address = ATLAS_REG_EC_TEMP_DATA, + .info_mask_separate = + BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), + .output = 1, + .scan_index = -1 + }, +}; + +static int atlas_check_ph_calibration(struct atlas_data *data) +{ + struct device *dev = &data->client->dev; + int ret; + unsigned int val; + + ret = regmap_read(data->regmap, ATLAS_REG_PH_CALIB_STATUS, &val); + if (ret) + return ret; + + if (!(val & ATLAS_REG_PH_CALIB_STATUS_MASK)) { + dev_warn(dev, "device has not been calibrated\n"); + return 0; + } + + if (!(val & ATLAS_REG_PH_CALIB_STATUS_LOW)) + dev_warn(dev, "device missing low point calibration\n"); + + if (!(val & ATLAS_REG_PH_CALIB_STATUS_MID)) + dev_warn(dev, "device missing mid point calibration\n"); + + if (!(val & ATLAS_REG_PH_CALIB_STATUS_HIGH)) + dev_warn(dev, "device missing high point calibration\n"); + + return 0; +} + +static int atlas_check_ec_calibration(struct atlas_data *data) +{ + struct device *dev = &data->client->dev; + int ret; + unsigned int val; + + ret = regmap_bulk_read(data->regmap, ATLAS_REG_EC_PROBE, &val, 2); + if (ret) + return ret; + + dev_info(dev, "probe set to K = %d.%.2d", be16_to_cpu(val) / 100, + be16_to_cpu(val) % 100); + + ret = regmap_read(data->regmap, ATLAS_REG_EC_CALIB_STATUS, &val); + if (ret) + return ret; + + if (!(val & ATLAS_REG_EC_CALIB_STATUS_MASK)) { + dev_warn(dev, "device has not been calibrated\n"); + return 0; + } + + if (!(val & ATLAS_REG_EC_CALIB_STATUS_DRY)) + dev_warn(dev, "device missing dry point calibration\n"); + + if (val & ATLAS_REG_EC_CALIB_STATUS_SINGLE) { + dev_warn(dev, "device using single point calibration\n"); + } else { + if (!(val & ATLAS_REG_EC_CALIB_STATUS_LOW)) + dev_warn(dev, "device missing low point calibration\n"); + + if (!(val & ATLAS_REG_EC_CALIB_STATUS_HIGH)) + dev_warn(dev, "device missing high point calibration\n"); + } + + return 0; +} + +struct atlas_device { + const struct iio_chan_spec *channels; + int num_channels; + int data_reg; + + int (*calibration)(struct atlas_data *data); + int delay; +}; + +static struct atlas_device atlas_devices[] = { + [ATLAS_PH_SM] = { + .channels = atlas_ph_channels, + .num_channels = 3, + .data_reg = ATLAS_REG_PH_DATA, + .calibration = &atlas_check_ph_calibration, + .delay = ATLAS_PH_INT_TIME_IN_US, + }, + [ATLAS_EC_SM] = { + .channels = atlas_ec_channels, + .num_channels = 5, + .data_reg = ATLAS_REG_EC_DATA, + .calibration = &atlas_check_ec_calibration, + .delay = ATLAS_EC_INT_TIME_IN_US, + }, + +}; + static int atlas_set_powermode(struct atlas_data *data, int on) { return regmap_write(data->regmap, ATLAS_REG_PWR_CONTROL, on); @@ -178,12 +337,13 @@ static irqreturn_t atlas_trigger_handler(int irq, void *private) struct atlas_data *data = iio_priv(indio_dev); int ret; - ret = regmap_bulk_read(data->regmap, ATLAS_REG_PH_DATA, - (u8 *) &data->buffer, sizeof(data->buffer[0])); + ret = regmap_bulk_read(data->regmap, data->chip->data_reg, + (u8 *) &data->buffer, + sizeof(__be32) * (data->chip->num_channels - 2)); if (!ret) iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); iio_trigger_notify_done(indio_dev->trig); @@ -200,7 +360,7 @@ static irqreturn_t atlas_interrupt_handler(int irq, void *private) return IRQ_HANDLED; } -static int atlas_read_ph_measurement(struct atlas_data *data, __be32 *val) +static int atlas_read_measurement(struct atlas_data *data, int reg, __be32 *val) { struct device *dev = &data->client->dev; int suspended = pm_runtime_suspended(dev); @@ -213,11 +373,9 @@ static int atlas_read_ph_measurement(struct atlas_data *data, __be32 *val) } if (suspended) - usleep_range(ATLAS_PH_INT_TIME_IN_US, - ATLAS_PH_INT_TIME_IN_US + 100000); + usleep_range(data->chip->delay, data->chip->delay + 100000); - ret = regmap_bulk_read(data->regmap, ATLAS_REG_PH_DATA, - (u8 *) val, sizeof(*val)); + ret = regmap_bulk_read(data->regmap, reg, (u8 *) val, sizeof(*val)); pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); @@ -242,12 +400,15 @@ static int atlas_read_raw(struct iio_dev *indio_dev, (u8 *) ®, sizeof(reg)); break; case IIO_PH: + case IIO_CONCENTRATION: + case IIO_ELECTRICALCONDUCTIVITY: mutex_lock(&indio_dev->mlock); if (iio_buffer_enabled(indio_dev)) ret = -EBUSY; else - ret = atlas_read_ph_measurement(data, ®); + ret = atlas_read_measurement(data, + chan->address, ®); mutex_unlock(&indio_dev->mlock); break; @@ -271,6 +432,14 @@ static int atlas_read_raw(struct iio_dev *indio_dev, *val = 1; /* 0.001 */ *val2 = 1000; break; + case IIO_ELECTRICALCONDUCTIVITY: + *val = 1; /* 0.00001 */ + *val = 100000; + break; + case IIO_CONCENTRATION: + *val = 0; /* 0.000000001 */ + *val2 = 1000; + return IIO_VAL_INT_PLUS_NANO; default: return -EINVAL; } @@ -303,37 +472,26 @@ static const struct iio_info atlas_info = { .write_raw = atlas_write_raw, }; -static int atlas_check_calibration(struct atlas_data *data) -{ - struct device *dev = &data->client->dev; - int ret; - unsigned int val; - - ret = regmap_read(data->regmap, ATLAS_REG_CALIB_STATUS, &val); - if (ret) - return ret; - - if (!(val & ATLAS_REG_CALIB_STATUS_MASK)) { - dev_warn(dev, "device has not been calibrated\n"); - return 0; - } - - if (!(val & ATLAS_REG_CALIB_STATUS_LOW)) - dev_warn(dev, "device missing low point calibration\n"); - - if (!(val & ATLAS_REG_CALIB_STATUS_MID)) - dev_warn(dev, "device missing mid point calibration\n"); - - if (!(val & ATLAS_REG_CALIB_STATUS_HIGH)) - dev_warn(dev, "device missing high point calibration\n"); +static const struct i2c_device_id atlas_id[] = { + { "atlas-ph-sm", ATLAS_PH_SM}, + { "atlas-ec-sm", ATLAS_EC_SM}, + {} +}; +MODULE_DEVICE_TABLE(i2c, atlas_id); - return 0; +static const struct of_device_id atlas_dt_ids[] = { + { .compatible = "atlas,ph-sm", .data = (void *)ATLAS_PH_SM, }, + { .compatible = "atlas,ec-sm", .data = (void *)ATLAS_EC_SM, }, + { } }; +MODULE_DEVICE_TABLE(of, atlas_dt_ids); static int atlas_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct atlas_data *data; + struct atlas_device *chip; + const struct of_device_id *of_id; struct iio_trigger *trig; struct iio_dev *indio_dev; int ret; @@ -342,10 +500,16 @@ static int atlas_probe(struct i2c_client *client, if (!indio_dev) return -ENOMEM; + of_id = of_match_device(atlas_dt_ids, &client->dev); + if (!of_id) + chip = &atlas_devices[id->driver_data]; + else + chip = &atlas_devices[(unsigned long)of_id->data]; + indio_dev->info = &atlas_info; indio_dev->name = ATLAS_DRV_NAME; - indio_dev->channels = atlas_channels; - indio_dev->num_channels = ARRAY_SIZE(atlas_channels); + indio_dev->channels = chip->channels; + indio_dev->num_channels = chip->num_channels; indio_dev->modes = INDIO_BUFFER_SOFTWARE | INDIO_DIRECT_MODE; indio_dev->dev.parent = &client->dev; @@ -358,6 +522,7 @@ static int atlas_probe(struct i2c_client *client, data = iio_priv(indio_dev); data->client = client; data->trig = trig; + data->chip = chip; trig->dev.parent = indio_dev->dev.parent; trig->ops = &atlas_interrupt_trigger_ops; iio_trigger_set_drvdata(trig, indio_dev); @@ -379,7 +544,7 @@ static int atlas_probe(struct i2c_client *client, return -EINVAL; } - ret = atlas_check_calibration(data); + ret = chip->calibration(data); if (ret) return ret; @@ -480,18 +645,6 @@ static const struct dev_pm_ops atlas_pm_ops = { atlas_runtime_resume, NULL) }; -static const struct i2c_device_id atlas_id[] = { - { "atlas-ph-sm", 0 }, - {} -}; -MODULE_DEVICE_TABLE(i2c, atlas_id); - -static const struct of_device_id atlas_dt_ids[] = { - { .compatible = "atlas,ph-sm" }, - { } -}; -MODULE_DEVICE_TABLE(of, atlas_dt_ids); - static struct i2c_driver atlas_driver = { .driver = { .name = ATLAS_DRV_NAME, diff --git a/drivers/iio/common/st_sensors/st_sensors_buffer.c b/drivers/iio/common/st_sensors/st_sensors_buffer.c index f1693dbebb8a..d06e728cea37 100644 --- a/drivers/iio/common/st_sensors/st_sensors_buffer.c +++ b/drivers/iio/common/st_sensors/st_sensors_buffer.c @@ -22,34 +22,32 @@ #include -int st_sensors_get_buffer_element(struct iio_dev *indio_dev, u8 *buf) +static int st_sensors_get_buffer_element(struct iio_dev *indio_dev, u8 *buf) { - int i, len; - int total = 0; + int i; struct st_sensor_data *sdata = iio_priv(indio_dev); unsigned int num_data_channels = sdata->num_data_channels; - for (i = 0; i < num_data_channels; i++) { - unsigned int bytes_to_read; - - if (test_bit(i, indio_dev->active_scan_mask)) { - bytes_to_read = indio_dev->channels[i].scan_type.storagebits >> 3; - len = sdata->tf->read_multiple_byte(&sdata->tb, - sdata->dev, indio_dev->channels[i].address, - bytes_to_read, - buf + total, sdata->multiread_bit); - - if (len < bytes_to_read) - return -EIO; - - /* Advance the buffer pointer */ - total += len; - } + for_each_set_bit(i, indio_dev->active_scan_mask, num_data_channels) { + const struct iio_chan_spec *channel = &indio_dev->channels[i]; + unsigned int bytes_to_read = channel->scan_type.realbits >> 3; + unsigned int storage_bytes = + channel->scan_type.storagebits >> 3; + + buf = PTR_ALIGN(buf, storage_bytes); + if (sdata->tf->read_multiple_byte(&sdata->tb, sdata->dev, + channel->address, + bytes_to_read, buf, + sdata->multiread_bit) < + bytes_to_read) + return -EIO; + + /* Advance the buffer pointer */ + buf += storage_bytes; } - return total; + return 0; } -EXPORT_SYMBOL(st_sensors_get_buffer_element); irqreturn_t st_sensors_trigger_handler(int irq, void *p) { @@ -59,11 +57,16 @@ irqreturn_t st_sensors_trigger_handler(int irq, void *p) struct st_sensor_data *sdata = iio_priv(indio_dev); s64 timestamp; - /* If we do timetamping here, do it before reading the values */ + /* + * If we do timetamping here, do it before reading the values, because + * once we've read the values, new interrupts can occur (when using + * the hardware trigger) and the hw_timestamp may get updated. + * By storing it in a local variable first, we are safe. + */ if (sdata->hw_irq_trigger) timestamp = sdata->hw_timestamp; else - timestamp = iio_get_time_ns(); + timestamp = iio_get_time_ns(indio_dev); len = st_sensors_get_buffer_element(indio_dev, sdata->buffer_data); if (len < 0) diff --git a/drivers/iio/common/st_sensors/st_sensors_core.c b/drivers/iio/common/st_sensors/st_sensors_core.c index 9e59c90f6a8d..2d5282e05482 100644 --- a/drivers/iio/common/st_sensors/st_sensors_core.c +++ b/drivers/iio/common/st_sensors/st_sensors_core.c @@ -228,7 +228,7 @@ int st_sensors_set_axis_enable(struct iio_dev *indio_dev, u8 axis_enable) } EXPORT_SYMBOL(st_sensors_set_axis_enable); -void st_sensors_power_enable(struct iio_dev *indio_dev) +int st_sensors_power_enable(struct iio_dev *indio_dev) { struct st_sensor_data *pdata = iio_priv(indio_dev); int err; @@ -237,18 +237,37 @@ void st_sensors_power_enable(struct iio_dev *indio_dev) pdata->vdd = devm_regulator_get_optional(indio_dev->dev.parent, "vdd"); if (!IS_ERR(pdata->vdd)) { err = regulator_enable(pdata->vdd); - if (err != 0) + if (err != 0) { dev_warn(&indio_dev->dev, "Failed to enable specified Vdd supply\n"); + return err; + } + } else { + err = PTR_ERR(pdata->vdd); + if (err != -ENODEV) + return err; } pdata->vdd_io = devm_regulator_get_optional(indio_dev->dev.parent, "vddio"); if (!IS_ERR(pdata->vdd_io)) { err = regulator_enable(pdata->vdd_io); - if (err != 0) + if (err != 0) { dev_warn(&indio_dev->dev, "Failed to enable specified Vdd_IO supply\n"); + goto st_sensors_disable_vdd; + } + } else { + err = PTR_ERR(pdata->vdd_io); + if (err != -ENODEV) + goto st_sensors_disable_vdd; } + + return 0; + +st_sensors_disable_vdd: + if (!IS_ERR_OR_NULL(pdata->vdd)) + regulator_disable(pdata->vdd); + return err; } EXPORT_SYMBOL(st_sensors_power_enable); @@ -256,10 +275,10 @@ void st_sensors_power_disable(struct iio_dev *indio_dev) { struct st_sensor_data *pdata = iio_priv(indio_dev); - if (!IS_ERR(pdata->vdd)) + if (!IS_ERR_OR_NULL(pdata->vdd)) regulator_disable(pdata->vdd); - if (!IS_ERR(pdata->vdd_io)) + if (!IS_ERR_OR_NULL(pdata->vdd_io)) regulator_disable(pdata->vdd_io); } EXPORT_SYMBOL(st_sensors_power_disable); @@ -471,7 +490,7 @@ static int st_sensors_read_axis_data(struct iio_dev *indio_dev, int err; u8 *outdata; struct st_sensor_data *sdata = iio_priv(indio_dev); - unsigned int byte_for_channel = ch->scan_type.storagebits >> 3; + unsigned int byte_for_channel = ch->scan_type.realbits >> 3; outdata = kmalloc(byte_for_channel, GFP_KERNEL); if (!outdata) @@ -531,7 +550,7 @@ int st_sensors_check_device_support(struct iio_dev *indio_dev, int num_sensors_list, const struct st_sensor_settings *sensor_settings) { - int i, n, err; + int i, n, err = 0; u8 wai; struct st_sensor_data *sdata = iio_priv(indio_dev); @@ -551,17 +570,21 @@ int st_sensors_check_device_support(struct iio_dev *indio_dev, return -ENODEV; } - err = sdata->tf->read_byte(&sdata->tb, sdata->dev, - sensor_settings[i].wai_addr, &wai); - if (err < 0) { - dev_err(&indio_dev->dev, "failed to read Who-Am-I register.\n"); - return err; - } + if (sensor_settings[i].wai_addr) { + err = sdata->tf->read_byte(&sdata->tb, sdata->dev, + sensor_settings[i].wai_addr, &wai); + if (err < 0) { + dev_err(&indio_dev->dev, + "failed to read Who-Am-I register.\n"); + return err; + } - if (sensor_settings[i].wai != wai) { - dev_err(&indio_dev->dev, "%s: WhoAmI mismatch (0x%x).\n", - indio_dev->name, wai); - return -EINVAL; + if (sensor_settings[i].wai != wai) { + dev_err(&indio_dev->dev, + "%s: WhoAmI mismatch (0x%x).\n", + indio_dev->name, wai); + return -EINVAL; + } } sdata->sensor_settings = diff --git a/drivers/iio/common/st_sensors/st_sensors_i2c.c b/drivers/iio/common/st_sensors/st_sensors_i2c.c index 98cfee296d46..b43aa36031f8 100644 --- a/drivers/iio/common/st_sensors/st_sensors_i2c.c +++ b/drivers/iio/common/st_sensors/st_sensors_i2c.c @@ -48,8 +48,8 @@ static int st_sensors_i2c_read_multiple_byte( if (multiread_bit) reg_addr |= ST_SENSORS_I2C_MULTIREAD; - return i2c_smbus_read_i2c_block_data(to_i2c_client(dev), - reg_addr, len, data); + return i2c_smbus_read_i2c_block_data_or_emulated(to_i2c_client(dev), + reg_addr, len, data); } static int st_sensors_i2c_write_byte(struct st_sensor_transfer_buffer *tb, diff --git a/drivers/iio/common/st_sensors/st_sensors_trigger.c b/drivers/iio/common/st_sensors/st_sensors_trigger.c index 296e4ff19ae8..e66f12ee8a55 100644 --- a/drivers/iio/common/st_sensors/st_sensors_trigger.c +++ b/drivers/iio/common/st_sensors/st_sensors_trigger.c @@ -17,6 +17,50 @@ #include #include "st_sensors_core.h" +/** + * st_sensors_new_samples_available() - check if more samples came in + * returns: + * 0 - no new samples available + * 1 - new samples available + * negative - error or unknown + */ +static int st_sensors_new_samples_available(struct iio_dev *indio_dev, + struct st_sensor_data *sdata) +{ + u8 status; + int ret; + + /* How would I know if I can't check it? */ + if (!sdata->sensor_settings->drdy_irq.addr_stat_drdy) + return -EINVAL; + + /* No scan mask, no interrupt */ + if (!indio_dev->active_scan_mask) + return 0; + + ret = sdata->tf->read_byte(&sdata->tb, sdata->dev, + sdata->sensor_settings->drdy_irq.addr_stat_drdy, + &status); + if (ret < 0) { + dev_err(sdata->dev, + "error checking samples available\n"); + return ret; + } + /* + * the lower bits of .active_scan_mask[0] is directly mapped + * to the channels on the sensor: either bit 0 for + * one-dimensional sensors, or e.g. x,y,z for accelerometers, + * gyroscopes or magnetometers. No sensor use more than 3 + * channels, so cut the other status bits here. + */ + status &= 0x07; + + if (status & (u8)indio_dev->active_scan_mask[0]) + return 1; + + return 0; +} + /** * st_sensors_irq_handler() - top half of the IRQ-based triggers * @irq: irq number @@ -29,7 +73,7 @@ irqreturn_t st_sensors_irq_handler(int irq, void *p) struct st_sensor_data *sdata = iio_priv(indio_dev); /* Get the time stamp as close in time as possible */ - sdata->hw_timestamp = iio_get_time_ns(); + sdata->hw_timestamp = iio_get_time_ns(indio_dev); return IRQ_WAKE_THREAD; } @@ -43,44 +87,43 @@ irqreturn_t st_sensors_irq_thread(int irq, void *p) struct iio_trigger *trig = p; struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig); struct st_sensor_data *sdata = iio_priv(indio_dev); - int ret; /* * If this trigger is backed by a hardware interrupt and we have a - * status register, check if this IRQ came from us + * status register, check if this IRQ came from us. Notice that + * we will process also if st_sensors_new_samples_available() + * returns negative: if we can't check status, then poll + * unconditionally. */ - if (sdata->sensor_settings->drdy_irq.addr_stat_drdy) { - u8 status; - - ret = sdata->tf->read_byte(&sdata->tb, sdata->dev, - sdata->sensor_settings->drdy_irq.addr_stat_drdy, - &status); - if (ret < 0) { - dev_err(sdata->dev, "could not read channel status\n"); - goto out_poll; - } - /* - * the lower bits of .active_scan_mask[0] is directly mapped - * to the channels on the sensor: either bit 0 for - * one-dimensional sensors, or e.g. x,y,z for accelerometers, - * gyroscopes or magnetometers. No sensor use more than 3 - * channels, so cut the other status bits here. - */ - status &= 0x07; + if (sdata->hw_irq_trigger && + st_sensors_new_samples_available(indio_dev, sdata)) { + iio_trigger_poll_chained(p); + } else { + dev_dbg(sdata->dev, "spurious IRQ\n"); + return IRQ_NONE; + } - /* - * If this was not caused by any channels on this sensor, - * return IRQ_NONE - */ - if (!indio_dev->active_scan_mask) - return IRQ_NONE; - if (!(status & (u8)indio_dev->active_scan_mask[0])) - return IRQ_NONE; + /* + * If we have proper level IRQs the handler will be re-entered if + * the line is still active, so return here and come back in through + * the top half if need be. + */ + if (!sdata->edge_irq) + return IRQ_HANDLED; + + /* + * If we are using egde IRQs, new samples arrived while processing + * the IRQ and those may be missed unless we pick them here, so poll + * again. If the sensor delivery frequency is very high, this thread + * turns into a polled loop handler. + */ + while (sdata->hw_irq_trigger && + st_sensors_new_samples_available(indio_dev, sdata)) { + dev_dbg(sdata->dev, "more samples came in during polling\n"); + sdata->hw_timestamp = iio_get_time_ns(indio_dev); + iio_trigger_poll_chained(p); } -out_poll: - /* It's our IRQ: proceed to handle the register polling */ - iio_trigger_poll_chained(p); return IRQ_HANDLED; } @@ -107,13 +150,18 @@ int st_sensors_allocate_trigger(struct iio_dev *indio_dev, * If the IRQ is triggered on falling edge, we need to mark the * interrupt as active low, if the hardware supports this. */ - if (irq_trig == IRQF_TRIGGER_FALLING) { + switch(irq_trig) { + case IRQF_TRIGGER_FALLING: + case IRQF_TRIGGER_LOW: if (!sdata->sensor_settings->drdy_irq.addr_ihl) { dev_err(&indio_dev->dev, - "falling edge specified for IRQ but hardware " - "only support rising edge, will request " - "rising edge\n"); - irq_trig = IRQF_TRIGGER_RISING; + "falling/low specified for IRQ " + "but hardware only support rising/high: " + "will request rising/high\n"); + if (irq_trig == IRQF_TRIGGER_FALLING) + irq_trig = IRQF_TRIGGER_RISING; + if (irq_trig == IRQF_TRIGGER_LOW) + irq_trig = IRQF_TRIGGER_HIGH; } else { /* Set up INT active low i.e. falling edge */ err = st_sensors_write_data_with_mask(indio_dev, @@ -122,20 +170,39 @@ int st_sensors_allocate_trigger(struct iio_dev *indio_dev, if (err < 0) goto iio_trigger_free; dev_info(&indio_dev->dev, - "interrupts on the falling edge\n"); + "interrupts on the falling edge or " + "active low level\n"); } - } else if (irq_trig == IRQF_TRIGGER_RISING) { + break; + case IRQF_TRIGGER_RISING: dev_info(&indio_dev->dev, "interrupts on the rising edge\n"); - - } else { + break; + case IRQF_TRIGGER_HIGH: + dev_info(&indio_dev->dev, + "interrupts active high level\n"); + break; + default: + /* This is the most preferred mode, if possible */ dev_err(&indio_dev->dev, - "unsupported IRQ trigger specified (%lx), only " - "rising and falling edges supported, enforce " + "unsupported IRQ trigger specified (%lx), enforce " "rising edge\n", irq_trig); irq_trig = IRQF_TRIGGER_RISING; } + /* Tell the interrupt handler that we're dealing with edges */ + if (irq_trig == IRQF_TRIGGER_FALLING || + irq_trig == IRQF_TRIGGER_RISING) + sdata->edge_irq = true; + else + /* + * If we're not using edges (i.e. level interrupts) we + * just mask off the IRQ, handle one interrupt, then + * if the line is still low, we return to the + * interrupt handler top half again and start over. + */ + irq_trig |= IRQF_ONESHOT; + /* * If the interrupt pin is Open Drain, by definition this * means that the interrupt line may be shared with other @@ -148,9 +215,6 @@ int st_sensors_allocate_trigger(struct iio_dev *indio_dev, sdata->sensor_settings->drdy_irq.addr_stat_drdy) irq_trig |= IRQF_SHARED; - /* Let's create an interrupt thread masking the hard IRQ here */ - irq_trig |= IRQF_ONESHOT; - err = request_threaded_irq(sdata->get_irq_data_ready(indio_dev), st_sensors_irq_handler, st_sensors_irq_thread, diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig index f7c71da42f15..ca814479fadf 100644 --- a/drivers/iio/dac/Kconfig +++ b/drivers/iio/dac/Kconfig @@ -248,11 +248,12 @@ config MCP4922 config STX104 tristate "Apex Embedded Systems STX104 DAC driver" depends on X86 && ISA_BUS_API + select GPIOLIB help - Say yes here to build support for the 2-channel DAC on the Apex - Embedded Systems STX104 integrated analog PC/104 card. The base port - addresses for the devices may be configured via the "base" module - parameter array. + Say yes here to build support for the 2-channel DAC and GPIO on the + Apex Embedded Systems STX104 integrated analog PC/104 card. The base + port addresses for the devices may be configured via the base array + module parameter. config VF610_DAC tristate "Vybrid vf610 DAC driver" diff --git a/drivers/iio/dac/ad5421.c b/drivers/iio/dac/ad5421.c index 968712be967f..559061ab1982 100644 --- a/drivers/iio/dac/ad5421.c +++ b/drivers/iio/dac/ad5421.c @@ -242,7 +242,7 @@ static irqreturn_t ad5421_fault_handler(int irq, void *data) 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_RISING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } if (events & AD5421_FAULT_UNDER_CURRENT) { @@ -251,7 +251,7 @@ static irqreturn_t ad5421_fault_handler(int irq, void *data) 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_FALLING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } if (events & AD5421_FAULT_TEMP_OVER_140) { @@ -260,7 +260,7 @@ static irqreturn_t ad5421_fault_handler(int irq, void *data) 0, IIO_EV_TYPE_MAG, IIO_EV_DIR_RISING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } old_fault = fault; diff --git a/drivers/iio/dac/ad5504.c b/drivers/iio/dac/ad5504.c index 4e4c20d6d8b5..788b3d6fd1cc 100644 --- a/drivers/iio/dac/ad5504.c +++ b/drivers/iio/dac/ad5504.c @@ -223,7 +223,7 @@ static irqreturn_t ad5504_event_handler(int irq, void *private) 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_RISING), - iio_get_time_ns()); + iio_get_time_ns((struct iio_dev *)private)); return IRQ_HANDLED; } diff --git a/drivers/iio/dac/ad5755.c b/drivers/iio/dac/ad5755.c index bfb350a85a16..0fde593ec0d9 100644 --- a/drivers/iio/dac/ad5755.c +++ b/drivers/iio/dac/ad5755.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -109,6 +110,51 @@ enum ad5755_type { ID_AD5737, }; +#ifdef CONFIG_OF +static const int ad5755_dcdc_freq_table[][2] = { + { 250000, AD5755_DC_DC_FREQ_250kHZ }, + { 410000, AD5755_DC_DC_FREQ_410kHZ }, + { 650000, AD5755_DC_DC_FREQ_650kHZ } +}; + +static const int ad5755_dcdc_maxv_table[][2] = { + { 23000000, AD5755_DC_DC_MAXV_23V }, + { 24500000, AD5755_DC_DC_MAXV_24V5 }, + { 27000000, AD5755_DC_DC_MAXV_27V }, + { 29500000, AD5755_DC_DC_MAXV_29V5 }, +}; + +static const int ad5755_slew_rate_table[][2] = { + { 64000, AD5755_SLEW_RATE_64k }, + { 32000, AD5755_SLEW_RATE_32k }, + { 16000, AD5755_SLEW_RATE_16k }, + { 8000, AD5755_SLEW_RATE_8k }, + { 4000, AD5755_SLEW_RATE_4k }, + { 2000, AD5755_SLEW_RATE_2k }, + { 1000, AD5755_SLEW_RATE_1k }, + { 500, AD5755_SLEW_RATE_500 }, + { 250, AD5755_SLEW_RATE_250 }, + { 125, AD5755_SLEW_RATE_125 }, + { 64, AD5755_SLEW_RATE_64 }, + { 32, AD5755_SLEW_RATE_32 }, + { 16, AD5755_SLEW_RATE_16 }, + { 8, AD5755_SLEW_RATE_8 }, + { 4, AD5755_SLEW_RATE_4 }, + { 0, AD5755_SLEW_RATE_0_5 }, +}; + +static const int ad5755_slew_step_table[][2] = { + { 256, AD5755_SLEW_STEP_SIZE_256 }, + { 128, AD5755_SLEW_STEP_SIZE_128 }, + { 64, AD5755_SLEW_STEP_SIZE_64 }, + { 32, AD5755_SLEW_STEP_SIZE_32 }, + { 16, AD5755_SLEW_STEP_SIZE_16 }, + { 4, AD5755_SLEW_STEP_SIZE_4 }, + { 2, AD5755_SLEW_STEP_SIZE_2 }, + { 1, AD5755_SLEW_STEP_SIZE_1 }, +}; +#endif + static int ad5755_write_unlocked(struct iio_dev *indio_dev, unsigned int reg, unsigned int val) { @@ -556,6 +602,129 @@ static const struct ad5755_platform_data ad5755_default_pdata = { }, }; +#ifdef CONFIG_OF +static struct ad5755_platform_data *ad5755_parse_dt(struct device *dev) +{ + struct device_node *np = dev->of_node; + struct device_node *pp; + struct ad5755_platform_data *pdata; + unsigned int tmp; + unsigned int tmparray[3]; + int devnr, i; + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return NULL; + + pdata->ext_dc_dc_compenstation_resistor = + of_property_read_bool(np, "adi,ext-dc-dc-compenstation-resistor"); + + if (!of_property_read_u32(np, "adi,dc-dc-phase", &tmp)) + pdata->dc_dc_phase = tmp; + else + pdata->dc_dc_phase = AD5755_DC_DC_PHASE_ALL_SAME_EDGE; + + pdata->dc_dc_freq = AD5755_DC_DC_FREQ_410kHZ; + if (!of_property_read_u32(np, "adi,dc-dc-freq-hz", &tmp)) { + for (i = 0; i < ARRAY_SIZE(ad5755_dcdc_freq_table); i++) { + if (tmp == ad5755_dcdc_freq_table[i][0]) { + pdata->dc_dc_freq = ad5755_dcdc_freq_table[i][1]; + break; + } + } + + if (i == ARRAY_SIZE(ad5755_dcdc_freq_table)) { + dev_err(dev, + "adi,dc-dc-freq out of range selecting 410kHz"); + } + } + + pdata->dc_dc_maxv = AD5755_DC_DC_MAXV_23V; + if (!of_property_read_u32(np, "adi,dc-dc-max-microvolt", &tmp)) { + for (i = 0; i < ARRAY_SIZE(ad5755_dcdc_maxv_table); i++) { + if (tmp == ad5755_dcdc_maxv_table[i][0]) { + pdata->dc_dc_maxv = ad5755_dcdc_maxv_table[i][1]; + break; + } + } + if (i == ARRAY_SIZE(ad5755_dcdc_maxv_table)) { + dev_err(dev, + "adi,dc-dc-maxv out of range selecting 23V"); + } + } + + devnr = 0; + for_each_child_of_node(np, pp) { + if (devnr > AD5755_NUM_CHANNELS) { + dev_err(dev, + "There is to many channels defined in DT\n"); + goto error_out; + } + + if (!of_property_read_u32(pp, "adi,mode", &tmp)) + pdata->dac[devnr].mode = tmp; + else + pdata->dac[devnr].mode = AD5755_MODE_CURRENT_4mA_20mA; + + pdata->dac[devnr].ext_current_sense_resistor = + of_property_read_bool(pp, "adi,ext-current-sense-resistor"); + + pdata->dac[devnr].enable_voltage_overrange = + of_property_read_bool(pp, "adi,enable-voltage-overrange"); + + if (!of_property_read_u32_array(pp, "adi,slew", tmparray, 3)) { + pdata->dac[devnr].slew.enable = tmparray[0]; + + pdata->dac[devnr].slew.rate = AD5755_SLEW_RATE_64k; + for (i = 0; i < ARRAY_SIZE(ad5755_slew_rate_table); i++) { + if (tmparray[1] == ad5755_slew_rate_table[i][0]) { + pdata->dac[devnr].slew.rate = + ad5755_slew_rate_table[i][1]; + break; + } + } + if (i == ARRAY_SIZE(ad5755_slew_rate_table)) { + dev_err(dev, + "channel %d slew rate out of range selecting 64kHz", + devnr); + } + + pdata->dac[devnr].slew.step_size = AD5755_SLEW_STEP_SIZE_1; + for (i = 0; i < ARRAY_SIZE(ad5755_slew_step_table); i++) { + if (tmparray[2] == ad5755_slew_step_table[i][0]) { + pdata->dac[devnr].slew.step_size = + ad5755_slew_step_table[i][1]; + break; + } + } + if (i == ARRAY_SIZE(ad5755_slew_step_table)) { + dev_err(dev, + "channel %d slew step size out of range selecting 1 LSB", + devnr); + } + } else { + pdata->dac[devnr].slew.enable = false; + pdata->dac[devnr].slew.rate = AD5755_SLEW_RATE_64k; + pdata->dac[devnr].slew.step_size = + AD5755_SLEW_STEP_SIZE_1; + } + devnr++; + } + + return pdata; + + error_out: + devm_kfree(dev, pdata); + return NULL; +} +#else +static +struct ad5755_platform_data *ad5755_parse_dt(struct device *dev) +{ + return NULL; +} +#endif + static int ad5755_probe(struct spi_device *spi) { enum ad5755_type type = spi_get_device_id(spi)->driver_data; @@ -583,8 +752,15 @@ static int ad5755_probe(struct spi_device *spi) indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->num_channels = AD5755_NUM_CHANNELS; - if (!pdata) + if (spi->dev.of_node) + pdata = ad5755_parse_dt(&spi->dev); + else + pdata = spi->dev.platform_data; + + if (!pdata) { + dev_warn(&spi->dev, "no platform data? using default\n"); pdata = &ad5755_default_pdata; + } ret = ad5755_init_channels(indio_dev, pdata); if (ret) @@ -607,6 +783,16 @@ static const struct spi_device_id ad5755_id[] = { }; MODULE_DEVICE_TABLE(spi, ad5755_id); +static const struct of_device_id ad5755_of_match[] = { + { .compatible = "adi,ad5755" }, + { .compatible = "adi,ad5755-1" }, + { .compatible = "adi,ad5757" }, + { .compatible = "adi,ad5735" }, + { .compatible = "adi,ad5737" }, + { } +}; +MODULE_DEVICE_TABLE(of, ad5755_of_match); + static struct spi_driver ad5755_driver = { .driver = { .name = "ad5755", diff --git a/drivers/iio/dac/stx104.c b/drivers/iio/dac/stx104.c index 27941220872f..792a97164cb2 100644 --- a/drivers/iio/dac/stx104.c +++ b/drivers/iio/dac/stx104.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include #include #include +#include #define STX104_NUM_CHAN 2 @@ -49,6 +51,20 @@ struct stx104_iio { unsigned base; }; +/** + * struct stx104_gpio - GPIO device private data structure + * @chip: instance of the gpio_chip + * @lock: synchronization lock to prevent I/O race conditions + * @base: base port address of the GPIO device + * @out_state: output bits state + */ +struct stx104_gpio { + struct gpio_chip chip; + spinlock_t lock; + unsigned int base; + unsigned int out_state; +}; + static int stx104_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int *val, int *val2, long mask) { @@ -88,15 +104,81 @@ static const struct iio_chan_spec stx104_channels[STX104_NUM_CHAN] = { STX104_CHAN(1) }; +static int stx104_gpio_get_direction(struct gpio_chip *chip, + unsigned int offset) +{ + if (offset < 4) + return 1; + + return 0; +} + +static int stx104_gpio_direction_input(struct gpio_chip *chip, + unsigned int offset) +{ + if (offset >= 4) + return -EINVAL; + + return 0; +} + +static int stx104_gpio_direction_output(struct gpio_chip *chip, + unsigned int offset, int value) +{ + if (offset < 4) + return -EINVAL; + + chip->set(chip, offset, value); + return 0; +} + +static int stx104_gpio_get(struct gpio_chip *chip, unsigned int offset) +{ + struct stx104_gpio *const stx104gpio = gpiochip_get_data(chip); + + if (offset >= 4) + return -EINVAL; + + return !!(inb(stx104gpio->base) & BIT(offset)); +} + +static void stx104_gpio_set(struct gpio_chip *chip, unsigned int offset, + int value) +{ + struct stx104_gpio *const stx104gpio = gpiochip_get_data(chip); + const unsigned int mask = BIT(offset) >> 4; + unsigned long flags; + + if (offset < 4) + return; + + spin_lock_irqsave(&stx104gpio->lock, flags); + + if (value) + stx104gpio->out_state |= mask; + else + stx104gpio->out_state &= ~mask; + + outb(stx104gpio->out_state, stx104gpio->base); + + spin_unlock_irqrestore(&stx104gpio->lock, flags); +} + static int stx104_probe(struct device *dev, unsigned int id) { struct iio_dev *indio_dev; struct stx104_iio *priv; + struct stx104_gpio *stx104gpio; + int err; indio_dev = devm_iio_device_alloc(dev, sizeof(*priv)); if (!indio_dev) return -ENOMEM; + stx104gpio = devm_kzalloc(dev, sizeof(*stx104gpio), GFP_KERNEL); + if (!stx104gpio) + return -ENOMEM; + if (!devm_request_region(dev, base[id], STX104_EXTENT, dev_name(dev))) { dev_err(dev, "Unable to lock port addresses (0x%X-0x%X)\n", @@ -117,14 +199,53 @@ static int stx104_probe(struct device *dev, unsigned int id) outw(0, base[id] + 4); outw(0, base[id] + 6); - return devm_iio_device_register(dev, indio_dev); + err = devm_iio_device_register(dev, indio_dev); + if (err) { + dev_err(dev, "IIO device registering failed (%d)\n", err); + return err; + } + + stx104gpio->chip.label = dev_name(dev); + stx104gpio->chip.parent = dev; + stx104gpio->chip.owner = THIS_MODULE; + stx104gpio->chip.base = -1; + stx104gpio->chip.ngpio = 8; + stx104gpio->chip.get_direction = stx104_gpio_get_direction; + stx104gpio->chip.direction_input = stx104_gpio_direction_input; + stx104gpio->chip.direction_output = stx104_gpio_direction_output; + stx104gpio->chip.get = stx104_gpio_get; + stx104gpio->chip.set = stx104_gpio_set; + stx104gpio->base = base[id] + 3; + stx104gpio->out_state = 0x0; + + spin_lock_init(&stx104gpio->lock); + + dev_set_drvdata(dev, stx104gpio); + + err = gpiochip_add_data(&stx104gpio->chip, stx104gpio); + if (err) { + dev_err(dev, "GPIO registering failed (%d)\n", err); + return err; + } + + return 0; +} + +static int stx104_remove(struct device *dev, unsigned int id) +{ + struct stx104_gpio *const stx104gpio = dev_get_drvdata(dev); + + gpiochip_remove(&stx104gpio->chip); + + return 0; } static struct isa_driver stx104_driver = { .probe = stx104_probe, .driver = { .name = "stx104" - } + }, + .remove = stx104_remove }; module_isa_driver(stx104_driver, num_stx104); diff --git a/drivers/iio/dummy/Kconfig b/drivers/iio/dummy/Kconfig index 71805ced1aae..aa5824d96a43 100644 --- a/drivers/iio/dummy/Kconfig +++ b/drivers/iio/dummy/Kconfig @@ -10,6 +10,7 @@ config IIO_DUMMY_EVGEN config IIO_SIMPLE_DUMMY tristate "An example driver with no hardware requirements" + depends on IIO_SW_DEVICE help Driver intended mainly as documentation for how to write a driver. May also be useful for testing userspace code diff --git a/drivers/iio/dummy/iio_simple_dummy.c b/drivers/iio/dummy/iio_simple_dummy.c index 43fe4ba7d0dc..ad3410e528b6 100644 --- a/drivers/iio/dummy/iio_simple_dummy.c +++ b/drivers/iio/dummy/iio_simple_dummy.c @@ -17,26 +17,18 @@ #include #include #include +#include #include #include #include #include +#include #include "iio_simple_dummy.h" -/* - * A few elements needed to fake a bus for this driver - * Note instances parameter controls how many of these - * dummy devices are registered. - */ -static unsigned instances = 1; -module_param(instances, uint, 0); - -/* Pointer array used to fake bus elements */ -static struct iio_dev **iio_dummy_devs; - -/* Fake a name for the part number, usually obtained from the id table */ -static const char *iio_dummy_part_number = "iio_dummy_part_no"; +static struct config_item_type iio_dummy_type = { + .ct_owner = THIS_MODULE, +}; /** * struct iio_dummy_accel_calibscale - realworld to register mapping @@ -572,12 +564,18 @@ static int iio_dummy_init_device(struct iio_dev *indio_dev) * const struct i2c_device_id *id) * SPI: iio_dummy_probe(struct spi_device *spi) */ -static int iio_dummy_probe(int index) +static struct iio_sw_device *iio_dummy_probe(const char *name) { int ret; struct iio_dev *indio_dev; struct iio_dummy_state *st; + struct iio_sw_device *swd; + swd = kzalloc(sizeof(*swd), GFP_KERNEL); + if (!swd) { + ret = -ENOMEM; + goto error_kzalloc; + } /* * Allocate an IIO device. * @@ -608,7 +606,7 @@ static int iio_dummy_probe(int index) * i2c_set_clientdata(client, indio_dev); * spi_set_drvdata(spi, indio_dev); */ - iio_dummy_devs[index] = indio_dev; + swd->device = indio_dev; /* * Set the device name. @@ -619,7 +617,7 @@ static int iio_dummy_probe(int index) * indio_dev->name = id->name; * indio_dev->name = spi_get_device_id(spi)->name; */ - indio_dev->name = iio_dummy_part_number; + indio_dev->name = kstrdup(name, GFP_KERNEL); /* Provide description of available channels */ indio_dev->channels = iio_dummy_channels; @@ -646,7 +644,9 @@ static int iio_dummy_probe(int index) if (ret < 0) goto error_unconfigure_buffer; - return 0; + iio_swd_group_init_type_name(swd, name, &iio_dummy_type); + + return swd; error_unconfigure_buffer: iio_simple_dummy_unconfigure_buffer(indio_dev); error_unregister_events: @@ -654,16 +654,18 @@ error_unregister_events: error_free_device: iio_device_free(indio_dev); error_ret: - return ret; + kfree(swd); +error_kzalloc: + return ERR_PTR(ret); } /** * iio_dummy_remove() - device instance removal function - * @index: device index. + * @swd: pointer to software IIO device abstraction * * Parameters follow those of iio_dummy_probe for buses. */ -static void iio_dummy_remove(int index) +static int iio_dummy_remove(struct iio_sw_device *swd) { /* * Get a pointer to the device instance iio_dev structure @@ -671,7 +673,7 @@ static void iio_dummy_remove(int index) * struct iio_dev *indio_dev = i2c_get_clientdata(client); * struct iio_dev *indio_dev = spi_get_drvdata(spi); */ - struct iio_dev *indio_dev = iio_dummy_devs[index]; + struct iio_dev *indio_dev = swd->device; /* Unregister the device */ iio_device_unregister(indio_dev); @@ -684,11 +686,13 @@ static void iio_dummy_remove(int index) iio_simple_dummy_events_unregister(indio_dev); /* Free all structures */ + kfree(indio_dev->name); iio_device_free(indio_dev); -} + return 0; +} /** - * iio_dummy_init() - device driver registration + * module_iio_sw_device_driver() - device driver registration * * Varies depending on bus type of the device. As there is no device * here, call probe directly. For information on device registration @@ -697,50 +701,18 @@ static void iio_dummy_remove(int index) * spi: * Documentation/spi/spi-summary */ -static __init int iio_dummy_init(void) -{ - int i, ret; - - if (instances > 10) { - instances = 1; - return -EINVAL; - } - - /* Fake a bus */ - iio_dummy_devs = kcalloc(instances, sizeof(*iio_dummy_devs), - GFP_KERNEL); - /* Here we have no actual device so call probe */ - for (i = 0; i < instances; i++) { - ret = iio_dummy_probe(i); - if (ret < 0) - goto error_remove_devs; - } - return 0; - -error_remove_devs: - while (i--) - iio_dummy_remove(i); - - kfree(iio_dummy_devs); - return ret; -} -module_init(iio_dummy_init); +static const struct iio_sw_device_ops iio_dummy_device_ops = { + .probe = iio_dummy_probe, + .remove = iio_dummy_remove, +}; -/** - * iio_dummy_exit() - device driver removal - * - * Varies depending on bus type of the device. - * As there is no device here, call remove directly. - */ -static __exit void iio_dummy_exit(void) -{ - int i; +static struct iio_sw_device_type iio_dummy_device = { + .name = "dummy", + .owner = THIS_MODULE, + .ops = &iio_dummy_device_ops, +}; - for (i = 0; i < instances; i++) - iio_dummy_remove(i); - kfree(iio_dummy_devs); -} -module_exit(iio_dummy_exit); +module_iio_sw_device_driver(iio_dummy_device); MODULE_AUTHOR("Jonathan Cameron "); MODULE_DESCRIPTION("IIO dummy driver"); diff --git a/drivers/iio/dummy/iio_simple_dummy_buffer.c b/drivers/iio/dummy/iio_simple_dummy_buffer.c index cf44a6f79431..b383892a5193 100644 --- a/drivers/iio/dummy/iio_simple_dummy_buffer.c +++ b/drivers/iio/dummy/iio_simple_dummy_buffer.c @@ -85,7 +85,8 @@ static irqreturn_t iio_simple_dummy_trigger_h(int irq, void *p) } } - iio_push_to_buffers_with_timestamp(indio_dev, data, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, data, + iio_get_time_ns(indio_dev)); kfree(data); diff --git a/drivers/iio/dummy/iio_simple_dummy_events.c b/drivers/iio/dummy/iio_simple_dummy_events.c index 6eb600ff7056..ed63ffd849f8 100644 --- a/drivers/iio/dummy/iio_simple_dummy_events.c +++ b/drivers/iio/dummy/iio_simple_dummy_events.c @@ -158,7 +158,7 @@ static irqreturn_t iio_simple_dummy_get_timestamp(int irq, void *private) struct iio_dev *indio_dev = private; struct iio_dummy_state *st = iio_priv(indio_dev); - st->event_timestamp = iio_get_time_ns(); + st->event_timestamp = iio_get_time_ns(indio_dev); return IRQ_WAKE_THREAD; } diff --git a/drivers/iio/gyro/bmg160_core.c b/drivers/iio/gyro/bmg160_core.c index 7ccc044063f6..f7fcfa886f72 100644 --- a/drivers/iio/gyro/bmg160_core.c +++ b/drivers/iio/gyro/bmg160_core.c @@ -50,6 +50,7 @@ #define BMG160_REG_PMU_BW 0x10 #define BMG160_NO_FILTER 0 #define BMG160_DEF_BW 100 +#define BMG160_REG_PMU_BW_RES BIT(7) #define BMG160_REG_INT_MAP_0 0x17 #define BMG160_INT_MAP_0_BIT_ANY BIT(1) @@ -100,7 +101,6 @@ struct bmg160_data { struct iio_trigger *motion_trig; struct mutex mutex; s16 buffer[8]; - u8 bw_bits; u32 dps_range; int ev_enable_state; int slope_thres; @@ -117,13 +117,16 @@ enum bmg160_axis { }; static const struct { - int val; + int odr; + int filter; int bw_bits; -} bmg160_samp_freq_table[] = { {100, 0x07}, - {200, 0x06}, - {400, 0x03}, - {1000, 0x02}, - {2000, 0x01} }; +} bmg160_samp_freq_table[] = { {100, 32, 0x07}, + {200, 64, 0x06}, + {100, 12, 0x05}, + {200, 23, 0x04}, + {400, 47, 0x03}, + {1000, 116, 0x02}, + {2000, 230, 0x01} }; static const struct { int scale; @@ -153,7 +156,7 @@ static int bmg160_convert_freq_to_bit(int val) int i; for (i = 0; i < ARRAY_SIZE(bmg160_samp_freq_table); ++i) { - if (bmg160_samp_freq_table[i].val == val) + if (bmg160_samp_freq_table[i].odr == val) return bmg160_samp_freq_table[i].bw_bits; } @@ -176,7 +179,53 @@ static int bmg160_set_bw(struct bmg160_data *data, int val) return ret; } - data->bw_bits = bw_bits; + return 0; +} + +static int bmg160_get_filter(struct bmg160_data *data, int *val) +{ + struct device *dev = regmap_get_device(data->regmap); + int ret; + int i; + unsigned int bw_bits; + + ret = regmap_read(data->regmap, BMG160_REG_PMU_BW, &bw_bits); + if (ret < 0) { + dev_err(dev, "Error reading reg_pmu_bw\n"); + return ret; + } + + /* Ignore the readonly reserved bit. */ + bw_bits &= ~BMG160_REG_PMU_BW_RES; + + for (i = 0; i < ARRAY_SIZE(bmg160_samp_freq_table); ++i) { + if (bmg160_samp_freq_table[i].bw_bits == bw_bits) + break; + } + + *val = bmg160_samp_freq_table[i].filter; + + return ret ? ret : IIO_VAL_INT; +} + + +static int bmg160_set_filter(struct bmg160_data *data, int val) +{ + struct device *dev = regmap_get_device(data->regmap); + int ret; + int i; + + for (i = 0; i < ARRAY_SIZE(bmg160_samp_freq_table); ++i) { + if (bmg160_samp_freq_table[i].filter == val) + break; + } + + ret = regmap_write(data->regmap, BMG160_REG_PMU_BW, + bmg160_samp_freq_table[i].bw_bits); + if (ret < 0) { + dev_err(dev, "Error writing reg_pmu_bw\n"); + return ret; + } return 0; } @@ -386,11 +435,23 @@ static int bmg160_setup_new_data_interrupt(struct bmg160_data *data, static int bmg160_get_bw(struct bmg160_data *data, int *val) { + struct device *dev = regmap_get_device(data->regmap); int i; + unsigned int bw_bits; + int ret; + + ret = regmap_read(data->regmap, BMG160_REG_PMU_BW, &bw_bits); + if (ret < 0) { + dev_err(dev, "Error reading reg_pmu_bw\n"); + return ret; + } + + /* Ignore the readonly reserved bit. */ + bw_bits &= ~BMG160_REG_PMU_BW_RES; for (i = 0; i < ARRAY_SIZE(bmg160_samp_freq_table); ++i) { - if (bmg160_samp_freq_table[i].bw_bits == data->bw_bits) { - *val = bmg160_samp_freq_table[i].val; + if (bmg160_samp_freq_table[i].bw_bits == bw_bits) { + *val = bmg160_samp_freq_table[i].odr; return IIO_VAL_INT; } } @@ -507,6 +568,8 @@ static int bmg160_read_raw(struct iio_dev *indio_dev, return IIO_VAL_INT; } else return -EINVAL; + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + return bmg160_get_filter(data, val); case IIO_CHAN_INFO_SCALE: *val = 0; switch (chan->type) { @@ -571,6 +634,26 @@ static int bmg160_write_raw(struct iio_dev *indio_dev, ret = bmg160_set_power_state(data, false); mutex_unlock(&data->mutex); return ret; + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + if (val2) + return -EINVAL; + + mutex_lock(&data->mutex); + ret = bmg160_set_power_state(data, true); + if (ret < 0) { + bmg160_set_power_state(data, false); + mutex_unlock(&data->mutex); + return ret; + } + ret = bmg160_set_filter(data, val); + if (ret < 0) { + bmg160_set_power_state(data, false); + mutex_unlock(&data->mutex); + return ret; + } + ret = bmg160_set_power_state(data, false); + mutex_unlock(&data->mutex); + return ret; case IIO_CHAN_INFO_SCALE: if (val) return -EINVAL; @@ -728,7 +811,8 @@ static const struct iio_event_spec bmg160_event = { .channel2 = IIO_MOD_##_axis, \ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | \ - BIT(IIO_CHAN_INFO_SAMP_FREQ), \ + BIT(IIO_CHAN_INFO_SAMP_FREQ) | \ + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), \ .scan_index = AXIS_##_axis, \ .scan_type = { \ .sign = 's', \ @@ -885,25 +969,25 @@ static irqreturn_t bmg160_event_handler(int irq, void *private) if (val & BMG160_ANY_MOTION_BIT_X) iio_push_event(indio_dev, IIO_MOD_EVENT_CODE(IIO_ANGL_VEL, - 0, - IIO_MOD_X, - IIO_EV_TYPE_ROC, - dir), - iio_get_time_ns()); + 0, + IIO_MOD_X, + IIO_EV_TYPE_ROC, + dir), + iio_get_time_ns(indio_dev)); if (val & BMG160_ANY_MOTION_BIT_Y) iio_push_event(indio_dev, IIO_MOD_EVENT_CODE(IIO_ANGL_VEL, - 0, - IIO_MOD_Y, - IIO_EV_TYPE_ROC, - dir), - iio_get_time_ns()); + 0, + IIO_MOD_Y, + IIO_EV_TYPE_ROC, + dir), + iio_get_time_ns(indio_dev)); if (val & BMG160_ANY_MOTION_BIT_Z) iio_push_event(indio_dev, IIO_MOD_EVENT_CODE(IIO_ANGL_VEL, - 0, - IIO_MOD_Z, - IIO_EV_TYPE_ROC, - dir), - iio_get_time_ns()); + 0, + IIO_MOD_Z, + IIO_EV_TYPE_ROC, + dir), + iio_get_time_ns(indio_dev)); ack_intr_status: if (!data->dready_trigger_on) { diff --git a/drivers/iio/gyro/st_gyro_core.c b/drivers/iio/gyro/st_gyro_core.c index a8012955a1f6..aea034d8fe0f 100644 --- a/drivers/iio/gyro/st_gyro_core.c +++ b/drivers/iio/gyro/st_gyro_core.c @@ -426,13 +426,15 @@ int st_gyro_common_probe(struct iio_dev *indio_dev) indio_dev->info = &gyro_info; mutex_init(&gdata->tb.buf_lock); - st_sensors_power_enable(indio_dev); + err = st_sensors_power_enable(indio_dev); + if (err) + return err; err = st_sensors_check_device_support(indio_dev, ARRAY_SIZE(st_gyro_sensors_settings), st_gyro_sensors_settings); if (err < 0) - return err; + goto st_gyro_power_off; gdata->num_data_channels = ST_GYRO_NUMBER_DATA_CHANNELS; gdata->multiread_bit = gdata->sensor_settings->multi_read_bit; @@ -446,11 +448,11 @@ int st_gyro_common_probe(struct iio_dev *indio_dev) err = st_sensors_init_sensor(indio_dev, (struct st_sensors_platform_data *)&gyro_pdata); if (err < 0) - return err; + goto st_gyro_power_off; err = st_gyro_allocate_ring(indio_dev); if (err < 0) - return err; + goto st_gyro_power_off; if (irq > 0) { err = st_sensors_allocate_trigger(indio_dev, @@ -473,6 +475,8 @@ st_gyro_device_register_error: st_sensors_deallocate_trigger(indio_dev); st_gyro_probe_trigger_error: st_gyro_deallocate_ring(indio_dev); +st_gyro_power_off: + st_sensors_power_disable(indio_dev); return err; } diff --git a/drivers/iio/health/afe4403.c b/drivers/iio/health/afe4403.c index 88e43f87b926..9a081465c42f 100644 --- a/drivers/iio/health/afe4403.c +++ b/drivers/iio/health/afe4403.c @@ -1,7 +1,7 @@ /* * AFE4403 Heart Rate Monitors and Low-Cost Pulse Oximeters * - * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com/ + * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/ * Andrew F. Davis * * This program is free software; you can redistribute it and/or modify @@ -39,127 +39,90 @@ #define AFE4403_TIAGAIN 0x20 #define AFE4403_TIA_AMB_GAIN 0x21 -/* AFE4403 GAIN register fields */ -#define AFE4403_TIAGAIN_RES_MASK GENMASK(2, 0) -#define AFE4403_TIAGAIN_RES_SHIFT 0 -#define AFE4403_TIAGAIN_CAP_MASK GENMASK(7, 3) -#define AFE4403_TIAGAIN_CAP_SHIFT 3 - -/* AFE4403 LEDCNTRL register fields */ -#define AFE440X_LEDCNTRL_LED1_MASK GENMASK(15, 8) -#define AFE440X_LEDCNTRL_LED1_SHIFT 8 -#define AFE440X_LEDCNTRL_LED2_MASK GENMASK(7, 0) -#define AFE440X_LEDCNTRL_LED2_SHIFT 0 -#define AFE440X_LEDCNTRL_LED_RANGE_MASK GENMASK(17, 16) -#define AFE440X_LEDCNTRL_LED_RANGE_SHIFT 16 - -/* AFE4403 CONTROL2 register fields */ -#define AFE440X_CONTROL2_PWR_DWN_TX BIT(2) -#define AFE440X_CONTROL2_EN_SLOW_DIAG BIT(8) -#define AFE440X_CONTROL2_DIAG_OUT_TRI BIT(10) -#define AFE440X_CONTROL2_TX_BRDG_MOD BIT(11) -#define AFE440X_CONTROL2_TX_REF_MASK GENMASK(18, 17) -#define AFE440X_CONTROL2_TX_REF_SHIFT 17 - -/* AFE4404 NULL fields */ -#define NULL_MASK 0 -#define NULL_SHIFT 0 - -/* AFE4403 LEDCNTRL values */ -#define AFE440X_LEDCNTRL_RANGE_TX_HALF 0x1 -#define AFE440X_LEDCNTRL_RANGE_TX_FULL 0x2 -#define AFE440X_LEDCNTRL_RANGE_TX_OFF 0x3 - -/* AFE4403 CONTROL2 values */ -#define AFE440X_CONTROL2_TX_REF_025 0x0 -#define AFE440X_CONTROL2_TX_REF_050 0x1 -#define AFE440X_CONTROL2_TX_REF_100 0x2 -#define AFE440X_CONTROL2_TX_REF_075 0x3 - -/* AFE4403 CONTROL3 values */ -#define AFE440X_CONTROL3_CLK_DIV_2 0x0 -#define AFE440X_CONTROL3_CLK_DIV_4 0x2 -#define AFE440X_CONTROL3_CLK_DIV_6 0x3 -#define AFE440X_CONTROL3_CLK_DIV_8 0x4 -#define AFE440X_CONTROL3_CLK_DIV_12 0x5 -#define AFE440X_CONTROL3_CLK_DIV_1 0x7 - -/* AFE4403 TIAGAIN_CAP values */ -#define AFE4403_TIAGAIN_CAP_5_P 0x0 -#define AFE4403_TIAGAIN_CAP_10_P 0x1 -#define AFE4403_TIAGAIN_CAP_20_P 0x2 -#define AFE4403_TIAGAIN_CAP_30_P 0x3 -#define AFE4403_TIAGAIN_CAP_55_P 0x8 -#define AFE4403_TIAGAIN_CAP_155_P 0x10 - -/* AFE4403 TIAGAIN_RES values */ -#define AFE4403_TIAGAIN_RES_500_K 0x0 -#define AFE4403_TIAGAIN_RES_250_K 0x1 -#define AFE4403_TIAGAIN_RES_100_K 0x2 -#define AFE4403_TIAGAIN_RES_50_K 0x3 -#define AFE4403_TIAGAIN_RES_25_K 0x4 -#define AFE4403_TIAGAIN_RES_10_K 0x5 -#define AFE4403_TIAGAIN_RES_1_M 0x6 -#define AFE4403_TIAGAIN_RES_NONE 0x7 +enum afe4403_fields { + /* Gains */ + F_RF_LED1, F_CF_LED1, + F_RF_LED, F_CF_LED, + + /* LED Current */ + F_ILED1, F_ILED2, + + /* sentinel */ + F_MAX_FIELDS +}; + +static const struct reg_field afe4403_reg_fields[] = { + /* Gains */ + [F_RF_LED1] = REG_FIELD(AFE4403_TIAGAIN, 0, 2), + [F_CF_LED1] = REG_FIELD(AFE4403_TIAGAIN, 3, 7), + [F_RF_LED] = REG_FIELD(AFE4403_TIA_AMB_GAIN, 0, 2), + [F_CF_LED] = REG_FIELD(AFE4403_TIA_AMB_GAIN, 3, 7), + /* LED Current */ + [F_ILED1] = REG_FIELD(AFE440X_LEDCNTRL, 0, 7), + [F_ILED2] = REG_FIELD(AFE440X_LEDCNTRL, 8, 15), +}; /** - * struct afe4403_data - * @dev - Device structure - * @spi - SPI device handle - * @regmap - Register map of the device - * @regulator - Pointer to the regulator for the IC - * @trig - IIO trigger for this device - * @irq - ADC_RDY line interrupt number + * struct afe4403_data - AFE4403 device instance data + * @dev: Device structure + * @spi: SPI device handle + * @regmap: Register map of the device + * @fields: Register fields of the device + * @regulator: Pointer to the regulator for the IC + * @trig: IIO trigger for this device + * @irq: ADC_RDY line interrupt number */ struct afe4403_data { struct device *dev; struct spi_device *spi; struct regmap *regmap; + struct regmap_field *fields[F_MAX_FIELDS]; struct regulator *regulator; struct iio_trigger *trig; int irq; }; enum afe4403_chan_id { + LED2 = 1, + ALED2, LED1, ALED1, - LED2, - ALED2, - LED1_ALED1, LED2_ALED2, - ILED1, - ILED2, + LED1_ALED1, }; -static const struct afe440x_reg_info afe4403_reg_info[] = { - [LED1] = AFE440X_REG_INFO(AFE440X_LED1VAL, 0, NULL), - [ALED1] = AFE440X_REG_INFO(AFE440X_ALED1VAL, 0, NULL), - [LED2] = AFE440X_REG_INFO(AFE440X_LED2VAL, 0, NULL), - [ALED2] = AFE440X_REG_INFO(AFE440X_ALED2VAL, 0, NULL), - [LED1_ALED1] = AFE440X_REG_INFO(AFE440X_LED1_ALED1VAL, 0, NULL), - [LED2_ALED2] = AFE440X_REG_INFO(AFE440X_LED2_ALED2VAL, 0, NULL), - [ILED1] = AFE440X_REG_INFO(AFE440X_LEDCNTRL, 0, AFE440X_LEDCNTRL_LED1), - [ILED2] = AFE440X_REG_INFO(AFE440X_LEDCNTRL, 0, AFE440X_LEDCNTRL_LED2), +static const unsigned int afe4403_channel_values[] = { + [LED2] = AFE440X_LED2VAL, + [ALED2] = AFE440X_ALED2VAL, + [LED1] = AFE440X_LED1VAL, + [ALED1] = AFE440X_ALED1VAL, + [LED2_ALED2] = AFE440X_LED2_ALED2VAL, + [LED1_ALED1] = AFE440X_LED1_ALED1VAL, +}; + +static const unsigned int afe4403_channel_leds[] = { + [LED2] = F_ILED2, + [LED1] = F_ILED1, }; static const struct iio_chan_spec afe4403_channels[] = { /* ADC values */ - AFE440X_INTENSITY_CHAN(LED1, "led1", 0), - AFE440X_INTENSITY_CHAN(ALED1, "led1_ambient", 0), - AFE440X_INTENSITY_CHAN(LED2, "led2", 0), - AFE440X_INTENSITY_CHAN(ALED2, "led2_ambient", 0), - AFE440X_INTENSITY_CHAN(LED1_ALED1, "led1-led1_ambient", 0), - AFE440X_INTENSITY_CHAN(LED2_ALED2, "led2-led2_ambient", 0), + AFE440X_INTENSITY_CHAN(LED2, 0), + AFE440X_INTENSITY_CHAN(ALED2, 0), + AFE440X_INTENSITY_CHAN(LED1, 0), + AFE440X_INTENSITY_CHAN(ALED1, 0), + AFE440X_INTENSITY_CHAN(LED2_ALED2, 0), + AFE440X_INTENSITY_CHAN(LED1_ALED1, 0), /* LED current */ - AFE440X_CURRENT_CHAN(ILED1, "led1"), - AFE440X_CURRENT_CHAN(ILED2, "led2"), + AFE440X_CURRENT_CHAN(LED2), + AFE440X_CURRENT_CHAN(LED1), }; static const struct afe440x_val_table afe4403_res_table[] = { { 500000 }, { 250000 }, { 100000 }, { 50000 }, { 25000 }, { 10000 }, { 1000000 }, { 0 }, }; -AFE440X_TABLE_ATTR(tia_resistance_available, afe4403_res_table); +AFE440X_TABLE_ATTR(in_intensity_resistance_available, afe4403_res_table); static const struct afe440x_val_table afe4403_cap_table[] = { { 0, 5000 }, { 0, 10000 }, { 0, 20000 }, { 0, 25000 }, @@ -171,7 +134,7 @@ static const struct afe440x_val_table afe4403_cap_table[] = { { 0, 205000 }, { 0, 210000 }, { 0, 220000 }, { 0, 225000 }, { 0, 230000 }, { 0, 235000 }, { 0, 245000 }, { 0, 250000 }, }; -AFE440X_TABLE_ATTR(tia_capacitance_available, afe4403_cap_table); +AFE440X_TABLE_ATTR(in_intensity_capacitance_available, afe4403_cap_table); static ssize_t afe440x_show_register(struct device *dev, struct device_attribute *attr, @@ -180,38 +143,21 @@ static ssize_t afe440x_show_register(struct device *dev, struct iio_dev *indio_dev = dev_to_iio_dev(dev); struct afe4403_data *afe = iio_priv(indio_dev); struct afe440x_attr *afe440x_attr = to_afe440x_attr(attr); - unsigned int reg_val, type; + unsigned int reg_val; int vals[2]; - int ret, val_len; + int ret; - ret = regmap_read(afe->regmap, afe440x_attr->reg, ®_val); + ret = regmap_field_read(afe->fields[afe440x_attr->field], ®_val); if (ret) return ret; - reg_val &= afe440x_attr->mask; - reg_val >>= afe440x_attr->shift; - - switch (afe440x_attr->type) { - case SIMPLE: - type = IIO_VAL_INT; - val_len = 1; - vals[0] = reg_val; - break; - case RESISTANCE: - case CAPACITANCE: - type = IIO_VAL_INT_PLUS_MICRO; - val_len = 2; - if (reg_val < afe440x_attr->table_size) { - vals[0] = afe440x_attr->val_table[reg_val].integer; - vals[1] = afe440x_attr->val_table[reg_val].fract; - break; - } - return -EINVAL; - default: + if (reg_val >= afe440x_attr->table_size) return -EINVAL; - } - return iio_format_value(buf, type, val_len, vals); + vals[0] = afe440x_attr->val_table[reg_val].integer; + vals[1] = afe440x_attr->val_table[reg_val].fract; + + return iio_format_value(buf, IIO_VAL_INT_PLUS_MICRO, 2, vals); } static ssize_t afe440x_store_register(struct device *dev, @@ -227,48 +173,43 @@ static ssize_t afe440x_store_register(struct device *dev, if (ret) return ret; - switch (afe440x_attr->type) { - case SIMPLE: - val = integer; - break; - case RESISTANCE: - case CAPACITANCE: - for (val = 0; val < afe440x_attr->table_size; val++) - if (afe440x_attr->val_table[val].integer == integer && - afe440x_attr->val_table[val].fract == fract) - break; - if (val == afe440x_attr->table_size) - return -EINVAL; - break; - default: + for (val = 0; val < afe440x_attr->table_size; val++) + if (afe440x_attr->val_table[val].integer == integer && + afe440x_attr->val_table[val].fract == fract) + break; + if (val == afe440x_attr->table_size) return -EINVAL; - } - ret = regmap_update_bits(afe->regmap, afe440x_attr->reg, - afe440x_attr->mask, - (val << afe440x_attr->shift)); + ret = regmap_field_write(afe->fields[afe440x_attr->field], val); if (ret) return ret; return count; } -static AFE440X_ATTR(tia_separate_en, AFE4403_TIAGAIN, AFE440X_TIAGAIN_ENSEPGAIN, SIMPLE, NULL, 0); +static AFE440X_ATTR(in_intensity1_resistance, F_RF_LED, afe4403_res_table); +static AFE440X_ATTR(in_intensity1_capacitance, F_CF_LED, afe4403_cap_table); + +static AFE440X_ATTR(in_intensity2_resistance, F_RF_LED, afe4403_res_table); +static AFE440X_ATTR(in_intensity2_capacitance, F_CF_LED, afe4403_cap_table); -static AFE440X_ATTR(tia_resistance1, AFE4403_TIAGAIN, AFE4403_TIAGAIN_RES, RESISTANCE, afe4403_res_table, ARRAY_SIZE(afe4403_res_table)); -static AFE440X_ATTR(tia_capacitance1, AFE4403_TIAGAIN, AFE4403_TIAGAIN_CAP, CAPACITANCE, afe4403_cap_table, ARRAY_SIZE(afe4403_cap_table)); +static AFE440X_ATTR(in_intensity3_resistance, F_RF_LED1, afe4403_res_table); +static AFE440X_ATTR(in_intensity3_capacitance, F_CF_LED1, afe4403_cap_table); -static AFE440X_ATTR(tia_resistance2, AFE4403_TIA_AMB_GAIN, AFE4403_TIAGAIN_RES, RESISTANCE, afe4403_res_table, ARRAY_SIZE(afe4403_res_table)); -static AFE440X_ATTR(tia_capacitance2, AFE4403_TIA_AMB_GAIN, AFE4403_TIAGAIN_RES, CAPACITANCE, afe4403_cap_table, ARRAY_SIZE(afe4403_cap_table)); +static AFE440X_ATTR(in_intensity4_resistance, F_RF_LED1, afe4403_res_table); +static AFE440X_ATTR(in_intensity4_capacitance, F_CF_LED1, afe4403_cap_table); static struct attribute *afe440x_attributes[] = { - &afe440x_attr_tia_separate_en.dev_attr.attr, - &afe440x_attr_tia_resistance1.dev_attr.attr, - &afe440x_attr_tia_capacitance1.dev_attr.attr, - &afe440x_attr_tia_resistance2.dev_attr.attr, - &afe440x_attr_tia_capacitance2.dev_attr.attr, - &dev_attr_tia_resistance_available.attr, - &dev_attr_tia_capacitance_available.attr, + &dev_attr_in_intensity_resistance_available.attr, + &dev_attr_in_intensity_capacitance_available.attr, + &afe440x_attr_in_intensity1_resistance.dev_attr.attr, + &afe440x_attr_in_intensity1_capacitance.dev_attr.attr, + &afe440x_attr_in_intensity2_resistance.dev_attr.attr, + &afe440x_attr_in_intensity2_capacitance.dev_attr.attr, + &afe440x_attr_in_intensity3_resistance.dev_attr.attr, + &afe440x_attr_in_intensity3_capacitance.dev_attr.attr, + &afe440x_attr_in_intensity4_resistance.dev_attr.attr, + &afe440x_attr_in_intensity4_capacitance.dev_attr.attr, NULL }; @@ -309,35 +250,26 @@ static int afe4403_read_raw(struct iio_dev *indio_dev, int *val, int *val2, long mask) { struct afe4403_data *afe = iio_priv(indio_dev); - const struct afe440x_reg_info reg_info = afe4403_reg_info[chan->address]; + unsigned int reg = afe4403_channel_values[chan->address]; + unsigned int field = afe4403_channel_leds[chan->address]; int ret; switch (chan->type) { case IIO_INTENSITY: switch (mask) { case IIO_CHAN_INFO_RAW: - ret = afe4403_read(afe, reg_info.reg, val); - if (ret) - return ret; - return IIO_VAL_INT; - case IIO_CHAN_INFO_OFFSET: - ret = regmap_read(afe->regmap, reg_info.offreg, - val); + ret = afe4403_read(afe, reg, val); if (ret) return ret; - *val &= reg_info.mask; - *val >>= reg_info.shift; return IIO_VAL_INT; } break; case IIO_CURRENT: switch (mask) { case IIO_CHAN_INFO_RAW: - ret = regmap_read(afe->regmap, reg_info.reg, val); + ret = regmap_field_read(afe->fields[field], val); if (ret) return ret; - *val &= reg_info.mask; - *val >>= reg_info.shift; return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: *val = 0; @@ -357,25 +289,13 @@ static int afe4403_write_raw(struct iio_dev *indio_dev, int val, int val2, long mask) { struct afe4403_data *afe = iio_priv(indio_dev); - const struct afe440x_reg_info reg_info = afe4403_reg_info[chan->address]; + unsigned int field = afe4403_channel_leds[chan->address]; switch (chan->type) { - case IIO_INTENSITY: - switch (mask) { - case IIO_CHAN_INFO_OFFSET: - return regmap_update_bits(afe->regmap, - reg_info.offreg, - reg_info.mask, - (val << reg_info.shift)); - } - break; case IIO_CURRENT: switch (mask) { case IIO_CHAN_INFO_RAW: - return regmap_update_bits(afe->regmap, - reg_info.reg, - reg_info.mask, - (val << reg_info.shift)); + return regmap_field_write(afe->fields[field], val); } break; default: @@ -410,7 +330,7 @@ static irqreturn_t afe4403_trigger_handler(int irq, void *private) for_each_set_bit(bit, indio_dev->active_scan_mask, indio_dev->masklength) { ret = spi_write_then_read(afe->spi, - &afe4403_reg_info[bit].reg, 1, + &afe4403_channel_values[bit], 1, rx, 3); if (ret) goto err; @@ -472,12 +392,8 @@ static const struct iio_trigger_ops afe4403_trigger_ops = { static const struct reg_sequence afe4403_reg_sequences[] = { AFE4403_TIMING_PAIRS, - { AFE440X_CONTROL1, AFE440X_CONTROL1_TIMEREN | 0x000007}, - { AFE4403_TIA_AMB_GAIN, AFE4403_TIAGAIN_RES_1_M }, - { AFE440X_LEDCNTRL, (0x14 << AFE440X_LEDCNTRL_LED1_SHIFT) | - (0x14 << AFE440X_LEDCNTRL_LED2_SHIFT) }, - { AFE440X_CONTROL2, AFE440X_CONTROL2_TX_REF_050 << - AFE440X_CONTROL2_TX_REF_SHIFT }, + { AFE440X_CONTROL1, AFE440X_CONTROL1_TIMEREN }, + { AFE4403_TIAGAIN, AFE440X_TIAGAIN_ENSEPGAIN }, }; static const struct regmap_range afe4403_yes_ranges[] = { @@ -498,13 +414,11 @@ static const struct regmap_config afe4403_regmap_config = { .volatile_table = &afe4403_volatile_table, }; -#ifdef CONFIG_OF static const struct of_device_id afe4403_of_match[] = { { .compatible = "ti,afe4403", }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, afe4403_of_match); -#endif static int __maybe_unused afe4403_suspend(struct device *dev) { @@ -553,7 +467,7 @@ static int afe4403_probe(struct spi_device *spi) { struct iio_dev *indio_dev; struct afe4403_data *afe; - int ret; + int i, ret; indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*afe)); if (!indio_dev) @@ -572,6 +486,15 @@ static int afe4403_probe(struct spi_device *spi) return PTR_ERR(afe->regmap); } + for (i = 0; i < F_MAX_FIELDS; i++) { + afe->fields[i] = devm_regmap_field_alloc(afe->dev, afe->regmap, + afe4403_reg_fields[i]); + if (IS_ERR(afe->fields[i])) { + dev_err(afe->dev, "Unable to allocate regmap fields\n"); + return PTR_ERR(afe->fields[i]); + } + } + afe->regulator = devm_regulator_get(afe->dev, "tx_sup"); if (IS_ERR(afe->regulator)) { dev_err(afe->dev, "Unable to get regulator\n"); @@ -694,7 +617,7 @@ MODULE_DEVICE_TABLE(spi, afe4403_ids); static struct spi_driver afe4403_spi_driver = { .driver = { .name = AFE4403_DRIVER_NAME, - .of_match_table = of_match_ptr(afe4403_of_match), + .of_match_table = afe4403_of_match, .pm = &afe4403_pm_ops, }, .probe = afe4403_probe, @@ -704,5 +627,5 @@ static struct spi_driver afe4403_spi_driver = { module_spi_driver(afe4403_spi_driver); MODULE_AUTHOR("Andrew F. Davis "); -MODULE_DESCRIPTION("TI AFE4403 Heart Rate and Pulse Oximeter"); +MODULE_DESCRIPTION("TI AFE4403 Heart Rate Monitor and Pulse Oximeter AFE"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/health/afe4404.c b/drivers/iio/health/afe4404.c index 5096a4643784..45266404f7e3 100644 --- a/drivers/iio/health/afe4404.c +++ b/drivers/iio/health/afe4404.c @@ -1,7 +1,7 @@ /* * AFE4404 Heart Rate Monitors and Low-Cost Pulse Oximeters * - * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com/ + * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/ * Andrew F. Davis * * This program is free software; you can redistribute it and/or modify @@ -48,118 +48,102 @@ #define AFE4404_AVG_LED2_ALED2VAL 0x3f #define AFE4404_AVG_LED1_ALED1VAL 0x40 -/* AFE4404 GAIN register fields */ -#define AFE4404_TIA_GAIN_RES_MASK GENMASK(2, 0) -#define AFE4404_TIA_GAIN_RES_SHIFT 0 -#define AFE4404_TIA_GAIN_CAP_MASK GENMASK(5, 3) -#define AFE4404_TIA_GAIN_CAP_SHIFT 3 +/* AFE4404 CONTROL2 register fields */ +#define AFE440X_CONTROL2_OSC_ENABLE BIT(9) -/* AFE4404 LEDCNTRL register fields */ -#define AFE4404_LEDCNTRL_ILED1_MASK GENMASK(5, 0) -#define AFE4404_LEDCNTRL_ILED1_SHIFT 0 -#define AFE4404_LEDCNTRL_ILED2_MASK GENMASK(11, 6) -#define AFE4404_LEDCNTRL_ILED2_SHIFT 6 -#define AFE4404_LEDCNTRL_ILED3_MASK GENMASK(17, 12) -#define AFE4404_LEDCNTRL_ILED3_SHIFT 12 +enum afe4404_fields { + /* Gains */ + F_TIA_GAIN_SEP, F_TIA_CF_SEP, + F_TIA_GAIN, TIA_CF, -/* AFE4404 CONTROL2 register fields */ -#define AFE440X_CONTROL2_ILED_2X_MASK BIT(17) -#define AFE440X_CONTROL2_ILED_2X_SHIFT 17 - -/* AFE4404 CONTROL3 register fields */ -#define AFE440X_CONTROL3_OSC_ENABLE BIT(9) - -/* AFE4404 OFFDAC register current fields */ -#define AFE4404_OFFDAC_CURR_LED1_MASK GENMASK(9, 5) -#define AFE4404_OFFDAC_CURR_LED1_SHIFT 5 -#define AFE4404_OFFDAC_CURR_LED2_MASK GENMASK(19, 15) -#define AFE4404_OFFDAC_CURR_LED2_SHIFT 15 -#define AFE4404_OFFDAC_CURR_LED3_MASK GENMASK(4, 0) -#define AFE4404_OFFDAC_CURR_LED3_SHIFT 0 -#define AFE4404_OFFDAC_CURR_ALED1_MASK GENMASK(14, 10) -#define AFE4404_OFFDAC_CURR_ALED1_SHIFT 10 -#define AFE4404_OFFDAC_CURR_ALED2_MASK GENMASK(4, 0) -#define AFE4404_OFFDAC_CURR_ALED2_SHIFT 0 - -/* AFE4404 NULL fields */ -#define NULL_MASK 0 -#define NULL_SHIFT 0 - -/* AFE4404 TIA_GAIN_CAP values */ -#define AFE4404_TIA_GAIN_CAP_5_P 0x0 -#define AFE4404_TIA_GAIN_CAP_2_5_P 0x1 -#define AFE4404_TIA_GAIN_CAP_10_P 0x2 -#define AFE4404_TIA_GAIN_CAP_7_5_P 0x3 -#define AFE4404_TIA_GAIN_CAP_20_P 0x4 -#define AFE4404_TIA_GAIN_CAP_17_5_P 0x5 -#define AFE4404_TIA_GAIN_CAP_25_P 0x6 -#define AFE4404_TIA_GAIN_CAP_22_5_P 0x7 - -/* AFE4404 TIA_GAIN_RES values */ -#define AFE4404_TIA_GAIN_RES_500_K 0x0 -#define AFE4404_TIA_GAIN_RES_250_K 0x1 -#define AFE4404_TIA_GAIN_RES_100_K 0x2 -#define AFE4404_TIA_GAIN_RES_50_K 0x3 -#define AFE4404_TIA_GAIN_RES_25_K 0x4 -#define AFE4404_TIA_GAIN_RES_10_K 0x5 -#define AFE4404_TIA_GAIN_RES_1_M 0x6 -#define AFE4404_TIA_GAIN_RES_2_M 0x7 + /* LED Current */ + F_ILED1, F_ILED2, F_ILED3, + + /* Offset DAC */ + F_OFFDAC_AMB2, F_OFFDAC_LED1, F_OFFDAC_AMB1, F_OFFDAC_LED2, + + /* sentinel */ + F_MAX_FIELDS +}; + +static const struct reg_field afe4404_reg_fields[] = { + /* Gains */ + [F_TIA_GAIN_SEP] = REG_FIELD(AFE4404_TIA_GAIN_SEP, 0, 2), + [F_TIA_CF_SEP] = REG_FIELD(AFE4404_TIA_GAIN_SEP, 3, 5), + [F_TIA_GAIN] = REG_FIELD(AFE4404_TIA_GAIN, 0, 2), + [TIA_CF] = REG_FIELD(AFE4404_TIA_GAIN, 3, 5), + /* LED Current */ + [F_ILED1] = REG_FIELD(AFE440X_LEDCNTRL, 0, 5), + [F_ILED2] = REG_FIELD(AFE440X_LEDCNTRL, 6, 11), + [F_ILED3] = REG_FIELD(AFE440X_LEDCNTRL, 12, 17), + /* Offset DAC */ + [F_OFFDAC_AMB2] = REG_FIELD(AFE4404_OFFDAC, 0, 4), + [F_OFFDAC_LED1] = REG_FIELD(AFE4404_OFFDAC, 5, 9), + [F_OFFDAC_AMB1] = REG_FIELD(AFE4404_OFFDAC, 10, 14), + [F_OFFDAC_LED2] = REG_FIELD(AFE4404_OFFDAC, 15, 19), +}; /** - * struct afe4404_data - * @dev - Device structure - * @regmap - Register map of the device - * @regulator - Pointer to the regulator for the IC - * @trig - IIO trigger for this device - * @irq - ADC_RDY line interrupt number + * struct afe4404_data - AFE4404 device instance data + * @dev: Device structure + * @regmap: Register map of the device + * @fields: Register fields of the device + * @regulator: Pointer to the regulator for the IC + * @trig: IIO trigger for this device + * @irq: ADC_RDY line interrupt number */ struct afe4404_data { struct device *dev; struct regmap *regmap; + struct regmap_field *fields[F_MAX_FIELDS]; struct regulator *regulator; struct iio_trigger *trig; int irq; }; enum afe4404_chan_id { + LED2 = 1, + ALED2, LED1, ALED1, - LED2, - ALED2, - LED3, - LED1_ALED1, LED2_ALED2, - ILED1, - ILED2, - ILED3, + LED1_ALED1, +}; + +static const unsigned int afe4404_channel_values[] = { + [LED2] = AFE440X_LED2VAL, + [ALED2] = AFE440X_ALED2VAL, + [LED1] = AFE440X_LED1VAL, + [ALED1] = AFE440X_ALED1VAL, + [LED2_ALED2] = AFE440X_LED2_ALED2VAL, + [LED1_ALED1] = AFE440X_LED1_ALED1VAL, }; -static const struct afe440x_reg_info afe4404_reg_info[] = { - [LED1] = AFE440X_REG_INFO(AFE440X_LED1VAL, AFE4404_OFFDAC, AFE4404_OFFDAC_CURR_LED1), - [ALED1] = AFE440X_REG_INFO(AFE440X_ALED1VAL, AFE4404_OFFDAC, AFE4404_OFFDAC_CURR_ALED1), - [LED2] = AFE440X_REG_INFO(AFE440X_LED2VAL, AFE4404_OFFDAC, AFE4404_OFFDAC_CURR_LED2), - [ALED2] = AFE440X_REG_INFO(AFE440X_ALED2VAL, AFE4404_OFFDAC, AFE4404_OFFDAC_CURR_ALED2), - [LED3] = AFE440X_REG_INFO(AFE440X_ALED2VAL, 0, NULL), - [LED1_ALED1] = AFE440X_REG_INFO(AFE440X_LED1_ALED1VAL, 0, NULL), - [LED2_ALED2] = AFE440X_REG_INFO(AFE440X_LED2_ALED2VAL, 0, NULL), - [ILED1] = AFE440X_REG_INFO(AFE440X_LEDCNTRL, 0, AFE4404_LEDCNTRL_ILED1), - [ILED2] = AFE440X_REG_INFO(AFE440X_LEDCNTRL, 0, AFE4404_LEDCNTRL_ILED2), - [ILED3] = AFE440X_REG_INFO(AFE440X_LEDCNTRL, 0, AFE4404_LEDCNTRL_ILED3), +static const unsigned int afe4404_channel_leds[] = { + [LED2] = F_ILED2, + [ALED2] = F_ILED3, + [LED1] = F_ILED1, +}; + +static const unsigned int afe4404_channel_offdacs[] = { + [LED2] = F_OFFDAC_LED2, + [ALED2] = F_OFFDAC_AMB2, + [LED1] = F_OFFDAC_LED1, + [ALED1] = F_OFFDAC_AMB1, }; static const struct iio_chan_spec afe4404_channels[] = { /* ADC values */ - AFE440X_INTENSITY_CHAN(LED1, "led1", BIT(IIO_CHAN_INFO_OFFSET)), - AFE440X_INTENSITY_CHAN(ALED1, "led1_ambient", BIT(IIO_CHAN_INFO_OFFSET)), - AFE440X_INTENSITY_CHAN(LED2, "led2", BIT(IIO_CHAN_INFO_OFFSET)), - AFE440X_INTENSITY_CHAN(ALED2, "led2_ambient", BIT(IIO_CHAN_INFO_OFFSET)), - AFE440X_INTENSITY_CHAN(LED3, "led3", BIT(IIO_CHAN_INFO_OFFSET)), - AFE440X_INTENSITY_CHAN(LED1_ALED1, "led1-led1_ambient", 0), - AFE440X_INTENSITY_CHAN(LED2_ALED2, "led2-led2_ambient", 0), + AFE440X_INTENSITY_CHAN(LED2, BIT(IIO_CHAN_INFO_OFFSET)), + AFE440X_INTENSITY_CHAN(ALED2, BIT(IIO_CHAN_INFO_OFFSET)), + AFE440X_INTENSITY_CHAN(LED1, BIT(IIO_CHAN_INFO_OFFSET)), + AFE440X_INTENSITY_CHAN(ALED1, BIT(IIO_CHAN_INFO_OFFSET)), + AFE440X_INTENSITY_CHAN(LED2_ALED2, 0), + AFE440X_INTENSITY_CHAN(LED1_ALED1, 0), /* LED current */ - AFE440X_CURRENT_CHAN(ILED1, "led1"), - AFE440X_CURRENT_CHAN(ILED2, "led2"), - AFE440X_CURRENT_CHAN(ILED3, "led3"), + AFE440X_CURRENT_CHAN(LED2), + AFE440X_CURRENT_CHAN(ALED2), + AFE440X_CURRENT_CHAN(LED1), }; static const struct afe440x_val_table afe4404_res_table[] = { @@ -172,7 +156,7 @@ static const struct afe440x_val_table afe4404_res_table[] = { { .integer = 1000000, .fract = 0 }, { .integer = 2000000, .fract = 0 }, }; -AFE440X_TABLE_ATTR(tia_resistance_available, afe4404_res_table); +AFE440X_TABLE_ATTR(in_intensity_resistance_available, afe4404_res_table); static const struct afe440x_val_table afe4404_cap_table[] = { { .integer = 0, .fract = 5000 }, @@ -184,7 +168,7 @@ static const struct afe440x_val_table afe4404_cap_table[] = { { .integer = 0, .fract = 25000 }, { .integer = 0, .fract = 22500 }, }; -AFE440X_TABLE_ATTR(tia_capacitance_available, afe4404_cap_table); +AFE440X_TABLE_ATTR(in_intensity_capacitance_available, afe4404_cap_table); static ssize_t afe440x_show_register(struct device *dev, struct device_attribute *attr, @@ -193,38 +177,21 @@ static ssize_t afe440x_show_register(struct device *dev, struct iio_dev *indio_dev = dev_to_iio_dev(dev); struct afe4404_data *afe = iio_priv(indio_dev); struct afe440x_attr *afe440x_attr = to_afe440x_attr(attr); - unsigned int reg_val, type; + unsigned int reg_val; int vals[2]; - int ret, val_len; + int ret; - ret = regmap_read(afe->regmap, afe440x_attr->reg, ®_val); + ret = regmap_field_read(afe->fields[afe440x_attr->field], ®_val); if (ret) return ret; - reg_val &= afe440x_attr->mask; - reg_val >>= afe440x_attr->shift; - - switch (afe440x_attr->type) { - case SIMPLE: - type = IIO_VAL_INT; - val_len = 1; - vals[0] = reg_val; - break; - case RESISTANCE: - case CAPACITANCE: - type = IIO_VAL_INT_PLUS_MICRO; - val_len = 2; - if (reg_val < afe440x_attr->table_size) { - vals[0] = afe440x_attr->val_table[reg_val].integer; - vals[1] = afe440x_attr->val_table[reg_val].fract; - break; - } - return -EINVAL; - default: + if (reg_val >= afe440x_attr->table_size) return -EINVAL; - } - return iio_format_value(buf, type, val_len, vals); + vals[0] = afe440x_attr->val_table[reg_val].integer; + vals[1] = afe440x_attr->val_table[reg_val].fract; + + return iio_format_value(buf, IIO_VAL_INT_PLUS_MICRO, 2, vals); } static ssize_t afe440x_store_register(struct device *dev, @@ -240,48 +207,43 @@ static ssize_t afe440x_store_register(struct device *dev, if (ret) return ret; - switch (afe440x_attr->type) { - case SIMPLE: - val = integer; - break; - case RESISTANCE: - case CAPACITANCE: - for (val = 0; val < afe440x_attr->table_size; val++) - if (afe440x_attr->val_table[val].integer == integer && - afe440x_attr->val_table[val].fract == fract) - break; - if (val == afe440x_attr->table_size) - return -EINVAL; - break; - default: + for (val = 0; val < afe440x_attr->table_size; val++) + if (afe440x_attr->val_table[val].integer == integer && + afe440x_attr->val_table[val].fract == fract) + break; + if (val == afe440x_attr->table_size) return -EINVAL; - } - ret = regmap_update_bits(afe->regmap, afe440x_attr->reg, - afe440x_attr->mask, - (val << afe440x_attr->shift)); + ret = regmap_field_write(afe->fields[afe440x_attr->field], val); if (ret) return ret; return count; } -static AFE440X_ATTR(tia_separate_en, AFE4404_TIA_GAIN_SEP, AFE440X_TIAGAIN_ENSEPGAIN, SIMPLE, NULL, 0); +static AFE440X_ATTR(in_intensity1_resistance, F_TIA_GAIN_SEP, afe4404_res_table); +static AFE440X_ATTR(in_intensity1_capacitance, F_TIA_CF_SEP, afe4404_cap_table); + +static AFE440X_ATTR(in_intensity2_resistance, F_TIA_GAIN_SEP, afe4404_res_table); +static AFE440X_ATTR(in_intensity2_capacitance, F_TIA_CF_SEP, afe4404_cap_table); -static AFE440X_ATTR(tia_resistance1, AFE4404_TIA_GAIN, AFE4404_TIA_GAIN_RES, RESISTANCE, afe4404_res_table, ARRAY_SIZE(afe4404_res_table)); -static AFE440X_ATTR(tia_capacitance1, AFE4404_TIA_GAIN, AFE4404_TIA_GAIN_CAP, CAPACITANCE, afe4404_cap_table, ARRAY_SIZE(afe4404_cap_table)); +static AFE440X_ATTR(in_intensity3_resistance, F_TIA_GAIN, afe4404_res_table); +static AFE440X_ATTR(in_intensity3_capacitance, TIA_CF, afe4404_cap_table); -static AFE440X_ATTR(tia_resistance2, AFE4404_TIA_GAIN_SEP, AFE4404_TIA_GAIN_RES, RESISTANCE, afe4404_res_table, ARRAY_SIZE(afe4404_res_table)); -static AFE440X_ATTR(tia_capacitance2, AFE4404_TIA_GAIN_SEP, AFE4404_TIA_GAIN_CAP, CAPACITANCE, afe4404_cap_table, ARRAY_SIZE(afe4404_cap_table)); +static AFE440X_ATTR(in_intensity4_resistance, F_TIA_GAIN, afe4404_res_table); +static AFE440X_ATTR(in_intensity4_capacitance, TIA_CF, afe4404_cap_table); static struct attribute *afe440x_attributes[] = { - &afe440x_attr_tia_separate_en.dev_attr.attr, - &afe440x_attr_tia_resistance1.dev_attr.attr, - &afe440x_attr_tia_capacitance1.dev_attr.attr, - &afe440x_attr_tia_resistance2.dev_attr.attr, - &afe440x_attr_tia_capacitance2.dev_attr.attr, - &dev_attr_tia_resistance_available.attr, - &dev_attr_tia_capacitance_available.attr, + &dev_attr_in_intensity_resistance_available.attr, + &dev_attr_in_intensity_capacitance_available.attr, + &afe440x_attr_in_intensity1_resistance.dev_attr.attr, + &afe440x_attr_in_intensity1_capacitance.dev_attr.attr, + &afe440x_attr_in_intensity2_resistance.dev_attr.attr, + &afe440x_attr_in_intensity2_capacitance.dev_attr.attr, + &afe440x_attr_in_intensity3_resistance.dev_attr.attr, + &afe440x_attr_in_intensity3_capacitance.dev_attr.attr, + &afe440x_attr_in_intensity4_resistance.dev_attr.attr, + &afe440x_attr_in_intensity4_capacitance.dev_attr.attr, NULL }; @@ -294,35 +256,32 @@ static int afe4404_read_raw(struct iio_dev *indio_dev, int *val, int *val2, long mask) { struct afe4404_data *afe = iio_priv(indio_dev); - const struct afe440x_reg_info reg_info = afe4404_reg_info[chan->address]; + unsigned int value_reg = afe4404_channel_values[chan->address]; + unsigned int led_field = afe4404_channel_leds[chan->address]; + unsigned int offdac_field = afe4404_channel_offdacs[chan->address]; int ret; switch (chan->type) { case IIO_INTENSITY: switch (mask) { case IIO_CHAN_INFO_RAW: - ret = regmap_read(afe->regmap, reg_info.reg, val); + ret = regmap_read(afe->regmap, value_reg, val); if (ret) return ret; return IIO_VAL_INT; case IIO_CHAN_INFO_OFFSET: - ret = regmap_read(afe->regmap, reg_info.offreg, - val); + ret = regmap_field_read(afe->fields[offdac_field], val); if (ret) return ret; - *val &= reg_info.mask; - *val >>= reg_info.shift; return IIO_VAL_INT; } break; case IIO_CURRENT: switch (mask) { case IIO_CHAN_INFO_RAW: - ret = regmap_read(afe->regmap, reg_info.reg, val); + ret = regmap_field_read(afe->fields[led_field], val); if (ret) return ret; - *val &= reg_info.mask; - *val >>= reg_info.shift; return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: *val = 0; @@ -342,25 +301,20 @@ static int afe4404_write_raw(struct iio_dev *indio_dev, int val, int val2, long mask) { struct afe4404_data *afe = iio_priv(indio_dev); - const struct afe440x_reg_info reg_info = afe4404_reg_info[chan->address]; + unsigned int led_field = afe4404_channel_leds[chan->address]; + unsigned int offdac_field = afe4404_channel_offdacs[chan->address]; switch (chan->type) { case IIO_INTENSITY: switch (mask) { case IIO_CHAN_INFO_OFFSET: - return regmap_update_bits(afe->regmap, - reg_info.offreg, - reg_info.mask, - (val << reg_info.shift)); + return regmap_field_write(afe->fields[offdac_field], val); } break; case IIO_CURRENT: switch (mask) { case IIO_CHAN_INFO_RAW: - return regmap_update_bits(afe->regmap, - reg_info.reg, - reg_info.mask, - (val << reg_info.shift)); + return regmap_field_write(afe->fields[led_field], val); } break; default: @@ -387,7 +341,7 @@ static irqreturn_t afe4404_trigger_handler(int irq, void *private) for_each_set_bit(bit, indio_dev->active_scan_mask, indio_dev->masklength) { - ret = regmap_read(afe->regmap, afe4404_reg_info[bit].reg, + ret = regmap_read(afe->regmap, afe4404_channel_values[bit], &buffer[i++]); if (ret) goto err; @@ -443,11 +397,8 @@ static const struct iio_trigger_ops afe4404_trigger_ops = { static const struct reg_sequence afe4404_reg_sequences[] = { AFE4404_TIMING_PAIRS, { AFE440X_CONTROL1, AFE440X_CONTROL1_TIMEREN }, - { AFE4404_TIA_GAIN, AFE4404_TIA_GAIN_RES_50_K }, - { AFE440X_LEDCNTRL, (0xf << AFE4404_LEDCNTRL_ILED1_SHIFT) | - (0x3 << AFE4404_LEDCNTRL_ILED2_SHIFT) | - (0x3 << AFE4404_LEDCNTRL_ILED3_SHIFT) }, - { AFE440X_CONTROL2, AFE440X_CONTROL3_OSC_ENABLE }, + { AFE4404_TIA_GAIN_SEP, AFE440X_TIAGAIN_ENSEPGAIN }, + { AFE440X_CONTROL2, AFE440X_CONTROL2_OSC_ENABLE }, }; static const struct regmap_range afe4404_yes_ranges[] = { @@ -469,13 +420,11 @@ static const struct regmap_config afe4404_regmap_config = { .volatile_table = &afe4404_volatile_table, }; -#ifdef CONFIG_OF static const struct of_device_id afe4404_of_match[] = { { .compatible = "ti,afe4404", }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, afe4404_of_match); -#endif static int __maybe_unused afe4404_suspend(struct device *dev) { @@ -525,7 +474,7 @@ static int afe4404_probe(struct i2c_client *client, { struct iio_dev *indio_dev; struct afe4404_data *afe; - int ret; + int i, ret; indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*afe)); if (!indio_dev) @@ -543,6 +492,15 @@ static int afe4404_probe(struct i2c_client *client, return PTR_ERR(afe->regmap); } + for (i = 0; i < F_MAX_FIELDS; i++) { + afe->fields[i] = devm_regmap_field_alloc(afe->dev, afe->regmap, + afe4404_reg_fields[i]); + if (IS_ERR(afe->fields[i])) { + dev_err(afe->dev, "Unable to allocate regmap fields\n"); + return PTR_ERR(afe->fields[i]); + } + } + afe->regulator = devm_regulator_get(afe->dev, "tx_sup"); if (IS_ERR(afe->regulator)) { dev_err(afe->dev, "Unable to get regulator\n"); @@ -665,7 +623,7 @@ MODULE_DEVICE_TABLE(i2c, afe4404_ids); static struct i2c_driver afe4404_i2c_driver = { .driver = { .name = AFE4404_DRIVER_NAME, - .of_match_table = of_match_ptr(afe4404_of_match), + .of_match_table = afe4404_of_match, .pm = &afe4404_pm_ops, }, .probe = afe4404_probe, @@ -675,5 +633,5 @@ static struct i2c_driver afe4404_i2c_driver = { module_i2c_driver(afe4404_i2c_driver); MODULE_AUTHOR("Andrew F. Davis "); -MODULE_DESCRIPTION("TI AFE4404 Heart Rate and Pulse Oximeter"); +MODULE_DESCRIPTION("TI AFE4404 Heart Rate Monitor and Pulse Oximeter AFE"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/health/afe440x.h b/drivers/iio/health/afe440x.h index c671ab78a23a..1a0f247043ca 100644 --- a/drivers/iio/health/afe440x.h +++ b/drivers/iio/health/afe440x.h @@ -71,8 +71,7 @@ #define AFE440X_CONTROL1_TIMEREN BIT(8) /* TIAGAIN register fields */ -#define AFE440X_TIAGAIN_ENSEPGAIN_MASK BIT(15) -#define AFE440X_TIAGAIN_ENSEPGAIN_SHIFT 15 +#define AFE440X_TIAGAIN_ENSEPGAIN BIT(15) /* CONTROL2 register fields */ #define AFE440X_CONTROL2_PDN_AFE BIT(0) @@ -89,22 +88,7 @@ #define AFE440X_CONTROL0_WRITE 0x0 #define AFE440X_CONTROL0_READ 0x1 -struct afe440x_reg_info { - unsigned int reg; - unsigned int offreg; - unsigned int shift; - unsigned int mask; -}; - -#define AFE440X_REG_INFO(_reg, _offreg, _sm) \ - { \ - .reg = _reg, \ - .offreg = _offreg, \ - .shift = _sm ## _SHIFT, \ - .mask = _sm ## _MASK, \ - } - -#define AFE440X_INTENSITY_CHAN(_index, _name, _mask) \ +#define AFE440X_INTENSITY_CHAN(_index, _mask) \ { \ .type = IIO_INTENSITY, \ .channel = _index, \ @@ -116,29 +100,23 @@ struct afe440x_reg_info { .storagebits = 32, \ .endianness = IIO_CPU, \ }, \ - .extend_name = _name, \ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \ _mask, \ + .indexed = true, \ } -#define AFE440X_CURRENT_CHAN(_index, _name) \ +#define AFE440X_CURRENT_CHAN(_index) \ { \ .type = IIO_CURRENT, \ .channel = _index, \ .address = _index, \ - .scan_index = _index, \ - .extend_name = _name, \ + .scan_index = -1, \ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \ BIT(IIO_CHAN_INFO_SCALE), \ + .indexed = true, \ .output = true, \ } -enum afe440x_reg_type { - SIMPLE, - RESISTANCE, - CAPACITANCE, -}; - struct afe440x_val_table { int integer; int fract; @@ -164,10 +142,7 @@ static DEVICE_ATTR_RO(_name) struct afe440x_attr { struct device_attribute dev_attr; - unsigned int reg; - unsigned int shift; - unsigned int mask; - enum afe440x_reg_type type; + unsigned int field; const struct afe440x_val_table *val_table; unsigned int table_size; }; @@ -175,17 +150,14 @@ struct afe440x_attr { #define to_afe440x_attr(_dev_attr) \ container_of(_dev_attr, struct afe440x_attr, dev_attr) -#define AFE440X_ATTR(_name, _reg, _field, _type, _table, _size) \ +#define AFE440X_ATTR(_name, _field, _table) \ struct afe440x_attr afe440x_attr_##_name = { \ .dev_attr = __ATTR(_name, (S_IRUGO | S_IWUSR), \ afe440x_show_register, \ afe440x_store_register), \ - .reg = _reg, \ - .shift = _field ## _SHIFT, \ - .mask = _field ## _MASK, \ - .type = _type, \ + .field = _field, \ .val_table = _table, \ - .table_size = _size, \ + .table_size = ARRAY_SIZE(_table), \ } #endif /* _AFE440X_H */ diff --git a/drivers/iio/humidity/am2315.c b/drivers/iio/humidity/am2315.c index 11535911a5c6..3e200f69e886 100644 --- a/drivers/iio/humidity/am2315.c +++ b/drivers/iio/humidity/am2315.c @@ -276,6 +276,7 @@ static const struct i2c_device_id am2315_i2c_id[] = { {"am2315", 0}, {} }; +MODULE_DEVICE_TABLE(i2c, am2315_i2c_id); static const struct acpi_device_id am2315_acpi_id[] = { {"AOS2315", 0}, diff --git a/drivers/iio/humidity/htu21.c b/drivers/iio/humidity/htu21.c index 11cbc38b450f..0fbbd8c40894 100644 --- a/drivers/iio/humidity/htu21.c +++ b/drivers/iio/humidity/htu21.c @@ -236,6 +236,7 @@ static const struct i2c_device_id htu21_id[] = { {"ms8607-humidity", MS8607}, {} }; +MODULE_DEVICE_TABLE(i2c, htu21_id); static struct i2c_driver htu21_driver = { .probe = htu21_probe, diff --git a/drivers/iio/iio_core.h b/drivers/iio/iio_core.h index 359883525ab7..4c45488e3a7f 100644 --- a/drivers/iio/iio_core.h +++ b/drivers/iio/iio_core.h @@ -79,4 +79,7 @@ void iio_device_unregister_eventset(struct iio_dev *indio_dev); void iio_device_wakeup_eventset(struct iio_dev *indio_dev); int iio_event_getfd(struct iio_dev *indio_dev); +struct iio_event_interface; +bool iio_event_enabled(const struct iio_event_interface *ev_int); + #endif diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c index b8a290ec984e..e0251b8c1a52 100644 --- a/drivers/iio/imu/bmi160/bmi160_core.c +++ b/drivers/iio/imu/bmi160/bmi160_core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "bmi160.h" @@ -410,7 +411,8 @@ static irqreturn_t bmi160_trigger_handler(int irq, void *p) buf[j++] = sample; } - iio_push_to_buffers_with_timestamp(indio_dev, buf, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, buf, + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); return IRQ_HANDLED; @@ -466,10 +468,36 @@ static int bmi160_write_raw(struct iio_dev *indio_dev, return 0; } +static +IIO_CONST_ATTR(in_accel_sampling_frequency_available, + "0.78125 1.5625 3.125 6.25 12.5 25 50 100 200 400 800 1600"); +static +IIO_CONST_ATTR(in_anglvel_sampling_frequency_available, + "25 50 100 200 400 800 1600 3200"); +static +IIO_CONST_ATTR(in_accel_scale_available, + "0.000598 0.001197 0.002394 0.004788"); +static +IIO_CONST_ATTR(in_anglvel_scale_available, + "0.001065 0.000532 0.000266 0.000133 0.000066"); + +static struct attribute *bmi160_attrs[] = { + &iio_const_attr_in_accel_sampling_frequency_available.dev_attr.attr, + &iio_const_attr_in_anglvel_sampling_frequency_available.dev_attr.attr, + &iio_const_attr_in_accel_scale_available.dev_attr.attr, + &iio_const_attr_in_anglvel_scale_available.dev_attr.attr, + NULL, +}; + +static const struct attribute_group bmi160_attrs_group = { + .attrs = bmi160_attrs, +}; + static const struct iio_info bmi160_info = { .driver_module = THIS_MODULE, .read_raw = bmi160_read_raw, .write_raw = bmi160_write_raw, + .attrs = &bmi160_attrs_group, }; static const char *bmi160_match_acpi_device(struct device *dev) diff --git a/drivers/iio/imu/inv_mpu6050/Kconfig b/drivers/iio/imu/inv_mpu6050/Kconfig index f756feecfa4c..5483b2ea754d 100644 --- a/drivers/iio/imu/inv_mpu6050/Kconfig +++ b/drivers/iio/imu/inv_mpu6050/Kconfig @@ -13,8 +13,8 @@ config INV_MPU6050_I2C select INV_MPU6050_IIO select REGMAP_I2C help - This driver supports the Invensense MPU6050/6500/9150 motion tracking - devices over I2C. + This driver supports the Invensense MPU6050/6500/9150 and ICM20608 + motion tracking devices over I2C. This driver can be built as a module. The module will be called inv-mpu6050-i2c. @@ -24,7 +24,7 @@ config INV_MPU6050_SPI select INV_MPU6050_IIO select REGMAP_SPI help - This driver supports the Invensense MPU6000/6500/9150 motion tracking - devices over SPI. + This driver supports the Invensense MPU6050/6500/9150 and ICM20608 + motion tracking devices over SPI. This driver can be built as a module. The module will be called inv-mpu6050-spi. diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c index f62b8bd9ad7e..dd6fc6d21f9d 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c @@ -56,6 +56,7 @@ static int asus_acpi_get_sensor_info(struct acpi_device *adev, int i; acpi_status status; union acpi_object *cpm; + int ret; status = acpi_evaluate_object(adev->handle, "CNF0", NULL, &buffer); if (ACPI_FAILURE(status)) @@ -82,10 +83,10 @@ static int asus_acpi_get_sensor_info(struct acpi_device *adev, } } } - + ret = cpm->package.count; kfree(buffer.pointer); - return cpm->package.count; + return ret; } static int acpi_i2c_check_resource(struct acpi_resource *ares, void *data) diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c index ee40dae5ab58..b9fcbf18aa99 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c @@ -113,6 +113,12 @@ static const struct inv_mpu6050_hw hw_info[] = { .reg = ®_set_6050, .config = &chip_config_6050, }, + { + .whoami = INV_ICM20608_WHOAMI_VALUE, + .name = "ICM20608", + .reg = ®_set_6500, + .config = &chip_config_6050, + }, }; int inv_mpu6050_switch_engine(struct inv_mpu6050_state *st, bool en, u32 mask) diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c index e1fd7fa53e3b..19580d1db597 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c @@ -170,6 +170,7 @@ static const struct i2c_device_id inv_mpu_id[] = { {"mpu6050", INV_MPU6050}, {"mpu6500", INV_MPU6500}, {"mpu9150", INV_MPU9150}, + {"icm20608", INV_ICM20608}, {} }; diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h b/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h index 3bf8544ccc9f..f0e8c5dd9fae 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h @@ -70,6 +70,7 @@ enum inv_devices { INV_MPU6500, INV_MPU6000, INV_MPU9150, + INV_ICM20608, INV_NUM_PARTS }; @@ -225,6 +226,7 @@ struct inv_mpu6050_state { #define INV_MPU6050_WHOAMI_VALUE 0x68 #define INV_MPU6500_WHOAMI_VALUE 0x70 #define INV_MPU9150_WHOAMI_VALUE 0x68 +#define INV_ICM20608_WHOAMI_VALUE 0xAF /* scan element definition */ enum inv_mpu6050_scan { diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c index d0700628ee6d..3a9f3eac91ab 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c @@ -107,7 +107,7 @@ irqreturn_t inv_mpu6050_irq_handler(int irq, void *p) struct inv_mpu6050_state *st = iio_priv(indio_dev); s64 timestamp; - timestamp = iio_get_time_ns(); + timestamp = iio_get_time_ns(indio_dev); kfifo_in_spinlocked(&st->timestamps, ×tamp, 1, &st->time_stamp_lock); diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c index 190a4a51c830..6e6476dfa188 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c @@ -82,6 +82,7 @@ static const struct spi_device_id inv_mpu_id[] = { {"mpu6000", INV_MPU6000}, {"mpu6500", INV_MPU6500}, {"mpu9150", INV_MPU9150}, + {"icm20608", INV_ICM20608}, {} }; diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index e6319a9346b2..f914d5d140e4 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -80,6 +80,7 @@ static const char * const iio_chan_type_name_spec[] = { [IIO_RESISTANCE] = "resistance", [IIO_PH] = "ph", [IIO_UVINDEX] = "uvindex", + [IIO_ELECTRICALCONDUCTIVITY] = "electricalconductivity", }; static const char * const iio_modifier_names[] = { @@ -177,6 +178,86 @@ ssize_t iio_read_const_attr(struct device *dev, } EXPORT_SYMBOL(iio_read_const_attr); +static int iio_device_set_clock(struct iio_dev *indio_dev, clockid_t clock_id) +{ + int ret; + const struct iio_event_interface *ev_int = indio_dev->event_interface; + + ret = mutex_lock_interruptible(&indio_dev->mlock); + if (ret) + return ret; + if ((ev_int && iio_event_enabled(ev_int)) || + iio_buffer_enabled(indio_dev)) { + mutex_unlock(&indio_dev->mlock); + return -EBUSY; + } + indio_dev->clock_id = clock_id; + mutex_unlock(&indio_dev->mlock); + + return 0; +} + +/** + * iio_get_time_ns() - utility function to get a time stamp for events etc + * @indio_dev: device + */ +s64 iio_get_time_ns(const struct iio_dev *indio_dev) +{ + struct timespec tp; + + switch (iio_device_get_clock(indio_dev)) { + case CLOCK_REALTIME: + ktime_get_real_ts(&tp); + break; + case CLOCK_MONOTONIC: + ktime_get_ts(&tp); + break; + case CLOCK_MONOTONIC_RAW: + getrawmonotonic(&tp); + break; + case CLOCK_REALTIME_COARSE: + tp = current_kernel_time(); + break; + case CLOCK_MONOTONIC_COARSE: + tp = get_monotonic_coarse(); + break; + case CLOCK_BOOTTIME: + get_monotonic_boottime(&tp); + break; + case CLOCK_TAI: + timekeeping_clocktai(&tp); + break; + default: + BUG(); + } + + return timespec_to_ns(&tp); +} +EXPORT_SYMBOL(iio_get_time_ns); + +/** + * iio_get_time_res() - utility function to get time stamp clock resolution in + * nano seconds. + * @indio_dev: device + */ +unsigned int iio_get_time_res(const struct iio_dev *indio_dev) +{ + switch (iio_device_get_clock(indio_dev)) { + case CLOCK_REALTIME: + case CLOCK_MONOTONIC: + case CLOCK_MONOTONIC_RAW: + case CLOCK_BOOTTIME: + case CLOCK_TAI: + return hrtimer_resolution; + case CLOCK_REALTIME_COARSE: + case CLOCK_MONOTONIC_COARSE: + return LOW_RES_NSEC; + default: + BUG(); + } +} +EXPORT_SYMBOL(iio_get_time_res); + static int __init iio_init(void) { int ret; @@ -989,11 +1070,91 @@ static ssize_t iio_show_dev_name(struct device *dev, static DEVICE_ATTR(name, S_IRUGO, iio_show_dev_name, NULL); +static ssize_t iio_show_timestamp_clock(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + const struct iio_dev *indio_dev = dev_to_iio_dev(dev); + const clockid_t clk = iio_device_get_clock(indio_dev); + const char *name; + ssize_t sz; + + switch (clk) { + case CLOCK_REALTIME: + name = "realtime\n"; + sz = sizeof("realtime\n"); + break; + case CLOCK_MONOTONIC: + name = "monotonic\n"; + sz = sizeof("monotonic\n"); + break; + case CLOCK_MONOTONIC_RAW: + name = "monotonic_raw\n"; + sz = sizeof("monotonic_raw\n"); + break; + case CLOCK_REALTIME_COARSE: + name = "realtime_coarse\n"; + sz = sizeof("realtime_coarse\n"); + break; + case CLOCK_MONOTONIC_COARSE: + name = "monotonic_coarse\n"; + sz = sizeof("monotonic_coarse\n"); + break; + case CLOCK_BOOTTIME: + name = "boottime\n"; + sz = sizeof("boottime\n"); + break; + case CLOCK_TAI: + name = "tai\n"; + sz = sizeof("tai\n"); + break; + default: + BUG(); + } + + memcpy(buf, name, sz); + return sz; +} + +static ssize_t iio_store_timestamp_clock(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + clockid_t clk; + int ret; + + if (sysfs_streq(buf, "realtime")) + clk = CLOCK_REALTIME; + else if (sysfs_streq(buf, "monotonic")) + clk = CLOCK_MONOTONIC; + else if (sysfs_streq(buf, "monotonic_raw")) + clk = CLOCK_MONOTONIC_RAW; + else if (sysfs_streq(buf, "realtime_coarse")) + clk = CLOCK_REALTIME_COARSE; + else if (sysfs_streq(buf, "monotonic_coarse")) + clk = CLOCK_MONOTONIC_COARSE; + else if (sysfs_streq(buf, "boottime")) + clk = CLOCK_BOOTTIME; + else if (sysfs_streq(buf, "tai")) + clk = CLOCK_TAI; + else + return -EINVAL; + + ret = iio_device_set_clock(dev_to_iio_dev(dev), clk); + if (ret) + return ret; + + return len; +} + +static DEVICE_ATTR(current_timestamp_clock, S_IRUGO | S_IWUSR, + iio_show_timestamp_clock, iio_store_timestamp_clock); + static int iio_device_register_sysfs(struct iio_dev *indio_dev) { int i, ret = 0, attrcount, attrn, attrcount_orig = 0; struct iio_dev_attr *p; - struct attribute **attr; + struct attribute **attr, *clk = NULL; /* First count elements in any existing group */ if (indio_dev->info->attrs) { @@ -1008,16 +1169,25 @@ static int iio_device_register_sysfs(struct iio_dev *indio_dev) */ if (indio_dev->channels) for (i = 0; i < indio_dev->num_channels; i++) { - ret = iio_device_add_channel_sysfs(indio_dev, - &indio_dev - ->channels[i]); + const struct iio_chan_spec *chan = + &indio_dev->channels[i]; + + if (chan->type == IIO_TIMESTAMP) + clk = &dev_attr_current_timestamp_clock.attr; + + ret = iio_device_add_channel_sysfs(indio_dev, chan); if (ret < 0) goto error_clear_attrs; attrcount += ret; } + if (indio_dev->event_interface) + clk = &dev_attr_current_timestamp_clock.attr; + if (indio_dev->name) attrcount++; + if (clk) + attrcount++; indio_dev->chan_attr_group.attrs = kcalloc(attrcount + 1, sizeof(indio_dev->chan_attr_group.attrs[0]), @@ -1038,6 +1208,8 @@ static int iio_device_register_sysfs(struct iio_dev *indio_dev) indio_dev->chan_attr_group.attrs[attrn++] = &p->dev_attr.attr; if (indio_dev->name) indio_dev->chan_attr_group.attrs[attrn++] = &dev_attr_name.attr; + if (clk) + indio_dev->chan_attr_group.attrs[attrn++] = clk; indio_dev->groups[indio_dev->groupcounter++] = &indio_dev->chan_attr_group; diff --git a/drivers/iio/industrialio-event.c b/drivers/iio/industrialio-event.c index cae332b1d7ea..0ebfc923a997 100644 --- a/drivers/iio/industrialio-event.c +++ b/drivers/iio/industrialio-event.c @@ -44,6 +44,11 @@ struct iio_event_interface { struct mutex read_lock; }; +bool iio_event_enabled(const struct iio_event_interface *ev_int) +{ + return !!test_bit(IIO_BUSY_BIT_POS, &ev_int->flags); +} + /** * iio_push_event() - try to add event to the list for userspace reading * @indio_dev: IIO device structure @@ -60,7 +65,7 @@ int iio_push_event(struct iio_dev *indio_dev, u64 ev_code, s64 timestamp) int copied; /* Does anyone care? */ - if (test_bit(IIO_BUSY_BIT_POS, &ev_int->flags)) { + if (iio_event_enabled(ev_int)) { ev.id = ev_code; ev.timestamp = timestamp; @@ -180,8 +185,14 @@ int iio_event_getfd(struct iio_dev *indio_dev) if (ev_int == NULL) return -ENODEV; - if (test_and_set_bit(IIO_BUSY_BIT_POS, &ev_int->flags)) - return -EBUSY; + fd = mutex_lock_interruptible(&indio_dev->mlock); + if (fd) + return fd; + + if (test_and_set_bit(IIO_BUSY_BIT_POS, &ev_int->flags)) { + fd = -EBUSY; + goto unlock; + } iio_device_get(indio_dev); @@ -194,6 +205,8 @@ int iio_event_getfd(struct iio_dev *indio_dev) kfifo_reset_out(&ev_int->det_events); } +unlock: + mutex_unlock(&indio_dev->mlock); return fd; } diff --git a/drivers/iio/industrialio-sw-device.c b/drivers/iio/industrialio-sw-device.c new file mode 100644 index 000000000000..81b49cfca452 --- /dev/null +++ b/drivers/iio/industrialio-sw-device.c @@ -0,0 +1,182 @@ +/* + * The Industrial I/O core, software IIO devices functions + * + * Copyright (c) 2016 Intel Corporation + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +static struct config_group *iio_devices_group; +static struct config_item_type iio_device_type_group_type; + +static struct config_item_type iio_devices_group_type = { + .ct_owner = THIS_MODULE, +}; + +static LIST_HEAD(iio_device_types_list); +static DEFINE_MUTEX(iio_device_types_lock); + +static +struct iio_sw_device_type *__iio_find_sw_device_type(const char *name, + unsigned len) +{ + struct iio_sw_device_type *d = NULL, *iter; + + list_for_each_entry(iter, &iio_device_types_list, list) + if (!strcmp(iter->name, name)) { + d = iter; + break; + } + + return d; +} + +int iio_register_sw_device_type(struct iio_sw_device_type *d) +{ + struct iio_sw_device_type *iter; + int ret = 0; + + mutex_lock(&iio_device_types_lock); + iter = __iio_find_sw_device_type(d->name, strlen(d->name)); + if (iter) + ret = -EBUSY; + else + list_add_tail(&d->list, &iio_device_types_list); + mutex_unlock(&iio_device_types_lock); + + if (ret) + return ret; + + d->group = configfs_register_default_group(iio_devices_group, d->name, + &iio_device_type_group_type); + if (IS_ERR(d->group)) + ret = PTR_ERR(d->group); + + return ret; +} +EXPORT_SYMBOL(iio_register_sw_device_type); + +void iio_unregister_sw_device_type(struct iio_sw_device_type *dt) +{ + struct iio_sw_device_type *iter; + + mutex_lock(&iio_device_types_lock); + iter = __iio_find_sw_device_type(dt->name, strlen(dt->name)); + if (iter) + list_del(&dt->list); + mutex_unlock(&iio_device_types_lock); + + configfs_unregister_default_group(dt->group); +} +EXPORT_SYMBOL(iio_unregister_sw_device_type); + +static +struct iio_sw_device_type *iio_get_sw_device_type(const char *name) +{ + struct iio_sw_device_type *dt; + + mutex_lock(&iio_device_types_lock); + dt = __iio_find_sw_device_type(name, strlen(name)); + if (dt && !try_module_get(dt->owner)) + dt = NULL; + mutex_unlock(&iio_device_types_lock); + + return dt; +} + +struct iio_sw_device *iio_sw_device_create(const char *type, const char *name) +{ + struct iio_sw_device *d; + struct iio_sw_device_type *dt; + + dt = iio_get_sw_device_type(type); + if (!dt) { + pr_err("Invalid device type: %s\n", type); + return ERR_PTR(-EINVAL); + } + d = dt->ops->probe(name); + if (IS_ERR(d)) + goto out_module_put; + + d->device_type = dt; + + return d; +out_module_put: + module_put(dt->owner); + return d; +} +EXPORT_SYMBOL(iio_sw_device_create); + +void iio_sw_device_destroy(struct iio_sw_device *d) +{ + struct iio_sw_device_type *dt = d->device_type; + + dt->ops->remove(d); + module_put(dt->owner); +} +EXPORT_SYMBOL(iio_sw_device_destroy); + +static struct config_group *device_make_group(struct config_group *group, + const char *name) +{ + struct iio_sw_device *d; + + d = iio_sw_device_create(group->cg_item.ci_name, name); + if (IS_ERR(d)) + return ERR_CAST(d); + + config_item_set_name(&d->group.cg_item, "%s", name); + + return &d->group; +} + +static void device_drop_group(struct config_group *group, + struct config_item *item) +{ + struct iio_sw_device *d = to_iio_sw_device(item); + + iio_sw_device_destroy(d); + config_item_put(item); +} + +static struct configfs_group_operations device_ops = { + .make_group = &device_make_group, + .drop_item = &device_drop_group, +}; + +static struct config_item_type iio_device_type_group_type = { + .ct_group_ops = &device_ops, + .ct_owner = THIS_MODULE, +}; + +static int __init iio_sw_device_init(void) +{ + iio_devices_group = + configfs_register_default_group(&iio_configfs_subsys.su_group, + "devices", + &iio_devices_group_type); + return PTR_ERR_OR_ZERO(iio_devices_group); +} +module_init(iio_sw_device_init); + +static void __exit iio_sw_device_exit(void) +{ + configfs_unregister_default_group(iio_devices_group); +} +module_exit(iio_sw_device_exit); + +MODULE_AUTHOR("Daniel Baluta "); +MODULE_DESCRIPTION("Industrial I/O software devices support"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c index 0c52dfe64977..7ad82fdd3e5b 100644 --- a/drivers/iio/industrialio-trigger.c +++ b/drivers/iio/industrialio-trigger.c @@ -64,10 +64,16 @@ static struct attribute *iio_trig_dev_attrs[] = { }; ATTRIBUTE_GROUPS(iio_trig_dev); +static struct iio_trigger *__iio_trigger_find_by_name(const char *name); + int iio_trigger_register(struct iio_trigger *trig_info) { int ret; + /* trig_info->ops is required for the module member */ + if (!trig_info->ops) + return -EINVAL; + trig_info->id = ida_simple_get(&iio_trigger_ida, 0, 0, GFP_KERNEL); if (trig_info->id < 0) return trig_info->id; @@ -82,11 +88,19 @@ int iio_trigger_register(struct iio_trigger *trig_info) /* Add to list of available triggers held by the IIO core */ mutex_lock(&iio_trigger_list_lock); + if (__iio_trigger_find_by_name(trig_info->name)) { + pr_err("Duplicate trigger name '%s'\n", trig_info->name); + ret = -EEXIST; + goto error_device_del; + } list_add_tail(&trig_info->list, &iio_trigger_list); mutex_unlock(&iio_trigger_list_lock); return 0; +error_device_del: + mutex_unlock(&iio_trigger_list_lock); + device_del(&trig_info->dev); error_unregister_id: ida_simple_remove(&iio_trigger_ida, trig_info->id); return ret; @@ -105,6 +119,18 @@ void iio_trigger_unregister(struct iio_trigger *trig_info) } EXPORT_SYMBOL(iio_trigger_unregister); +/* Search for trigger by name, assuming iio_trigger_list_lock held */ +static struct iio_trigger *__iio_trigger_find_by_name(const char *name) +{ + struct iio_trigger *iter; + + list_for_each_entry(iter, &iio_trigger_list, list) + if (!strcmp(iter->name, name)) + return iter; + + return NULL; +} + static struct iio_trigger *iio_trigger_find_by_name(const char *name, size_t len) { @@ -164,8 +190,7 @@ EXPORT_SYMBOL(iio_trigger_poll_chained); void iio_trigger_notify_done(struct iio_trigger *trig) { - if (atomic_dec_and_test(&trig->use_count) && trig->ops && - trig->ops->try_reenable) + if (atomic_dec_and_test(&trig->use_count) && trig->ops->try_reenable) if (trig->ops->try_reenable(trig)) /* Missed an interrupt so launch new poll now */ iio_trigger_poll(trig); @@ -224,7 +249,7 @@ static int iio_trigger_attach_poll_func(struct iio_trigger *trig, goto out_put_irq; /* Enable trigger in driver */ - if (trig->ops && trig->ops->set_trigger_state && notinuse) { + if (trig->ops->set_trigger_state && notinuse) { ret = trig->ops->set_trigger_state(trig, true); if (ret < 0) goto out_free_irq; @@ -249,7 +274,7 @@ static int iio_trigger_detach_poll_func(struct iio_trigger *trig, = (bitmap_weight(trig->pool, CONFIG_IIO_CONSUMERS_PER_TRIGGER) == 1); - if (trig->ops && trig->ops->set_trigger_state && no_other_users) { + if (trig->ops->set_trigger_state && no_other_users) { ret = trig->ops->set_trigger_state(trig, false); if (ret) return ret; @@ -264,7 +289,7 @@ static int iio_trigger_detach_poll_func(struct iio_trigger *trig, irqreturn_t iio_pollfunc_store_time(int irq, void *p) { struct iio_poll_func *pf = p; - pf->timestamp = iio_get_time_ns(); + pf->timestamp = iio_get_time_ns(pf->indio_dev); return IRQ_WAKE_THREAD; } EXPORT_SYMBOL(iio_pollfunc_store_time); @@ -371,7 +396,7 @@ static ssize_t iio_trigger_write_current(struct device *dev, return ret; } - if (trig && trig->ops && trig->ops->validate_device) { + if (trig && trig->ops->validate_device) { ret = trig->ops->validate_device(trig, indio_dev); if (ret) return ret; diff --git a/drivers/iio/light/acpi-als.c b/drivers/iio/light/acpi-als.c index 53201d99a16c..f0b47c501f4e 100644 --- a/drivers/iio/light/acpi-als.c +++ b/drivers/iio/light/acpi-als.c @@ -118,7 +118,7 @@ static void acpi_als_notify(struct acpi_device *device, u32 event) struct iio_dev *indio_dev = acpi_driver_data(device); struct acpi_als *als = iio_priv(indio_dev); s32 *buffer = als->evt_buffer; - s64 time_ns = iio_get_time_ns(); + s64 time_ns = iio_get_time_ns(indio_dev); s32 val; int ret; diff --git a/drivers/iio/light/adjd_s311.c b/drivers/iio/light/adjd_s311.c index 09ad5f1ce539..0113fc843a81 100644 --- a/drivers/iio/light/adjd_s311.c +++ b/drivers/iio/light/adjd_s311.c @@ -118,7 +118,7 @@ static irqreturn_t adjd_s311_trigger_handler(int irq, void *p) struct iio_poll_func *pf = p; struct iio_dev *indio_dev = pf->indio_dev; struct adjd_s311_data *data = iio_priv(indio_dev); - s64 time_ns = iio_get_time_ns(); + s64 time_ns = iio_get_time_ns(indio_dev); int i, j = 0; int ret = adjd_s311_req_data(indio_dev); diff --git a/drivers/iio/light/apds9300.c b/drivers/iio/light/apds9300.c index e1b9fa5a7e91..649b26f67813 100644 --- a/drivers/iio/light/apds9300.c +++ b/drivers/iio/light/apds9300.c @@ -396,7 +396,7 @@ static irqreturn_t apds9300_interrupt_handler(int irq, void *private) IIO_UNMOD_EVENT_CODE(IIO_INTENSITY, 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), - iio_get_time_ns()); + iio_get_time_ns(dev_info)); apds9300_clear_intr(data); diff --git a/drivers/iio/light/apds9960.c b/drivers/iio/light/apds9960.c index 651d57b8abbf..a4304edc3e0f 100644 --- a/drivers/iio/light/apds9960.c +++ b/drivers/iio/light/apds9960.c @@ -807,7 +807,7 @@ static irqreturn_t apds9960_interrupt_handler(int irq, void *private) IIO_UNMOD_EVENT_CODE(IIO_INTENSITY, 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); regmap_write(data->regmap, APDS9960_REG_CICLEAR, 1); } @@ -816,7 +816,7 @@ static irqreturn_t apds9960_interrupt_handler(int irq, void *private) IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); regmap_write(data->regmap, APDS9960_REG_PICLEAR, 1); } diff --git a/drivers/iio/light/cm36651.c b/drivers/iio/light/cm36651.c index c8d7b5ea7e78..9d66e89c57ef 100644 --- a/drivers/iio/light/cm36651.c +++ b/drivers/iio/light/cm36651.c @@ -268,7 +268,7 @@ static irqreturn_t cm36651_irq_handler(int irq, void *data) CM36651_CMD_READ_RAW_PROXIMITY, IIO_EV_TYPE_THRESH, ev_dir); - iio_push_event(indio_dev, ev_code, iio_get_time_ns()); + iio_push_event(indio_dev, ev_code, iio_get_time_ns(indio_dev)); return IRQ_HANDLED; } diff --git a/drivers/iio/light/gp2ap020a00f.c b/drivers/iio/light/gp2ap020a00f.c index 6d41086f7c64..6ada9149f142 100644 --- a/drivers/iio/light/gp2ap020a00f.c +++ b/drivers/iio/light/gp2ap020a00f.c @@ -851,7 +851,7 @@ static irqreturn_t gp2ap020a00f_prox_sensing_handler(int irq, void *data) GP2AP020A00F_SCAN_MODE_PROXIMITY, IIO_EV_TYPE_ROC, IIO_EV_DIR_RISING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } else { iio_push_event(indio_dev, IIO_UNMOD_EVENT_CODE( @@ -859,7 +859,7 @@ static irqreturn_t gp2ap020a00f_prox_sensing_handler(int irq, void *data) GP2AP020A00F_SCAN_MODE_PROXIMITY, IIO_EV_TYPE_ROC, IIO_EV_DIR_FALLING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } } @@ -925,7 +925,7 @@ static irqreturn_t gp2ap020a00f_thresh_event_handler(int irq, void *data) IIO_MOD_LIGHT_CLEAR, IIO_EV_TYPE_THRESH, IIO_EV_DIR_RISING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } if (test_bit(GP2AP020A00F_FLAG_ALS_FALLING_EV, &priv->flags)) { @@ -939,7 +939,7 @@ static irqreturn_t gp2ap020a00f_thresh_event_handler(int irq, void *data) IIO_MOD_LIGHT_CLEAR, IIO_EV_TYPE_THRESH, IIO_EV_DIR_FALLING), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } } @@ -1287,22 +1287,14 @@ static int gp2ap020a00f_read_raw(struct iio_dev *indio_dev, struct gp2ap020a00f_data *data = iio_priv(indio_dev); int err = -EINVAL; - mutex_lock(&data->lock); - - switch (mask) { - case IIO_CHAN_INFO_RAW: - if (iio_buffer_enabled(indio_dev)) { - err = -EBUSY; - goto error_unlock; - } + if (mask == IIO_CHAN_INFO_RAW) { + err = iio_device_claim_direct_mode(indio_dev); + if (err) + return err; err = gp2ap020a00f_read_channel(data, chan, val); - break; + iio_device_release_direct_mode(indio_dev); } - -error_unlock: - mutex_unlock(&data->lock); - return err < 0 ? err : IIO_VAL_INT; } diff --git a/drivers/iio/light/isl29125.c b/drivers/iio/light/isl29125.c index e2945a20e5f6..1d2c0c8a1d4f 100644 --- a/drivers/iio/light/isl29125.c +++ b/drivers/iio/light/isl29125.c @@ -44,13 +44,15 @@ #define ISL29125_MODE_B 0x3 #define ISL29125_MODE_RGB 0x5 +#define ISL29125_SENSING_RANGE_0 5722 /* 375 lux full range */ +#define ISL29125_SENSING_RANGE_1 152590 /* 10k lux full range */ + #define ISL29125_MODE_RANGE BIT(3) #define ISL29125_STATUS_CONV BIT(1) struct isl29125_data { struct i2c_client *client; - struct mutex lock; u8 conf1; u16 buffer[8]; /* 3x 16-bit, padding, 8 bytes timestamp */ }; @@ -128,11 +130,11 @@ static int isl29125_read_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: - if (iio_buffer_enabled(indio_dev)) - return -EBUSY; - mutex_lock(&data->lock); + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; ret = isl29125_read_data(data, chan->scan_index); - mutex_unlock(&data->lock); + iio_device_release_direct_mode(indio_dev); if (ret < 0) return ret; *val = ret; @@ -140,9 +142,9 @@ static int isl29125_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_SCALE: *val = 0; if (data->conf1 & ISL29125_MODE_RANGE) - *val2 = 152590; /* 10k lux full range */ + *val2 = ISL29125_SENSING_RANGE_1; /*10k lux full range*/ else - *val2 = 5722; /* 375 lux full range */ + *val2 = ISL29125_SENSING_RANGE_0; /*375 lux full range*/ return IIO_VAL_INT_PLUS_MICRO; } return -EINVAL; @@ -158,9 +160,9 @@ static int isl29125_write_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_SCALE: if (val != 0) return -EINVAL; - if (val2 == 152590) + if (val2 == ISL29125_SENSING_RANGE_1) data->conf1 |= ISL29125_MODE_RANGE; - else if (val2 == 5722) + else if (val2 == ISL29125_SENSING_RANGE_0) data->conf1 &= ~ISL29125_MODE_RANGE; else return -EINVAL; @@ -189,7 +191,7 @@ static irqreturn_t isl29125_trigger_handler(int irq, void *p) } iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -259,7 +261,6 @@ static int isl29125_probe(struct i2c_client *client, data = iio_priv(indio_dev); i2c_set_clientdata(client, indio_dev); data->client = client; - mutex_init(&data->lock); indio_dev->dev.parent = &client->dev; indio_dev->info = &isl29125_info; diff --git a/drivers/iio/light/jsa1212.c b/drivers/iio/light/jsa1212.c index 99a62816c3b4..e8a8931b4f50 100644 --- a/drivers/iio/light/jsa1212.c +++ b/drivers/iio/light/jsa1212.c @@ -325,9 +325,6 @@ static int jsa1212_probe(struct i2c_client *client, struct regmap *regmap; int ret; - if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA)) - return -EOPNOTSUPP; - indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data)); if (!indio_dev) return -ENOMEM; diff --git a/drivers/iio/light/lm3533-als.c b/drivers/iio/light/lm3533-als.c index e56937c40a18..f409c2047c05 100644 --- a/drivers/iio/light/lm3533-als.c +++ b/drivers/iio/light/lm3533-als.c @@ -267,7 +267,7 @@ static irqreturn_t lm3533_als_isr(int irq, void *dev_id) 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); out: return IRQ_HANDLED; } diff --git a/drivers/iio/light/ltr501.c b/drivers/iio/light/ltr501.c index 6bf89d8f3741..3afc53a3d0b6 100644 --- a/drivers/iio/light/ltr501.c +++ b/drivers/iio/light/ltr501.c @@ -1256,7 +1256,8 @@ static irqreturn_t ltr501_trigger_handler(int irq, void *p) buf[j++] = psdata & LTR501_PS_DATA_MASK; } - iio_push_to_buffers_with_timestamp(indio_dev, buf, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, buf, + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -1282,14 +1283,14 @@ static irqreturn_t ltr501_interrupt_handler(int irq, void *private) IIO_UNMOD_EVENT_CODE(IIO_INTENSITY, 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); if (status & LTR501_STATUS_PS_INTR) iio_push_event(indio_dev, IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); return IRQ_HANDLED; } diff --git a/drivers/iio/light/max44000.c b/drivers/iio/light/max44000.c index f17cb2ea18f5..6511b20a2a29 100644 --- a/drivers/iio/light/max44000.c +++ b/drivers/iio/light/max44000.c @@ -511,7 +511,8 @@ static irqreturn_t max44000_trigger_handler(int irq, void *p) } mutex_unlock(&data->lock); - iio_push_to_buffers_with_timestamp(indio_dev, buf, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, buf, + iio_get_time_ns(indio_dev)); iio_trigger_notify_done(indio_dev->trig); return IRQ_HANDLED; diff --git a/drivers/iio/light/opt3001.c b/drivers/iio/light/opt3001.c index b776c8ed4387..78c9b3a6453a 100644 --- a/drivers/iio/light/opt3001.c +++ b/drivers/iio/light/opt3001.c @@ -713,13 +713,13 @@ static irqreturn_t opt3001_irq(int irq, void *_iio) IIO_UNMOD_EVENT_CODE(IIO_LIGHT, 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_RISING), - iio_get_time_ns()); + iio_get_time_ns(iio)); if (ret & OPT3001_CONFIGURATION_FL) iio_push_event(iio, IIO_UNMOD_EVENT_CODE(IIO_LIGHT, 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_FALLING), - iio_get_time_ns()); + iio_get_time_ns(iio)); } else if (ret & OPT3001_CONFIGURATION_CRF) { ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_RESULT); if (ret < 0) { diff --git a/drivers/iio/light/stk3310.c b/drivers/iio/light/stk3310.c index 9e847f8f4f0c..45cf8b0a4363 100644 --- a/drivers/iio/light/stk3310.c +++ b/drivers/iio/light/stk3310.c @@ -528,7 +528,7 @@ static irqreturn_t stk3310_irq_handler(int irq, void *private) struct iio_dev *indio_dev = private; struct stk3310_data *data = iio_priv(indio_dev); - data->timestamp = iio_get_time_ns(); + data->timestamp = iio_get_time_ns(indio_dev); return IRQ_WAKE_THREAD; } diff --git a/drivers/iio/light/tcs3414.c b/drivers/iio/light/tcs3414.c index f90f8c5919fe..a795afb7667b 100644 --- a/drivers/iio/light/tcs3414.c +++ b/drivers/iio/light/tcs3414.c @@ -53,7 +53,6 @@ struct tcs3414_data { struct i2c_client *client; - struct mutex lock; u8 control; u8 gain; u8 timing; @@ -134,16 +133,16 @@ static int tcs3414_read_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: - if (iio_buffer_enabled(indio_dev)) - return -EBUSY; - mutex_lock(&data->lock); + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; ret = tcs3414_req_data(data); if (ret < 0) { - mutex_unlock(&data->lock); + iio_device_release_direct_mode(indio_dev); return ret; } ret = i2c_smbus_read_word_data(data->client, chan->address); - mutex_unlock(&data->lock); + iio_device_release_direct_mode(indio_dev); if (ret < 0) return ret; *val = ret; @@ -217,7 +216,7 @@ static irqreturn_t tcs3414_trigger_handler(int irq, void *p) } iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -288,7 +287,6 @@ static int tcs3414_probe(struct i2c_client *client, data = iio_priv(indio_dev); i2c_set_clientdata(client, indio_dev); data->client = client; - mutex_init(&data->lock); indio_dev->dev.parent = &client->dev; indio_dev->info = &tcs3414_info; diff --git a/drivers/iio/light/tcs3472.c b/drivers/iio/light/tcs3472.c index 1b530bf04c89..3aa71e34ae28 100644 --- a/drivers/iio/light/tcs3472.c +++ b/drivers/iio/light/tcs3472.c @@ -52,7 +52,6 @@ struct tcs3472_data { struct i2c_client *client; - struct mutex lock; u8 enable; u8 control; u8 atime; @@ -117,17 +116,16 @@ static int tcs3472_read_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: - if (iio_buffer_enabled(indio_dev)) - return -EBUSY; - - mutex_lock(&data->lock); + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; ret = tcs3472_req_data(data); if (ret < 0) { - mutex_unlock(&data->lock); + iio_device_release_direct_mode(indio_dev); return ret; } ret = i2c_smbus_read_word_data(data->client, chan->address); - mutex_unlock(&data->lock); + iio_device_release_direct_mode(indio_dev); if (ret < 0) return ret; *val = ret; @@ -204,7 +202,7 @@ static irqreturn_t tcs3472_trigger_handler(int irq, void *p) } iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); @@ -263,7 +261,6 @@ static int tcs3472_probe(struct i2c_client *client, data = iio_priv(indio_dev); i2c_set_clientdata(client, indio_dev); data->client = client; - mutex_init(&data->lock); indio_dev->dev.parent = &client->dev; indio_dev->info = &tcs3472_info; diff --git a/drivers/iio/light/tsl2563.c b/drivers/iio/light/tsl2563.c index 57b108c30e98..04598ae993d4 100644 --- a/drivers/iio/light/tsl2563.c +++ b/drivers/iio/light/tsl2563.c @@ -630,7 +630,7 @@ static irqreturn_t tsl2563_event_handler(int irq, void *private) 0, IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), - iio_get_time_ns()); + iio_get_time_ns(dev_info)); /* clear the interrupt and push the event */ i2c_smbus_write_byte(chip->client, TSL2563_CMD | TSL2563_CLEARINT); diff --git a/drivers/iio/light/us5182d.c b/drivers/iio/light/us5182d.c index 45bc2f742f46..20c40f780964 100644 --- a/drivers/iio/light/us5182d.c +++ b/drivers/iio/light/us5182d.c @@ -833,7 +833,7 @@ static irqreturn_t us5182d_irq_thread_handler(int irq, void *private) dir = ret & US5182D_CFG0_PROX ? IIO_EV_DIR_RISING : IIO_EV_DIR_FALLING; ev = IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 1, IIO_EV_TYPE_THRESH, dir); - iio_push_event(indio_dev, ev, iio_get_time_ns()); + iio_push_event(indio_dev, ev, iio_get_time_ns(indio_dev)); ret = i2c_smbus_write_byte_data(data->client, US5182D_REG_CFG0, ret & ~US5182D_CFG0_PX_IRQ); diff --git a/drivers/iio/magnetometer/Kconfig b/drivers/iio/magnetometer/Kconfig index 84e6559ccc65..1f842abcb4a4 100644 --- a/drivers/iio/magnetometer/Kconfig +++ b/drivers/iio/magnetometer/Kconfig @@ -44,6 +44,7 @@ config BMC150_MAGN_I2C This driver is only implementing magnetometer part, which has its own address and register map. + This driver also supports I2C Bosch BMC156 and BMM150 chips. To compile this driver as a module, choose M here: the module will be called bmc150_magn_i2c. @@ -60,6 +61,7 @@ config BMC150_MAGN_SPI This driver is only implementing magnetometer part, which has its own address and register map. + This driver also supports SPI Bosch BMC156 and BMM150 chips. To compile this driver as a module, choose M here: the module will be called bmc150_magn_spi. diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c index 609a2c401b5d..af8606cc7812 100644 --- a/drivers/iio/magnetometer/ak8975.c +++ b/drivers/iio/magnetometer/ak8975.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -379,37 +380,40 @@ struct ak8975_data { u8 cntl_cache; struct iio_mount_matrix orientation; struct regulator *vdd; + struct regulator *vid; }; /* Enable attached power regulator if any. */ -static int ak8975_power_on(struct i2c_client *client) +static int ak8975_power_on(const struct ak8975_data *data) { - const struct iio_dev *indio_dev = i2c_get_clientdata(client); - struct ak8975_data *data = iio_priv(indio_dev); int ret; - data->vdd = devm_regulator_get(&client->dev, "vdd"); - if (IS_ERR_OR_NULL(data->vdd)) { - ret = PTR_ERR(data->vdd); - if (ret == -ENODEV) - ret = 0; - } else { - ret = regulator_enable(data->vdd); + ret = regulator_enable(data->vdd); + if (ret) { + dev_warn(&data->client->dev, + "Failed to enable specified Vdd supply\n"); + return ret; } - - if (ret) - dev_err(&client->dev, "failed to enable Vdd supply: %d\n", ret); - return ret; + ret = regulator_enable(data->vid); + if (ret) { + dev_warn(&data->client->dev, + "Failed to enable specified Vid supply\n"); + return ret; + } + /* + * According to the datasheet the power supply rise time i 200us + * and the minimum wait time before mode setting is 100us, in + * total 300 us. Add some margin and say minimum 500us here. + */ + usleep_range(500, 1000); + return 0; } /* Disable attached power regulator if any. */ -static void ak8975_power_off(const struct i2c_client *client) +static void ak8975_power_off(const struct ak8975_data *data) { - const struct iio_dev *indio_dev = i2c_get_clientdata(client); - const struct ak8975_data *data = iio_priv(indio_dev); - - if (!IS_ERR_OR_NULL(data->vdd)) - regulator_disable(data->vdd); + regulator_disable(data->vid); + regulator_disable(data->vdd); } /* @@ -430,8 +434,8 @@ static int ak8975_who_i_am(struct i2c_client *client, * AK8975 | DEVICE_ID | NA * AK8963 | DEVICE_ID | NA */ - ret = i2c_smbus_read_i2c_block_data(client, AK09912_REG_WIA1, - 2, wia_val); + ret = i2c_smbus_read_i2c_block_data_or_emulated( + client, AK09912_REG_WIA1, 2, wia_val); if (ret < 0) { dev_err(&client->dev, "Error reading WIA\n"); return ret; @@ -543,9 +547,9 @@ static int ak8975_setup(struct i2c_client *client) } /* Get asa data and store in the device data. */ - ret = i2c_smbus_read_i2c_block_data(client, - data->def->ctrl_regs[ASA_BASE], - 3, data->asa); + ret = i2c_smbus_read_i2c_block_data_or_emulated( + client, data->def->ctrl_regs[ASA_BASE], + 3, data->asa); if (ret < 0) { dev_err(&client->dev, "Not able to read asa data\n"); return ret; @@ -686,22 +690,31 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val) struct ak8975_data *data = iio_priv(indio_dev); const struct i2c_client *client = data->client; const struct ak_def *def = data->def; + u16 buff; int ret; + pm_runtime_get_sync(&data->client->dev); + mutex_lock(&data->lock); ret = ak8975_start_read_axis(data, client); if (ret) goto exit; - ret = i2c_smbus_read_word_data(client, def->data_regs[index]); + ret = i2c_smbus_read_i2c_block_data_or_emulated( + client, def->data_regs[index], + sizeof(buff), (u8*)&buff); if (ret < 0) goto exit; mutex_unlock(&data->lock); - /* Clamp to valid range. */ - *val = clamp_t(s16, ret, -def->range, def->range); + pm_runtime_mark_last_busy(&data->client->dev); + pm_runtime_put_autosuspend(&data->client->dev); + + /* Swap bytes and convert to valid range. */ + buff = le16_to_cpu(buff); + *val = clamp_t(s16, buff, -def->range, def->range); return IIO_VAL_INT; exit: @@ -825,7 +838,8 @@ static void ak8975_fill_buffer(struct iio_dev *indio_dev) buff[1] = clamp_t(s16, le16_to_cpu(buff[1]), -def->range, def->range); buff[2] = clamp_t(s16, le16_to_cpu(buff[2]), -def->range, def->range); - iio_push_to_buffers_with_timestamp(indio_dev, buff, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, buff, + iio_get_time_ns(indio_dev)); return; unlock: @@ -919,7 +933,15 @@ static int ak8975_probe(struct i2c_client *client, data->def = &ak_def_array[chipset]; - err = ak8975_power_on(client); + /* Fetch the regulators */ + data->vdd = devm_regulator_get(&client->dev, "vdd"); + if (IS_ERR(data->vdd)) + return PTR_ERR(data->vdd); + data->vid = devm_regulator_get(&client->dev, "vid"); + if (IS_ERR(data->vid)) + return PTR_ERR(data->vid); + + err = ak8975_power_on(data); if (err) return err; @@ -959,26 +981,93 @@ static int ak8975_probe(struct i2c_client *client, goto cleanup_buffer; } + /* Enable runtime PM */ + pm_runtime_get_noresume(&client->dev); + pm_runtime_set_active(&client->dev); + pm_runtime_enable(&client->dev); + /* + * The device comes online in 500us, so add two orders of magnitude + * of delay before autosuspending: 50 ms. + */ + pm_runtime_set_autosuspend_delay(&client->dev, 50); + pm_runtime_use_autosuspend(&client->dev); + pm_runtime_put(&client->dev); + return 0; cleanup_buffer: iio_triggered_buffer_cleanup(indio_dev); power_off: - ak8975_power_off(client); + ak8975_power_off(data); return err; } static int ak8975_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); + struct ak8975_data *data = iio_priv(indio_dev); + pm_runtime_get_sync(&client->dev); + pm_runtime_put_noidle(&client->dev); + pm_runtime_disable(&client->dev); iio_device_unregister(indio_dev); iio_triggered_buffer_cleanup(indio_dev); - ak8975_power_off(client); + ak8975_set_mode(data, POWER_DOWN); + ak8975_power_off(data); return 0; } +#ifdef CONFIG_PM +static int ak8975_runtime_suspend(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct iio_dev *indio_dev = i2c_get_clientdata(client); + struct ak8975_data *data = iio_priv(indio_dev); + int ret; + + /* Set the device in power down if it wasn't already */ + ret = ak8975_set_mode(data, POWER_DOWN); + if (ret < 0) { + dev_err(&client->dev, "Error in setting power-down mode\n"); + return ret; + } + /* Next cut the regulators */ + ak8975_power_off(data); + + return 0; +} + +static int ak8975_runtime_resume(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct iio_dev *indio_dev = i2c_get_clientdata(client); + struct ak8975_data *data = iio_priv(indio_dev); + int ret; + + /* Take up the regulators */ + ak8975_power_on(data); + /* + * We come up in powered down mode, the reading routines will + * put us in the mode to read values later. + */ + ret = ak8975_set_mode(data, POWER_DOWN); + if (ret < 0) { + dev_err(&client->dev, "Error in setting power-down mode\n"); + return ret; + } + + return 0; +} +#endif /* CONFIG_PM */ + +static const struct dev_pm_ops ak8975_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(ak8975_runtime_suspend, + ak8975_runtime_resume, NULL) +}; + static const struct i2c_device_id ak8975_id[] = { {"ak8975", AK8975}, {"ak8963", AK8963}, @@ -1006,6 +1095,7 @@ MODULE_DEVICE_TABLE(of, ak8975_of_match); static struct i2c_driver ak8975_driver = { .driver = { .name = "ak8975", + .pm = &ak8975_dev_pm_ops, .of_match_table = of_match_ptr(ak8975_of_match), .acpi_match_table = ACPI_PTR(ak_acpi_match), }, diff --git a/drivers/iio/magnetometer/bmc150_magn_i2c.c b/drivers/iio/magnetometer/bmc150_magn_i2c.c index eddc7f0d0096..ee05722587aa 100644 --- a/drivers/iio/magnetometer/bmc150_magn_i2c.c +++ b/drivers/iio/magnetometer/bmc150_magn_i2c.c @@ -2,6 +2,7 @@ * 3-axis magnetometer driver supporting following I2C Bosch-Sensortec chips: * - BMC150 * - BMC156 + * - BMM150 * * Copyright (c) 2016, Intel Corporation. * @@ -49,6 +50,7 @@ static int bmc150_magn_i2c_remove(struct i2c_client *client) static const struct acpi_device_id bmc150_magn_acpi_match[] = { {"BMC150B", 0}, {"BMC156B", 0}, + {"BMM150B", 0}, {}, }; MODULE_DEVICE_TABLE(acpi, bmc150_magn_acpi_match); @@ -56,6 +58,7 @@ MODULE_DEVICE_TABLE(acpi, bmc150_magn_acpi_match); static const struct i2c_device_id bmc150_magn_i2c_id[] = { {"bmc150_magn", 0}, {"bmc156_magn", 0}, + {"bmm150_magn", 0}, {} }; MODULE_DEVICE_TABLE(i2c, bmc150_magn_i2c_id); diff --git a/drivers/iio/magnetometer/bmc150_magn_spi.c b/drivers/iio/magnetometer/bmc150_magn_spi.c index c4c738a07695..7d4152d4d01e 100644 --- a/drivers/iio/magnetometer/bmc150_magn_spi.c +++ b/drivers/iio/magnetometer/bmc150_magn_spi.c @@ -2,6 +2,7 @@ * 3-axis magnetometer driver support following SPI Bosch-Sensortec chips: * - BMC150 * - BMC156 + * - BMM150 * * Copyright (c) 2016, Intel Corporation. * @@ -41,6 +42,7 @@ static int bmc150_magn_spi_remove(struct spi_device *spi) static const struct spi_device_id bmc150_magn_spi_id[] = { {"bmc150_magn", 0}, {"bmc156_magn", 0}, + {"bmm150_magn", 0}, {} }; MODULE_DEVICE_TABLE(spi, bmc150_magn_spi_id); @@ -48,6 +50,7 @@ MODULE_DEVICE_TABLE(spi, bmc150_magn_spi_id); static const struct acpi_device_id bmc150_magn_acpi_match[] = { {"BMC150B", 0}, {"BMC156B", 0}, + {"BMM150B", 0}, {}, }; MODULE_DEVICE_TABLE(acpi, bmc150_magn_acpi_match); diff --git a/drivers/iio/magnetometer/hmc5843_core.c b/drivers/iio/magnetometer/hmc5843_core.c index 77882b466e0f..ba3e2a374ee5 100644 --- a/drivers/iio/magnetometer/hmc5843_core.c +++ b/drivers/iio/magnetometer/hmc5843_core.c @@ -451,7 +451,7 @@ static irqreturn_t hmc5843_trigger_handler(int irq, void *p) goto done; iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); diff --git a/drivers/iio/magnetometer/mag3110.c b/drivers/iio/magnetometer/mag3110.c index 261d517428e4..f2be4a049056 100644 --- a/drivers/iio/magnetometer/mag3110.c +++ b/drivers/iio/magnetometer/mag3110.c @@ -261,7 +261,7 @@ static irqreturn_t mag3110_trigger_handler(int irq, void *p) } iio_push_to_buffers_with_timestamp(indio_dev, buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); diff --git a/drivers/iio/magnetometer/st_magn_core.c b/drivers/iio/magnetometer/st_magn_core.c index 8250fc322c56..3e1f06b2224c 100644 --- a/drivers/iio/magnetometer/st_magn_core.c +++ b/drivers/iio/magnetometer/st_magn_core.c @@ -589,13 +589,15 @@ int st_magn_common_probe(struct iio_dev *indio_dev) indio_dev->info = &magn_info; mutex_init(&mdata->tb.buf_lock); - st_sensors_power_enable(indio_dev); + err = st_sensors_power_enable(indio_dev); + if (err) + return err; err = st_sensors_check_device_support(indio_dev, ARRAY_SIZE(st_magn_sensors_settings), st_magn_sensors_settings); if (err < 0) - return err; + goto st_magn_power_off; mdata->num_data_channels = ST_MAGN_NUMBER_DATA_CHANNELS; mdata->multiread_bit = mdata->sensor_settings->multi_read_bit; @@ -608,11 +610,11 @@ int st_magn_common_probe(struct iio_dev *indio_dev) err = st_sensors_init_sensor(indio_dev, NULL); if (err < 0) - return err; + goto st_magn_power_off; err = st_magn_allocate_ring(indio_dev); if (err < 0) - return err; + goto st_magn_power_off; if (irq > 0) { err = st_sensors_allocate_trigger(indio_dev, @@ -635,6 +637,8 @@ st_magn_device_register_error: st_sensors_deallocate_trigger(indio_dev); st_magn_probe_trigger_error: st_magn_deallocate_ring(indio_dev); +st_magn_power_off: + st_sensors_power_disable(indio_dev); return err; } diff --git a/drivers/iio/potentiometer/Kconfig b/drivers/iio/potentiometer/Kconfig index 6acb23810bb4..2e9da1cf3297 100644 --- a/drivers/iio/potentiometer/Kconfig +++ b/drivers/iio/potentiometer/Kconfig @@ -10,11 +10,22 @@ config DS1803 depends on I2C help Say yes here to build support for the Maxim Integrated DS1803 - digital potentiomenter chip. + digital potentiometer chip. To compile this driver as a module, choose M here: the module will be called ds1803. +config MAX5487 + tristate "Maxim MAX5487/MAX5488/MAX5489 Digital Potentiometer driver" + depends on SPI + help + Say yes here to build support for the Maxim + MAX5487, MAX5488, MAX5489 digital potentiometer + chips. + + To compile this driver as a module, choose M here: the + module will be called max5487. + config MCP4131 tristate "Microchip MCP413X/414X/415X/416X/423X/424X/425X/426X Digital Potentiometer driver" depends on SPI @@ -28,7 +39,7 @@ config MCP4131 MCP4241, MCP4242, MCP4251, MCP4252, MCP4261, MCP4262, - digital potentiomenter chips. + digital potentiometer chips. To compile this driver as a module, choose M here: the module will be called mcp4131. @@ -38,9 +49,11 @@ config MCP4531 depends on I2C help Say yes here to build support for the Microchip - MCP4531, MCP4532, MCP4551, MCP4552, - MCP4631, MCP4632, MCP4651, MCP4652 - digital potentiomenter chips. + MCP4531, MCP4532, MCP4541, MCP4542, + MCP4551, MCP4552, MCP4561, MCP4562, + MCP4631, MCP4632, MCP4641, MCP4642, + MCP4651, MCP4652, MCP4661, MCP4662 + digital potentiometer chips. To compile this driver as a module, choose M here: the module will be called mcp4531. diff --git a/drivers/iio/potentiometer/Makefile b/drivers/iio/potentiometer/Makefile index 6007faa2fb02..8adb58f38c0b 100644 --- a/drivers/iio/potentiometer/Makefile +++ b/drivers/iio/potentiometer/Makefile @@ -4,6 +4,7 @@ # When adding new entries keep the list in alphabetical order obj-$(CONFIG_DS1803) += ds1803.o +obj-$(CONFIG_MAX5487) += max5487.o obj-$(CONFIG_MCP4131) += mcp4131.o obj-$(CONFIG_MCP4531) += mcp4531.o obj-$(CONFIG_TPL0102) += tpl0102.o diff --git a/drivers/iio/potentiometer/max5487.c b/drivers/iio/potentiometer/max5487.c new file mode 100644 index 000000000000..6c50939a2e83 --- /dev/null +++ b/drivers/iio/potentiometer/max5487.c @@ -0,0 +1,161 @@ +/* + * max5487.c - Support for MAX5487, MAX5488, MAX5489 digital potentiometers + * + * Copyright (C) 2016 Cristina-Gabriela Moraru + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ +#include +#include +#include + +#include +#include + +#define MAX5487_WRITE_WIPER_A (0x01 << 8) +#define MAX5487_WRITE_WIPER_B (0x02 << 8) + +/* copy both wiper regs to NV regs */ +#define MAX5487_COPY_AB_TO_NV (0x23 << 8) +/* copy both NV regs to wiper regs */ +#define MAX5487_COPY_NV_TO_AB (0x33 << 8) + +#define MAX5487_MAX_POS 255 + +struct max5487_data { + struct spi_device *spi; + int kohms; +}; + +#define MAX5487_CHANNEL(ch, addr) { \ + .type = IIO_RESISTANCE, \ + .indexed = 1, \ + .output = 1, \ + .channel = ch, \ + .address = addr, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ +} + +static const struct iio_chan_spec max5487_channels[] = { + MAX5487_CHANNEL(0, MAX5487_WRITE_WIPER_A), + MAX5487_CHANNEL(1, MAX5487_WRITE_WIPER_B), +}; + +static int max5487_write_cmd(struct spi_device *spi, u16 cmd) +{ + return spi_write(spi, (const void *) &cmd, sizeof(u16)); +} + +static int max5487_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct max5487_data *data = iio_priv(indio_dev); + + if (mask != IIO_CHAN_INFO_SCALE) + return -EINVAL; + + *val = 1000 * data->kohms; + *val2 = MAX5487_MAX_POS; + + return IIO_VAL_FRACTIONAL; +} + +static int max5487_write_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int val, int val2, long mask) +{ + struct max5487_data *data = iio_priv(indio_dev); + + if (mask != IIO_CHAN_INFO_RAW) + return -EINVAL; + + if (val < 0 || val > MAX5487_MAX_POS) + return -EINVAL; + + return max5487_write_cmd(data->spi, chan->address | val); +} + +static const struct iio_info max5487_info = { + .read_raw = max5487_read_raw, + .write_raw = max5487_write_raw, + .driver_module = THIS_MODULE, +}; + +static int max5487_spi_probe(struct spi_device *spi) +{ + struct iio_dev *indio_dev; + struct max5487_data *data; + const struct spi_device_id *id = spi_get_device_id(spi); + int ret; + + indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*data)); + if (!indio_dev) + return -ENOMEM; + + dev_set_drvdata(&spi->dev, indio_dev); + data = iio_priv(indio_dev); + + data->spi = spi; + data->kohms = id->driver_data; + + indio_dev->info = &max5487_info; + indio_dev->name = id->name; + indio_dev->dev.parent = &spi->dev; + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->channels = max5487_channels; + indio_dev->num_channels = ARRAY_SIZE(max5487_channels); + + /* restore both wiper regs from NV regs */ + ret = max5487_write_cmd(data->spi, MAX5487_COPY_NV_TO_AB); + if (ret < 0) + return ret; + + return iio_device_register(indio_dev); +} + +static int max5487_spi_remove(struct spi_device *spi) +{ + struct iio_dev *indio_dev = dev_get_drvdata(&spi->dev); + + iio_device_unregister(indio_dev); + + /* save both wiper regs to NV regs */ + return max5487_write_cmd(spi, MAX5487_COPY_AB_TO_NV); +} + +static const struct spi_device_id max5487_id[] = { + { "MAX5487", 10 }, + { "MAX5488", 50 }, + { "MAX5489", 100 }, + { } +}; +MODULE_DEVICE_TABLE(spi, max5487_id); + +static const struct acpi_device_id max5487_acpi_match[] = { + { "MAX5487", 10 }, + { "MAX5488", 50 }, + { "MAX5489", 100 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, max5487_acpi_match); + +static struct spi_driver max5487_driver = { + .driver = { + .name = "max5487", + .owner = THIS_MODULE, + .acpi_match_table = ACPI_PTR(max5487_acpi_match), + }, + .id_table = max5487_id, + .probe = max5487_spi_probe, + .remove = max5487_spi_remove +}; +module_spi_driver(max5487_driver); + +MODULE_AUTHOR("Cristina-Gabriela Moraru "); +MODULE_DESCRIPTION("max5487 SPI driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/potentiometer/mcp4531.c b/drivers/iio/potentiometer/mcp4531.c index 3b72e1a595db..13b6ae2fcf7b 100644 --- a/drivers/iio/potentiometer/mcp4531.c +++ b/drivers/iio/potentiometer/mcp4531.c @@ -8,12 +8,20 @@ * DEVID #Wipers #Positions Resistor Opts (kOhm) i2c address * mcp4531 1 129 5, 10, 50, 100 010111x * mcp4532 1 129 5, 10, 50, 100 01011xx + * mcp4541 1 129 5, 10, 50, 100 010111x + * mcp4542 1 129 5, 10, 50, 100 01011xx * mcp4551 1 257 5, 10, 50, 100 010111x * mcp4552 1 257 5, 10, 50, 100 01011xx + * mcp4561 1 257 5, 10, 50, 100 010111x + * mcp4562 1 257 5, 10, 50, 100 01011xx * mcp4631 2 129 5, 10, 50, 100 0101xxx * mcp4632 2 129 5, 10, 50, 100 01011xx + * mcp4641 2 129 5, 10, 50, 100 0101xxx + * mcp4642 2 129 5, 10, 50, 100 01011xx * mcp4651 2 257 5, 10, 50, 100 0101xxx * mcp4652 2 257 5, 10, 50, 100 01011xx + * mcp4661 2 257 5, 10, 50, 100 0101xxx + * mcp4662 2 257 5, 10, 50, 100 01011xx * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published by @@ -23,6 +31,8 @@ #include #include #include +#include +#include #include @@ -37,18 +47,34 @@ enum mcp4531_type { MCP453x_103, MCP453x_503, MCP453x_104, + MCP454x_502, + MCP454x_103, + MCP454x_503, + MCP454x_104, MCP455x_502, MCP455x_103, MCP455x_503, MCP455x_104, + MCP456x_502, + MCP456x_103, + MCP456x_503, + MCP456x_104, MCP463x_502, MCP463x_103, MCP463x_503, MCP463x_104, + MCP464x_502, + MCP464x_103, + MCP464x_503, + MCP464x_104, MCP465x_502, MCP465x_103, MCP465x_503, MCP465x_104, + MCP466x_502, + MCP466x_103, + MCP466x_503, + MCP466x_104, }; static const struct mcp4531_cfg mcp4531_cfg[] = { @@ -56,18 +82,34 @@ static const struct mcp4531_cfg mcp4531_cfg[] = { [MCP453x_103] = { .wipers = 1, .max_pos = 128, .kohms = 10, }, [MCP453x_503] = { .wipers = 1, .max_pos = 128, .kohms = 50, }, [MCP453x_104] = { .wipers = 1, .max_pos = 128, .kohms = 100, }, + [MCP454x_502] = { .wipers = 1, .max_pos = 128, .kohms = 5, }, + [MCP454x_103] = { .wipers = 1, .max_pos = 128, .kohms = 10, }, + [MCP454x_503] = { .wipers = 1, .max_pos = 128, .kohms = 50, }, + [MCP454x_104] = { .wipers = 1, .max_pos = 128, .kohms = 100, }, [MCP455x_502] = { .wipers = 1, .max_pos = 256, .kohms = 5, }, [MCP455x_103] = { .wipers = 1, .max_pos = 256, .kohms = 10, }, [MCP455x_503] = { .wipers = 1, .max_pos = 256, .kohms = 50, }, [MCP455x_104] = { .wipers = 1, .max_pos = 256, .kohms = 100, }, + [MCP456x_502] = { .wipers = 1, .max_pos = 256, .kohms = 5, }, + [MCP456x_103] = { .wipers = 1, .max_pos = 256, .kohms = 10, }, + [MCP456x_503] = { .wipers = 1, .max_pos = 256, .kohms = 50, }, + [MCP456x_104] = { .wipers = 1, .max_pos = 256, .kohms = 100, }, [MCP463x_502] = { .wipers = 2, .max_pos = 128, .kohms = 5, }, [MCP463x_103] = { .wipers = 2, .max_pos = 128, .kohms = 10, }, [MCP463x_503] = { .wipers = 2, .max_pos = 128, .kohms = 50, }, [MCP463x_104] = { .wipers = 2, .max_pos = 128, .kohms = 100, }, + [MCP464x_502] = { .wipers = 2, .max_pos = 128, .kohms = 5, }, + [MCP464x_103] = { .wipers = 2, .max_pos = 128, .kohms = 10, }, + [MCP464x_503] = { .wipers = 2, .max_pos = 128, .kohms = 50, }, + [MCP464x_104] = { .wipers = 2, .max_pos = 128, .kohms = 100, }, [MCP465x_502] = { .wipers = 2, .max_pos = 256, .kohms = 5, }, [MCP465x_103] = { .wipers = 2, .max_pos = 256, .kohms = 10, }, [MCP465x_503] = { .wipers = 2, .max_pos = 256, .kohms = 50, }, [MCP465x_104] = { .wipers = 2, .max_pos = 256, .kohms = 100, }, + [MCP466x_502] = { .wipers = 2, .max_pos = 256, .kohms = 5, }, + [MCP466x_103] = { .wipers = 2, .max_pos = 256, .kohms = 10, }, + [MCP466x_503] = { .wipers = 2, .max_pos = 256, .kohms = 50, }, + [MCP466x_104] = { .wipers = 2, .max_pos = 256, .kohms = 100, }, }; #define MCP4531_WRITE (0 << 2) @@ -148,12 +190,89 @@ static const struct iio_info mcp4531_info = { .driver_module = THIS_MODULE, }; +#ifdef CONFIG_OF + +#define MCP4531_COMPATIBLE(of_compatible, cfg) { \ + .compatible = of_compatible, \ + .data = &mcp4531_cfg[cfg], \ +} + +static const struct of_device_id mcp4531_of_match[] = { + MCP4531_COMPATIBLE("microchip,mcp4531-502", MCP453x_502), + MCP4531_COMPATIBLE("microchip,mcp4531-103", MCP453x_103), + MCP4531_COMPATIBLE("microchip,mcp4531-503", MCP453x_503), + MCP4531_COMPATIBLE("microchip,mcp4531-104", MCP453x_104), + MCP4531_COMPATIBLE("microchip,mcp4532-502", MCP453x_502), + MCP4531_COMPATIBLE("microchip,mcp4532-103", MCP453x_103), + MCP4531_COMPATIBLE("microchip,mcp4532-503", MCP453x_503), + MCP4531_COMPATIBLE("microchip,mcp4532-104", MCP453x_104), + MCP4531_COMPATIBLE("microchip,mcp4541-502", MCP454x_502), + MCP4531_COMPATIBLE("microchip,mcp4541-103", MCP454x_103), + MCP4531_COMPATIBLE("microchip,mcp4541-503", MCP454x_503), + MCP4531_COMPATIBLE("microchip,mcp4541-104", MCP454x_104), + MCP4531_COMPATIBLE("microchip,mcp4542-502", MCP454x_502), + MCP4531_COMPATIBLE("microchip,mcp4542-103", MCP454x_103), + MCP4531_COMPATIBLE("microchip,mcp4542-503", MCP454x_503), + MCP4531_COMPATIBLE("microchip,mcp4542-104", MCP454x_104), + MCP4531_COMPATIBLE("microchip,mcp4551-502", MCP455x_502), + MCP4531_COMPATIBLE("microchip,mcp4551-103", MCP455x_103), + MCP4531_COMPATIBLE("microchip,mcp4551-503", MCP455x_503), + MCP4531_COMPATIBLE("microchip,mcp4551-104", MCP455x_104), + MCP4531_COMPATIBLE("microchip,mcp4552-502", MCP455x_502), + MCP4531_COMPATIBLE("microchip,mcp4552-103", MCP455x_103), + MCP4531_COMPATIBLE("microchip,mcp4552-503", MCP455x_503), + MCP4531_COMPATIBLE("microchip,mcp4552-104", MCP455x_104), + MCP4531_COMPATIBLE("microchip,mcp4561-502", MCP456x_502), + MCP4531_COMPATIBLE("microchip,mcp4561-103", MCP456x_103), + MCP4531_COMPATIBLE("microchip,mcp4561-503", MCP456x_503), + MCP4531_COMPATIBLE("microchip,mcp4561-104", MCP456x_104), + MCP4531_COMPATIBLE("microchip,mcp4562-502", MCP456x_502), + MCP4531_COMPATIBLE("microchip,mcp4562-103", MCP456x_103), + MCP4531_COMPATIBLE("microchip,mcp4562-503", MCP456x_503), + MCP4531_COMPATIBLE("microchip,mcp4562-104", MCP456x_104), + MCP4531_COMPATIBLE("microchip,mcp4631-502", MCP463x_502), + MCP4531_COMPATIBLE("microchip,mcp4631-103", MCP463x_103), + MCP4531_COMPATIBLE("microchip,mcp4631-503", MCP463x_503), + MCP4531_COMPATIBLE("microchip,mcp4631-104", MCP463x_104), + MCP4531_COMPATIBLE("microchip,mcp4632-502", MCP463x_502), + MCP4531_COMPATIBLE("microchip,mcp4632-103", MCP463x_103), + MCP4531_COMPATIBLE("microchip,mcp4632-503", MCP463x_503), + MCP4531_COMPATIBLE("microchip,mcp4632-104", MCP463x_104), + MCP4531_COMPATIBLE("microchip,mcp4641-502", MCP464x_502), + MCP4531_COMPATIBLE("microchip,mcp4641-103", MCP464x_103), + MCP4531_COMPATIBLE("microchip,mcp4641-503", MCP464x_503), + MCP4531_COMPATIBLE("microchip,mcp4641-104", MCP464x_104), + MCP4531_COMPATIBLE("microchip,mcp4642-502", MCP464x_502), + MCP4531_COMPATIBLE("microchip,mcp4642-103", MCP464x_103), + MCP4531_COMPATIBLE("microchip,mcp4642-503", MCP464x_503), + MCP4531_COMPATIBLE("microchip,mcp4642-104", MCP464x_104), + MCP4531_COMPATIBLE("microchip,mcp4651-502", MCP465x_502), + MCP4531_COMPATIBLE("microchip,mcp4651-103", MCP465x_103), + MCP4531_COMPATIBLE("microchip,mcp4651-503", MCP465x_503), + MCP4531_COMPATIBLE("microchip,mcp4651-104", MCP465x_104), + MCP4531_COMPATIBLE("microchip,mcp4652-502", MCP465x_502), + MCP4531_COMPATIBLE("microchip,mcp4652-103", MCP465x_103), + MCP4531_COMPATIBLE("microchip,mcp4652-503", MCP465x_503), + MCP4531_COMPATIBLE("microchip,mcp4652-104", MCP465x_104), + MCP4531_COMPATIBLE("microchip,mcp4661-502", MCP466x_502), + MCP4531_COMPATIBLE("microchip,mcp4661-103", MCP466x_103), + MCP4531_COMPATIBLE("microchip,mcp4661-503", MCP466x_503), + MCP4531_COMPATIBLE("microchip,mcp4661-104", MCP466x_104), + MCP4531_COMPATIBLE("microchip,mcp4662-502", MCP466x_502), + MCP4531_COMPATIBLE("microchip,mcp4662-103", MCP466x_103), + MCP4531_COMPATIBLE("microchip,mcp4662-503", MCP466x_503), + MCP4531_COMPATIBLE("microchip,mcp4662-104", MCP466x_104), + { /* sentinel */ } +}; +#endif + static int mcp4531_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct device *dev = &client->dev; struct mcp4531_data *data; struct iio_dev *indio_dev; + const struct of_device_id *match; if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WORD_DATA)) { @@ -167,7 +286,12 @@ static int mcp4531_probe(struct i2c_client *client, data = iio_priv(indio_dev); i2c_set_clientdata(client, indio_dev); data->client = client; - data->cfg = &mcp4531_cfg[id->driver_data]; + + match = of_match_device(of_match_ptr(mcp4531_of_match), dev); + if (match) + data->cfg = of_device_get_match_data(dev); + else + data->cfg = &mcp4531_cfg[id->driver_data]; indio_dev->dev.parent = dev; indio_dev->info = &mcp4531_info; @@ -187,6 +311,14 @@ static const struct i2c_device_id mcp4531_id[] = { { "mcp4532-103", MCP453x_103 }, { "mcp4532-503", MCP453x_503 }, { "mcp4532-104", MCP453x_104 }, + { "mcp4541-502", MCP454x_502 }, + { "mcp4541-103", MCP454x_103 }, + { "mcp4541-503", MCP454x_503 }, + { "mcp4541-104", MCP454x_104 }, + { "mcp4542-502", MCP454x_502 }, + { "mcp4542-103", MCP454x_103 }, + { "mcp4542-503", MCP454x_503 }, + { "mcp4542-104", MCP454x_104 }, { "mcp4551-502", MCP455x_502 }, { "mcp4551-103", MCP455x_103 }, { "mcp4551-503", MCP455x_503 }, @@ -195,6 +327,14 @@ static const struct i2c_device_id mcp4531_id[] = { { "mcp4552-103", MCP455x_103 }, { "mcp4552-503", MCP455x_503 }, { "mcp4552-104", MCP455x_104 }, + { "mcp4561-502", MCP456x_502 }, + { "mcp4561-103", MCP456x_103 }, + { "mcp4561-503", MCP456x_503 }, + { "mcp4561-104", MCP456x_104 }, + { "mcp4562-502", MCP456x_502 }, + { "mcp4562-103", MCP456x_103 }, + { "mcp4562-503", MCP456x_503 }, + { "mcp4562-104", MCP456x_104 }, { "mcp4631-502", MCP463x_502 }, { "mcp4631-103", MCP463x_103 }, { "mcp4631-503", MCP463x_503 }, @@ -203,6 +343,14 @@ static const struct i2c_device_id mcp4531_id[] = { { "mcp4632-103", MCP463x_103 }, { "mcp4632-503", MCP463x_503 }, { "mcp4632-104", MCP463x_104 }, + { "mcp4641-502", MCP464x_502 }, + { "mcp4641-103", MCP464x_103 }, + { "mcp4641-503", MCP464x_503 }, + { "mcp4641-104", MCP464x_104 }, + { "mcp4642-502", MCP464x_502 }, + { "mcp4642-103", MCP464x_103 }, + { "mcp4642-503", MCP464x_503 }, + { "mcp4642-104", MCP464x_104 }, { "mcp4651-502", MCP465x_502 }, { "mcp4651-103", MCP465x_103 }, { "mcp4651-503", MCP465x_503 }, @@ -211,6 +359,14 @@ static const struct i2c_device_id mcp4531_id[] = { { "mcp4652-103", MCP465x_103 }, { "mcp4652-503", MCP465x_503 }, { "mcp4652-104", MCP465x_104 }, + { "mcp4661-502", MCP466x_502 }, + { "mcp4661-103", MCP466x_103 }, + { "mcp4661-503", MCP466x_503 }, + { "mcp4661-104", MCP466x_104 }, + { "mcp4662-502", MCP466x_502 }, + { "mcp4662-103", MCP466x_103 }, + { "mcp4662-503", MCP466x_503 }, + { "mcp4662-104", MCP466x_104 }, {} }; MODULE_DEVICE_TABLE(i2c, mcp4531_id); @@ -218,6 +374,7 @@ MODULE_DEVICE_TABLE(i2c, mcp4531_id); static struct i2c_driver mcp4531_driver = { .driver = { .name = "mcp4531", + .of_match_table = of_match_ptr(mcp4531_of_match), }, .probe = mcp4531_probe, .id_table = mcp4531_id, diff --git a/drivers/iio/potentiometer/tpl0102.c b/drivers/iio/potentiometer/tpl0102.c index 5c304d42d713..7b6b54531ea2 100644 --- a/drivers/iio/potentiometer/tpl0102.c +++ b/drivers/iio/potentiometer/tpl0102.c @@ -116,10 +116,6 @@ static int tpl0102_probe(struct i2c_client *client, struct tpl0102_data *data; struct iio_dev *indio_dev; - if (!i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_WORD_DATA)) - return -EOPNOTSUPP; - indio_dev = devm_iio_device_alloc(dev, sizeof(*data)); if (!indio_dev) return -ENOMEM; diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig index cda9f128f3a4..d130cdc78f43 100644 --- a/drivers/iio/pressure/Kconfig +++ b/drivers/iio/pressure/Kconfig @@ -6,16 +6,33 @@ menu "Pressure sensors" config BMP280 - tristate "Bosch Sensortec BMP180 and BMP280 pressure sensor driver" - depends on I2C + tristate "Bosch Sensortec BMP180/BMP280 pressure sensor I2C driver" + depends on (I2C || SPI_MASTER) depends on !(BMP085_I2C=y || BMP085_I2C=m) - select REGMAP_I2C + depends on !(BMP085_SPI=y || BMP085_SPI=m) + select REGMAP + select BMP280_I2C if (I2C) + select BMP280_SPI if (SPI_MASTER) help Say yes here to build support for Bosch Sensortec BMP180 and BMP280 - pressure and temperature sensors. + pressure and temperature sensors. Also supports the BE280 with + an additional humidity sensor channel. - To compile this driver as a module, choose M here: the module - will be called bmp280. + To compile this driver as a module, choose M here: the core module + will be called bmp280 and you will also get bmp280-i2c for I2C + and/or bmp280-spi for SPI support. + +config BMP280_I2C + tristate + depends on BMP280 + depends on I2C + select REGMAP_I2C + +config BMP280_SPI + tristate + depends on BMP280 + depends on SPI_MASTER + select REGMAP config HID_SENSOR_PRESS depends on HID_SENSOR_HUB @@ -130,7 +147,7 @@ config IIO_ST_PRESS select IIO_TRIGGERED_BUFFER if (IIO_BUFFER) help Say yes here to build support for STMicroelectronics pressure - sensors: LPS001WP, LPS25H, LPS331AP. + sensors: LPS001WP, LPS25H, LPS331AP, LPS22HB. This driver can also be built as a module. If so, these modules will be created: diff --git a/drivers/iio/pressure/Makefile b/drivers/iio/pressure/Makefile index 17d6e7afa1ff..7f395bed5e88 100644 --- a/drivers/iio/pressure/Makefile +++ b/drivers/iio/pressure/Makefile @@ -4,6 +4,9 @@ # When adding new entries keep the list in alphabetical order obj-$(CONFIG_BMP280) += bmp280.o +bmp280-objs := bmp280-core.o bmp280-regmap.o +obj-$(CONFIG_BMP280_I2C) += bmp280-i2c.o +obj-$(CONFIG_BMP280_SPI) += bmp280-spi.o obj-$(CONFIG_HID_SENSOR_PRESS) += hid-sensor-press.o obj-$(CONFIG_HP03) += hp03.o obj-$(CONFIG_MPL115) += mpl115.o diff --git a/drivers/iio/pressure/bmp280.c b/drivers/iio/pressure/bmp280-core.c similarity index 58% rename from drivers/iio/pressure/bmp280.c rename to drivers/iio/pressure/bmp280-core.c index 724452d61846..6943688e66df 100644 --- a/drivers/iio/pressure/bmp280.c +++ b/drivers/iio/pressure/bmp280-core.c @@ -1,5 +1,9 @@ /* + * Copyright (c) 2010 Christoph Mair + * Copyright (c) 2012 Bosch Sensortec GmbH + * Copyright (c) 2012 Unixphere AB * Copyright (c) 2014 Intel Corporation + * Copyright (c) 2016 Linus Walleij * * Driver for Bosch Sensortec BMP180 and BMP280 digital pressure sensor. * @@ -10,99 +14,63 @@ * Datasheet: * https://ae-bst.resource.bosch.com/media/_tech/media/datasheets/BST-BMP180-DS000-121.pdf * https://ae-bst.resource.bosch.com/media/_tech/media/datasheets/BST-BMP280-DS001-12.pdf + * https://ae-bst.resource.bosch.com/media/_tech/media/datasheets/BST-BME280_DS001-11.pdf */ #define pr_fmt(fmt) "bmp280: " fmt +#include #include -#include -#include #include #include #include #include +#include +#include +#include +#include /* For irq_get_irq_data() */ +#include +#include +#include -/* BMP280 specific registers */ -#define BMP280_REG_TEMP_XLSB 0xFC -#define BMP280_REG_TEMP_LSB 0xFB -#define BMP280_REG_TEMP_MSB 0xFA -#define BMP280_REG_PRESS_XLSB 0xF9 -#define BMP280_REG_PRESS_LSB 0xF8 -#define BMP280_REG_PRESS_MSB 0xF7 - -#define BMP280_REG_CONFIG 0xF5 -#define BMP280_REG_STATUS 0xF3 - -#define BMP280_REG_COMP_TEMP_START 0x88 -#define BMP280_COMP_TEMP_REG_COUNT 6 - -#define BMP280_REG_COMP_PRESS_START 0x8E -#define BMP280_COMP_PRESS_REG_COUNT 18 - -#define BMP280_FILTER_MASK (BIT(4) | BIT(3) | BIT(2)) -#define BMP280_FILTER_OFF 0 -#define BMP280_FILTER_2X BIT(2) -#define BMP280_FILTER_4X BIT(3) -#define BMP280_FILTER_8X (BIT(3) | BIT(2)) -#define BMP280_FILTER_16X BIT(4) - -#define BMP280_OSRS_TEMP_MASK (BIT(7) | BIT(6) | BIT(5)) -#define BMP280_OSRS_TEMP_SKIP 0 -#define BMP280_OSRS_TEMP_X(osrs_t) ((osrs_t) << 5) -#define BMP280_OSRS_TEMP_1X BMP280_OSRS_TEMP_X(1) -#define BMP280_OSRS_TEMP_2X BMP280_OSRS_TEMP_X(2) -#define BMP280_OSRS_TEMP_4X BMP280_OSRS_TEMP_X(3) -#define BMP280_OSRS_TEMP_8X BMP280_OSRS_TEMP_X(4) -#define BMP280_OSRS_TEMP_16X BMP280_OSRS_TEMP_X(5) - -#define BMP280_OSRS_PRESS_MASK (BIT(4) | BIT(3) | BIT(2)) -#define BMP280_OSRS_PRESS_SKIP 0 -#define BMP280_OSRS_PRESS_X(osrs_p) ((osrs_p) << 2) -#define BMP280_OSRS_PRESS_1X BMP280_OSRS_PRESS_X(1) -#define BMP280_OSRS_PRESS_2X BMP280_OSRS_PRESS_X(2) -#define BMP280_OSRS_PRESS_4X BMP280_OSRS_PRESS_X(3) -#define BMP280_OSRS_PRESS_8X BMP280_OSRS_PRESS_X(4) -#define BMP280_OSRS_PRESS_16X BMP280_OSRS_PRESS_X(5) - -#define BMP280_MODE_MASK (BIT(1) | BIT(0)) -#define BMP280_MODE_SLEEP 0 -#define BMP280_MODE_FORCED BIT(0) -#define BMP280_MODE_NORMAL (BIT(1) | BIT(0)) - -/* BMP180 specific registers */ -#define BMP180_REG_OUT_XLSB 0xF8 -#define BMP180_REG_OUT_LSB 0xF7 -#define BMP180_REG_OUT_MSB 0xF6 - -#define BMP180_REG_CALIB_START 0xAA -#define BMP180_REG_CALIB_COUNT 22 - -#define BMP180_MEAS_SCO BIT(5) -#define BMP180_MEAS_TEMP (0x0E | BMP180_MEAS_SCO) -#define BMP180_MEAS_PRESS_X(oss) ((oss) << 6 | 0x14 | BMP180_MEAS_SCO) -#define BMP180_MEAS_PRESS_1X BMP180_MEAS_PRESS_X(0) -#define BMP180_MEAS_PRESS_2X BMP180_MEAS_PRESS_X(1) -#define BMP180_MEAS_PRESS_4X BMP180_MEAS_PRESS_X(2) -#define BMP180_MEAS_PRESS_8X BMP180_MEAS_PRESS_X(3) - -/* BMP180 and BMP280 common registers */ -#define BMP280_REG_CTRL_MEAS 0xF4 -#define BMP280_REG_RESET 0xE0 -#define BMP280_REG_ID 0xD0 - -#define BMP180_CHIP_ID 0x55 -#define BMP280_CHIP_ID 0x58 -#define BMP280_SOFT_RESET_VAL 0xB6 +#include "bmp280.h" + +/* + * These enums are used for indexing into the array of calibration + * coefficients for BMP180. + */ +enum { AC1, AC2, AC3, AC4, AC5, AC6, B1, B2, MB, MC, MD }; + +struct bmp180_calib { + s16 AC1; + s16 AC2; + s16 AC3; + u16 AC4; + u16 AC5; + u16 AC6; + s16 B1; + s16 B2; + s16 MB; + s16 MC; + s16 MD; +}; struct bmp280_data { - struct i2c_client *client; + struct device *dev; struct mutex lock; struct regmap *regmap; + struct completion done; + bool use_eoc; const struct bmp280_chip_info *chip_info; + struct bmp180_calib calib; + struct regulator *vddd; + struct regulator *vdda; + unsigned int start_up_time; /* in milliseconds */ /* log of base 2 of oversampling rate */ u8 oversampling_press; u8 oversampling_temp; + u8 oversampling_humid; /* * Carryover value from temperature conversion, used in pressure @@ -112,17 +80,19 @@ struct bmp280_data { }; struct bmp280_chip_info { - const struct regmap_config *regmap_config; - const int *oversampling_temp_avail; int num_oversampling_temp_avail; const int *oversampling_press_avail; int num_oversampling_press_avail; + const int *oversampling_humid_avail; + int num_oversampling_humid_avail; + int (*chip_config)(struct bmp280_data *); int (*read_temp)(struct bmp280_data *, int *); int (*read_press)(struct bmp280_data *, int *, int *); + int (*read_humid)(struct bmp280_data *, int *, int *); }; /* @@ -143,45 +113,75 @@ static const struct iio_chan_spec bmp280_channels[] = { .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), }, + { + .type = IIO_HUMIDITYRELATIVE, + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) | + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + }, }; -static bool bmp280_is_writeable_reg(struct device *dev, unsigned int reg) -{ - switch (reg) { - case BMP280_REG_CONFIG: - case BMP280_REG_CTRL_MEAS: - case BMP280_REG_RESET: - return true; - default: - return false; - }; -} +/* + * Returns humidity in percent, resolution is 0.01 percent. Output value of + * "47445" represents 47445/1024 = 46.333 %RH. + * + * Taken from BME280 datasheet, Section 4.2.3, "Compensation formula". + */ -static bool bmp280_is_volatile_reg(struct device *dev, unsigned int reg) +static u32 bmp280_compensate_humidity(struct bmp280_data *data, + s32 adc_humidity) { - switch (reg) { - case BMP280_REG_TEMP_XLSB: - case BMP280_REG_TEMP_LSB: - case BMP280_REG_TEMP_MSB: - case BMP280_REG_PRESS_XLSB: - case BMP280_REG_PRESS_LSB: - case BMP280_REG_PRESS_MSB: - case BMP280_REG_STATUS: - return true; - default: - return false; + struct device *dev = data->dev; + unsigned int H1, H3, tmp; + int H2, H4, H5, H6, ret, var; + + ret = regmap_read(data->regmap, BMP280_REG_COMP_H1, &H1); + if (ret < 0) { + dev_err(dev, "failed to read H1 comp value\n"); + return ret; } -} -static const struct regmap_config bmp280_regmap_config = { - .reg_bits = 8, - .val_bits = 8, + ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H2, &tmp, 2); + if (ret < 0) { + dev_err(dev, "failed to read H2 comp value\n"); + return ret; + } + H2 = sign_extend32(le16_to_cpu(tmp), 15); + + ret = regmap_read(data->regmap, BMP280_REG_COMP_H3, &H3); + if (ret < 0) { + dev_err(dev, "failed to read H3 comp value\n"); + return ret; + } + + ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H4, &tmp, 2); + if (ret < 0) { + dev_err(dev, "failed to read H4 comp value\n"); + return ret; + } + H4 = sign_extend32(((be16_to_cpu(tmp) >> 4) & 0xff0) | + (be16_to_cpu(tmp) & 0xf), 11); + + ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H5, &tmp, 2); + if (ret < 0) { + dev_err(dev, "failed to read H5 comp value\n"); + return ret; + } + H5 = sign_extend32(((le16_to_cpu(tmp) >> 4) & 0xfff), 11); + + ret = regmap_read(data->regmap, BMP280_REG_COMP_H6, &tmp); + if (ret < 0) { + dev_err(dev, "failed to read H6 comp value\n"); + return ret; + } + H6 = sign_extend32(tmp, 7); - .max_register = BMP280_REG_TEMP_XLSB, - .cache_type = REGCACHE_RBTREE, + var = ((s32)data->t_fine) - 76800; + var = ((((adc_humidity << 14) - (H4 << 20) - (H5 * var)) + 16384) >> 15) + * (((((((var * H6) >> 10) * (((var * H3) >> 11) + 32768)) >> 10) + + 2097152) * H2 + 8192) >> 14); + var -= ((((var >> 15) * (var >> 15)) >> 7) * H1) >> 4; - .writeable_reg = bmp280_is_writeable_reg, - .volatile_reg = bmp280_is_volatile_reg, + return var >> 12; }; /* @@ -201,7 +201,7 @@ static s32 bmp280_compensate_temp(struct bmp280_data *data, ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_TEMP_START, buf, BMP280_COMP_TEMP_REG_COUNT); if (ret < 0) { - dev_err(&data->client->dev, + dev_err(data->dev, "failed to read temperature calibration parameters\n"); return ret; } @@ -241,7 +241,7 @@ static u32 bmp280_compensate_press(struct bmp280_data *data, ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_PRESS_START, buf, BMP280_COMP_PRESS_REG_COUNT); if (ret < 0) { - dev_err(&data->client->dev, + dev_err(data->dev, "failed to read pressure calibration parameters\n"); return ret; } @@ -276,7 +276,7 @@ static int bmp280_read_temp(struct bmp280_data *data, ret = regmap_bulk_read(data->regmap, BMP280_REG_TEMP_MSB, (u8 *) &tmp, 3); if (ret < 0) { - dev_err(&data->client->dev, "failed to read temperature\n"); + dev_err(data->dev, "failed to read temperature\n"); return ret; } @@ -311,7 +311,7 @@ static int bmp280_read_press(struct bmp280_data *data, ret = regmap_bulk_read(data->regmap, BMP280_REG_PRESS_MSB, (u8 *) &tmp, 3); if (ret < 0) { - dev_err(&data->client->dev, "failed to read pressure\n"); + dev_err(data->dev, "failed to read pressure\n"); return ret; } @@ -324,6 +324,34 @@ static int bmp280_read_press(struct bmp280_data *data, return IIO_VAL_FRACTIONAL; } +static int bmp280_read_humid(struct bmp280_data *data, int *val, int *val2) +{ + int ret; + __be16 tmp = 0; + s32 adc_humidity; + u32 comp_humidity; + + /* Read and compensate temperature so we get a reading of t_fine. */ + ret = bmp280_read_temp(data, NULL); + if (ret < 0) + return ret; + + ret = regmap_bulk_read(data->regmap, BMP280_REG_HUMIDITY_MSB, + (u8 *) &tmp, 2); + if (ret < 0) { + dev_err(data->dev, "failed to read humidity\n"); + return ret; + } + + adc_humidity = be16_to_cpu(tmp); + comp_humidity = bmp280_compensate_humidity(data, adc_humidity); + + *val = comp_humidity; + *val2 = 1024; + + return IIO_VAL_FRACTIONAL; +} + static int bmp280_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int *val, int *val2, long mask) @@ -331,11 +359,15 @@ static int bmp280_read_raw(struct iio_dev *indio_dev, int ret; struct bmp280_data *data = iio_priv(indio_dev); + pm_runtime_get_sync(data->dev); mutex_lock(&data->lock); switch (mask) { case IIO_CHAN_INFO_PROCESSED: switch (chan->type) { + case IIO_HUMIDITYRELATIVE: + ret = data->chip_info->read_humid(data, val, val2); + break; case IIO_PRESSURE: ret = data->chip_info->read_press(data, val, val2); break; @@ -349,6 +381,10 @@ static int bmp280_read_raw(struct iio_dev *indio_dev, break; case IIO_CHAN_INFO_OVERSAMPLING_RATIO: switch (chan->type) { + case IIO_HUMIDITYRELATIVE: + *val = 1 << data->oversampling_humid; + ret = IIO_VAL_INT; + break; case IIO_PRESSURE: *val = 1 << data->oversampling_press; ret = IIO_VAL_INT; @@ -368,10 +404,29 @@ static int bmp280_read_raw(struct iio_dev *indio_dev, } mutex_unlock(&data->lock); + pm_runtime_mark_last_busy(data->dev); + pm_runtime_put_autosuspend(data->dev); return ret; } +static int bmp280_write_oversampling_ratio_humid(struct bmp280_data *data, + int val) +{ + int i; + const int *avail = data->chip_info->oversampling_humid_avail; + const int n = data->chip_info->num_oversampling_humid_avail; + + for (i = 0; i < n; i++) { + if (avail[i] == val) { + data->oversampling_humid = ilog2(val); + + return data->chip_info->chip_config(data); + } + } + return -EINVAL; +} + static int bmp280_write_oversampling_ratio_temp(struct bmp280_data *data, int val) { @@ -415,8 +470,12 @@ static int bmp280_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + pm_runtime_get_sync(data->dev); mutex_lock(&data->lock); switch (chan->type) { + case IIO_HUMIDITYRELATIVE: + ret = bmp280_write_oversampling_ratio_humid(data, val); + break; case IIO_PRESSURE: ret = bmp280_write_oversampling_ratio_press(data, val); break; @@ -428,6 +487,8 @@ static int bmp280_write_raw(struct iio_dev *indio_dev, break; } mutex_unlock(&data->lock); + pm_runtime_mark_last_busy(data->dev); + pm_runtime_put_autosuspend(data->dev); break; default: return -EINVAL; @@ -502,7 +563,7 @@ static int bmp280_chip_config(struct bmp280_data *data) BMP280_MODE_MASK, osrs | BMP280_MODE_NORMAL); if (ret < 0) { - dev_err(&data->client->dev, + dev_err(data->dev, "failed to write ctrl_meas register\n"); return ret; } @@ -511,7 +572,7 @@ static int bmp280_chip_config(struct bmp280_data *data) BMP280_FILTER_MASK, BMP280_FILTER_4X); if (ret < 0) { - dev_err(&data->client->dev, + dev_err(data->dev, "failed to write config register\n"); return ret; } @@ -522,8 +583,6 @@ static int bmp280_chip_config(struct bmp280_data *data) static const int bmp280_oversampling_avail[] = { 1, 2, 4, 8, 16 }; static const struct bmp280_chip_info bmp280_chip_info = { - .regmap_config = &bmp280_regmap_config, - .oversampling_temp_avail = bmp280_oversampling_avail, .num_oversampling_temp_avail = ARRAY_SIZE(bmp280_oversampling_avail), @@ -535,39 +594,32 @@ static const struct bmp280_chip_info bmp280_chip_info = { .read_press = bmp280_read_press, }; -static bool bmp180_is_writeable_reg(struct device *dev, unsigned int reg) +static int bme280_chip_config(struct bmp280_data *data) { - switch (reg) { - case BMP280_REG_CTRL_MEAS: - case BMP280_REG_RESET: - return true; - default: - return false; - }; -} + int ret = bmp280_chip_config(data); + u8 osrs = BMP280_OSRS_HUMIDITIY_X(data->oversampling_humid + 1); -static bool bmp180_is_volatile_reg(struct device *dev, unsigned int reg) -{ - switch (reg) { - case BMP180_REG_OUT_XLSB: - case BMP180_REG_OUT_LSB: - case BMP180_REG_OUT_MSB: - case BMP280_REG_CTRL_MEAS: - return true; - default: - return false; - } + if (ret < 0) + return ret; + + return regmap_update_bits(data->regmap, BMP280_REG_CTRL_HUMIDITY, + BMP280_OSRS_HUMIDITY_MASK, osrs); } -static const struct regmap_config bmp180_regmap_config = { - .reg_bits = 8, - .val_bits = 8, +static const struct bmp280_chip_info bme280_chip_info = { + .oversampling_temp_avail = bmp280_oversampling_avail, + .num_oversampling_temp_avail = ARRAY_SIZE(bmp280_oversampling_avail), + + .oversampling_press_avail = bmp280_oversampling_avail, + .num_oversampling_press_avail = ARRAY_SIZE(bmp280_oversampling_avail), - .max_register = BMP180_REG_OUT_XLSB, - .cache_type = REGCACHE_RBTREE, + .oversampling_humid_avail = bmp280_oversampling_avail, + .num_oversampling_humid_avail = ARRAY_SIZE(bmp280_oversampling_avail), - .writeable_reg = bmp180_is_writeable_reg, - .volatile_reg = bmp180_is_volatile_reg, + .chip_config = bme280_chip_config, + .read_temp = bmp280_read_temp, + .read_press = bmp280_read_press, + .read_humid = bmp280_read_humid, }; static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas) @@ -577,16 +629,32 @@ static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas) unsigned int delay_us; unsigned int ctrl; + if (data->use_eoc) + init_completion(&data->done); + ret = regmap_write(data->regmap, BMP280_REG_CTRL_MEAS, ctrl_meas); if (ret) return ret; - if (ctrl_meas == BMP180_MEAS_TEMP) - delay_us = 4500; - else - delay_us = conversion_time_max[data->oversampling_press]; - - usleep_range(delay_us, delay_us + 1000); + if (data->use_eoc) { + /* + * If we have a completion interrupt, use it, wait up to + * 100ms. The longest conversion time listed is 76.5 ms for + * advanced resolution mode. + */ + ret = wait_for_completion_timeout(&data->done, + 1 + msecs_to_jiffies(100)); + if (!ret) + dev_err(data->dev, "timeout waiting for completion\n"); + } else { + if (ctrl_meas == BMP180_MEAS_TEMP) + delay_us = 4500; + else + delay_us = + conversion_time_max[data->oversampling_press]; + + usleep_range(delay_us, delay_us + 1000); + } ret = regmap_read(data->regmap, BMP280_REG_CTRL_MEAS, &ctrl); if (ret) @@ -617,26 +685,6 @@ static int bmp180_read_adc_temp(struct bmp280_data *data, int *val) return 0; } -/* - * These enums are used for indexing into the array of calibration - * coefficients for BMP180. - */ -enum { AC1, AC2, AC3, AC4, AC5, AC6, B1, B2, MB, MC, MD }; - -struct bmp180_calib { - s16 AC1; - s16 AC2; - s16 AC3; - u16 AC4; - u16 AC5; - u16 AC6; - s16 B1; - s16 B2; - s16 MB; - s16 MC; - s16 MD; -}; - static int bmp180_read_calib(struct bmp280_data *data, struct bmp180_calib *calib) { @@ -656,6 +704,9 @@ static int bmp180_read_calib(struct bmp280_data *data, return -EIO; } + /* Toss the calibration data into the entropy pool */ + add_device_randomness(buf, sizeof(buf)); + calib->AC1 = be16_to_cpu(buf[AC1]); calib->AC2 = be16_to_cpu(buf[AC2]); calib->AC3 = be16_to_cpu(buf[AC3]); @@ -679,19 +730,11 @@ static int bmp180_read_calib(struct bmp280_data *data, */ static s32 bmp180_compensate_temp(struct bmp280_data *data, s32 adc_temp) { - int ret; s32 x1, x2; - struct bmp180_calib calib; + struct bmp180_calib *calib = &data->calib; - ret = bmp180_read_calib(data, &calib); - if (ret < 0) { - dev_err(&data->client->dev, - "failed to read calibration coefficients\n"); - return ret; - } - - x1 = ((adc_temp - calib.AC6) * calib.AC5) >> 15; - x2 = (calib.MC << 11) / (x1 + calib.MD); + x1 = ((adc_temp - calib->AC6) * calib->AC5) >> 15; + x2 = (calib->MC << 11) / (x1 + calib->MD); data->t_fine = x1 + x2; return (data->t_fine + 8) >> 4; @@ -746,29 +789,21 @@ static int bmp180_read_adc_press(struct bmp280_data *data, int *val) */ static u32 bmp180_compensate_press(struct bmp280_data *data, s32 adc_press) { - int ret; s32 x1, x2, x3, p; s32 b3, b6; u32 b4, b7; s32 oss = data->oversampling_press; - struct bmp180_calib calib; - - ret = bmp180_read_calib(data, &calib); - if (ret < 0) { - dev_err(&data->client->dev, - "failed to read calibration coefficients\n"); - return ret; - } + struct bmp180_calib *calib = &data->calib; b6 = data->t_fine - 4000; - x1 = (calib.B2 * (b6 * b6 >> 12)) >> 11; - x2 = calib.AC2 * b6 >> 11; + x1 = (calib->B2 * (b6 * b6 >> 12)) >> 11; + x2 = calib->AC2 * b6 >> 11; x3 = x1 + x2; - b3 = ((((s32)calib.AC1 * 4 + x3) << oss) + 2) / 4; - x1 = calib.AC3 * b6 >> 13; - x2 = (calib.B1 * ((b6 * b6) >> 12)) >> 16; + b3 = ((((s32)calib->AC1 * 4 + x3) << oss) + 2) / 4; + x1 = calib->AC3 * b6 >> 13; + x2 = (calib->B1 * ((b6 * b6) >> 12)) >> 16; x3 = (x1 + x2 + 2) >> 2; - b4 = calib.AC4 * (u32)(x3 + 32768) >> 15; + b4 = calib->AC4 * (u32)(x3 + 32768) >> 15; b7 = ((u32)adc_press - b3) * (50000 >> oss); if (b7 < 0x80000000) p = (b7 * 2) / b4; @@ -815,8 +850,6 @@ static const int bmp180_oversampling_temp_avail[] = { 1 }; static const int bmp180_oversampling_press_avail[] = { 1, 2, 4, 8 }; static const struct bmp280_chip_info bmp180_chip_info = { - .regmap_config = &bmp180_regmap_config, - .oversampling_temp_avail = bmp180_oversampling_temp_avail, .num_oversampling_temp_avail = ARRAY_SIZE(bmp180_oversampling_temp_avail), @@ -830,92 +863,254 @@ static const struct bmp280_chip_info bmp180_chip_info = { .read_press = bmp180_read_press, }; -static int bmp280_probe(struct i2c_client *client, - const struct i2c_device_id *id) +static irqreturn_t bmp085_eoc_irq(int irq, void *d) +{ + struct bmp280_data *data = d; + + complete(&data->done); + + return IRQ_HANDLED; +} + +static int bmp085_fetch_eoc_irq(struct device *dev, + const char *name, + int irq, + struct bmp280_data *data) +{ + unsigned long irq_trig; + int ret; + + irq_trig = irqd_get_trigger_type(irq_get_irq_data(irq)); + if (irq_trig != IRQF_TRIGGER_RISING) { + dev_err(dev, "non-rising trigger given for EOC interrupt, " + "trying to enforce it\n"); + irq_trig = IRQF_TRIGGER_RISING; + } + ret = devm_request_threaded_irq(dev, + irq, + bmp085_eoc_irq, + NULL, + irq_trig, + name, + data); + if (ret) { + /* Bail out without IRQ but keep the driver in place */ + dev_err(dev, "unable to request DRDY IRQ\n"); + return 0; + } + + data->use_eoc = true; + return 0; +} + +int bmp280_common_probe(struct device *dev, + struct regmap *regmap, + unsigned int chip, + const char *name, + int irq) { int ret; struct iio_dev *indio_dev; struct bmp280_data *data; unsigned int chip_id; + struct gpio_desc *gpiod; - indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data)); + indio_dev = devm_iio_device_alloc(dev, sizeof(*data)); if (!indio_dev) return -ENOMEM; data = iio_priv(indio_dev); mutex_init(&data->lock); - data->client = client; + data->dev = dev; - indio_dev->dev.parent = &client->dev; - indio_dev->name = id->name; + indio_dev->dev.parent = dev; + indio_dev->name = name; indio_dev->channels = bmp280_channels; - indio_dev->num_channels = ARRAY_SIZE(bmp280_channels); indio_dev->info = &bmp280_info; indio_dev->modes = INDIO_DIRECT_MODE; - switch (id->driver_data) { + switch (chip) { case BMP180_CHIP_ID: + indio_dev->num_channels = 2; data->chip_info = &bmp180_chip_info; data->oversampling_press = ilog2(8); data->oversampling_temp = ilog2(1); + data->start_up_time = 10; break; case BMP280_CHIP_ID: + indio_dev->num_channels = 2; data->chip_info = &bmp280_chip_info; data->oversampling_press = ilog2(16); data->oversampling_temp = ilog2(2); + data->start_up_time = 2; + break; + case BME280_CHIP_ID: + indio_dev->num_channels = 3; + data->chip_info = &bme280_chip_info; + data->oversampling_press = ilog2(16); + data->oversampling_humid = ilog2(16); + data->oversampling_temp = ilog2(2); + data->start_up_time = 2; break; default: return -EINVAL; } - data->regmap = devm_regmap_init_i2c(client, - data->chip_info->regmap_config); - if (IS_ERR(data->regmap)) { - dev_err(&client->dev, "failed to allocate register map\n"); - return PTR_ERR(data->regmap); + /* Bring up regulators */ + data->vddd = devm_regulator_get(dev, "vddd"); + if (IS_ERR(data->vddd)) { + dev_err(dev, "failed to get VDDD regulator\n"); + return PTR_ERR(data->vddd); + } + ret = regulator_enable(data->vddd); + if (ret) { + dev_err(dev, "failed to enable VDDD regulator\n"); + return ret; + } + data->vdda = devm_regulator_get(dev, "vdda"); + if (IS_ERR(data->vdda)) { + dev_err(dev, "failed to get VDDA regulator\n"); + ret = PTR_ERR(data->vddd); + goto out_disable_vddd; + } + ret = regulator_enable(data->vdda); + if (ret) { + dev_err(dev, "failed to enable VDDA regulator\n"); + goto out_disable_vddd; + } + /* Wait to make sure we started up properly */ + mdelay(data->start_up_time); + + /* Bring chip out of reset if there is an assigned GPIO line */ + gpiod = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH); + /* Deassert the signal */ + if (!IS_ERR(gpiod)) { + dev_info(dev, "release reset\n"); + gpiod_set_value(gpiod, 0); } - ret = regmap_read(data->regmap, BMP280_REG_ID, &chip_id); + data->regmap = regmap; + ret = regmap_read(regmap, BMP280_REG_ID, &chip_id); if (ret < 0) - return ret; - if (chip_id != id->driver_data) { - dev_err(&client->dev, "bad chip id. expected %lx got %x\n", - id->driver_data, chip_id); - return -EINVAL; + goto out_disable_vdda; + if (chip_id != chip) { + dev_err(dev, "bad chip id: expected %x got %x\n", + chip, chip_id); + ret = -EINVAL; + goto out_disable_vdda; } ret = data->chip_info->chip_config(data); if (ret < 0) - return ret; + goto out_disable_vdda; + + dev_set_drvdata(dev, indio_dev); + + /* + * The BMP085 and BMP180 has calibration in an E2PROM, read it out + * at probe time. It will not change. + */ + if (chip_id == BMP180_CHIP_ID) { + ret = bmp180_read_calib(data, &data->calib); + if (ret < 0) { + dev_err(data->dev, + "failed to read calibration coefficients\n"); + goto out_disable_vdda; + } + } + + /* + * Attempt to grab an optional EOC IRQ - only the BMP085 has this + * however as it happens, the BMP085 shares the chip ID of BMP180 + * so we look for an IRQ if we have that. + */ + if (irq > 0 || (chip_id == BMP180_CHIP_ID)) { + ret = bmp085_fetch_eoc_irq(dev, name, irq, data); + if (ret) + goto out_disable_vdda; + } + + /* Enable runtime PM */ + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + /* + * Set autosuspend to two orders of magnitude larger than the + * start-up time. + */ + pm_runtime_set_autosuspend_delay(dev, data->start_up_time *100); + pm_runtime_use_autosuspend(dev); + pm_runtime_put(dev); + + ret = iio_device_register(indio_dev); + if (ret) + goto out_runtime_pm_disable; + - return devm_iio_device_register(&client->dev, indio_dev); + return 0; + +out_runtime_pm_disable: + pm_runtime_get_sync(data->dev); + pm_runtime_put_noidle(data->dev); + pm_runtime_disable(data->dev); +out_disable_vdda: + regulator_disable(data->vdda); +out_disable_vddd: + regulator_disable(data->vddd); + return ret; } +EXPORT_SYMBOL(bmp280_common_probe); -static const struct acpi_device_id bmp280_acpi_match[] = { - {"BMP0280", BMP280_CHIP_ID }, - {"BMP0180", BMP180_CHIP_ID }, - {"BMP0085", BMP180_CHIP_ID }, - { }, -}; -MODULE_DEVICE_TABLE(acpi, bmp280_acpi_match); +int bmp280_common_remove(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct bmp280_data *data = iio_priv(indio_dev); -static const struct i2c_device_id bmp280_id[] = { - {"bmp280", BMP280_CHIP_ID }, - {"bmp180", BMP180_CHIP_ID }, - {"bmp085", BMP180_CHIP_ID }, - { }, -}; -MODULE_DEVICE_TABLE(i2c, bmp280_id); + iio_device_unregister(indio_dev); + pm_runtime_get_sync(data->dev); + pm_runtime_put_noidle(data->dev); + pm_runtime_disable(data->dev); + regulator_disable(data->vdda); + regulator_disable(data->vddd); + return 0; +} +EXPORT_SYMBOL(bmp280_common_remove); -static struct i2c_driver bmp280_driver = { - .driver = { - .name = "bmp280", - .acpi_match_table = ACPI_PTR(bmp280_acpi_match), - }, - .probe = bmp280_probe, - .id_table = bmp280_id, +#ifdef CONFIG_PM +static int bmp280_runtime_suspend(struct device *dev) +{ + struct bmp280_data *data = dev_get_drvdata(dev); + int ret; + + ret = regulator_disable(data->vdda); + if (ret) + return ret; + return regulator_disable(data->vddd); +} + +static int bmp280_runtime_resume(struct device *dev) +{ + struct bmp280_data *data = dev_get_drvdata(dev); + int ret; + + ret = regulator_enable(data->vddd); + if (ret) + return ret; + ret = regulator_enable(data->vdda); + if (ret) + return ret; + msleep(data->start_up_time); + return data->chip_info->chip_config(data); +} +#endif /* CONFIG_PM */ + +const struct dev_pm_ops bmp280_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(bmp280_runtime_suspend, + bmp280_runtime_resume, NULL) }; -module_i2c_driver(bmp280_driver); +EXPORT_SYMBOL(bmp280_dev_pm_ops); MODULE_AUTHOR("Vlad Dogaru "); MODULE_DESCRIPTION("Driver for Bosch Sensortec BMP180/BMP280 pressure and temperature sensor"); diff --git a/drivers/iio/pressure/bmp280-i2c.c b/drivers/iio/pressure/bmp280-i2c.c new file mode 100644 index 000000000000..03742b15b72a --- /dev/null +++ b/drivers/iio/pressure/bmp280-i2c.c @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include + +#include "bmp280.h" + +static int bmp280_i2c_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct regmap *regmap; + const struct regmap_config *regmap_config; + + switch (id->driver_data) { + case BMP180_CHIP_ID: + regmap_config = &bmp180_regmap_config; + break; + case BMP280_CHIP_ID: + case BME280_CHIP_ID: + regmap_config = &bmp280_regmap_config; + break; + default: + return -EINVAL; + } + + regmap = devm_regmap_init_i2c(client, regmap_config); + if (IS_ERR(regmap)) { + dev_err(&client->dev, "failed to allocate register map\n"); + return PTR_ERR(regmap); + } + + return bmp280_common_probe(&client->dev, + regmap, + id->driver_data, + id->name, + client->irq); +} + +static int bmp280_i2c_remove(struct i2c_client *client) +{ + return bmp280_common_remove(&client->dev); +} + +static const struct acpi_device_id bmp280_acpi_i2c_match[] = { + {"BMP0280", BMP280_CHIP_ID }, + {"BMP0180", BMP180_CHIP_ID }, + {"BMP0085", BMP180_CHIP_ID }, + {"BME0280", BME280_CHIP_ID }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, bmp280_acpi_i2c_match); + +#ifdef CONFIG_OF +static const struct of_device_id bmp280_of_i2c_match[] = { + { .compatible = "bosch,bme280", .data = (void *)BME280_CHIP_ID }, + { .compatible = "bosch,bmp280", .data = (void *)BMP280_CHIP_ID }, + { .compatible = "bosch,bmp180", .data = (void *)BMP180_CHIP_ID }, + { .compatible = "bosch,bmp085", .data = (void *)BMP180_CHIP_ID }, + { }, +}; +MODULE_DEVICE_TABLE(of, bmp280_of_i2c_match); +#else +#define bmp280_of_i2c_match NULL +#endif + +static const struct i2c_device_id bmp280_i2c_id[] = { + {"bmp280", BMP280_CHIP_ID }, + {"bmp180", BMP180_CHIP_ID }, + {"bmp085", BMP180_CHIP_ID }, + {"bme280", BME280_CHIP_ID }, + { }, +}; +MODULE_DEVICE_TABLE(i2c, bmp280_i2c_id); + +static struct i2c_driver bmp280_i2c_driver = { + .driver = { + .name = "bmp280", + .acpi_match_table = ACPI_PTR(bmp280_acpi_i2c_match), + .of_match_table = of_match_ptr(bmp280_of_i2c_match), + .pm = &bmp280_dev_pm_ops, + }, + .probe = bmp280_i2c_probe, + .remove = bmp280_i2c_remove, + .id_table = bmp280_i2c_id, +}; +module_i2c_driver(bmp280_i2c_driver); + +MODULE_AUTHOR("Vlad Dogaru "); +MODULE_DESCRIPTION("Driver for Bosch Sensortec BMP180/BMP280 pressure and temperature sensor"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/pressure/bmp280-regmap.c b/drivers/iio/pressure/bmp280-regmap.c new file mode 100644 index 000000000000..6807113ec09f --- /dev/null +++ b/drivers/iio/pressure/bmp280-regmap.c @@ -0,0 +1,84 @@ +#include +#include +#include + +#include "bmp280.h" + +static bool bmp180_is_writeable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case BMP280_REG_CTRL_MEAS: + case BMP280_REG_RESET: + return true; + default: + return false; + }; +} + +static bool bmp180_is_volatile_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case BMP180_REG_OUT_XLSB: + case BMP180_REG_OUT_LSB: + case BMP180_REG_OUT_MSB: + case BMP280_REG_CTRL_MEAS: + return true; + default: + return false; + } +} + +const struct regmap_config bmp180_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + + .max_register = BMP180_REG_OUT_XLSB, + .cache_type = REGCACHE_RBTREE, + + .writeable_reg = bmp180_is_writeable_reg, + .volatile_reg = bmp180_is_volatile_reg, +}; +EXPORT_SYMBOL(bmp180_regmap_config); + +static bool bmp280_is_writeable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case BMP280_REG_CONFIG: + case BMP280_REG_CTRL_HUMIDITY: + case BMP280_REG_CTRL_MEAS: + case BMP280_REG_RESET: + return true; + default: + return false; + }; +} + +static bool bmp280_is_volatile_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case BMP280_REG_HUMIDITY_LSB: + case BMP280_REG_HUMIDITY_MSB: + case BMP280_REG_TEMP_XLSB: + case BMP280_REG_TEMP_LSB: + case BMP280_REG_TEMP_MSB: + case BMP280_REG_PRESS_XLSB: + case BMP280_REG_PRESS_LSB: + case BMP280_REG_PRESS_MSB: + case BMP280_REG_STATUS: + return true; + default: + return false; + } +} + +const struct regmap_config bmp280_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + + .max_register = BMP280_REG_HUMIDITY_LSB, + .cache_type = REGCACHE_RBTREE, + + .writeable_reg = bmp280_is_writeable_reg, + .volatile_reg = bmp280_is_volatile_reg, +}; +EXPORT_SYMBOL(bmp280_regmap_config); diff --git a/drivers/iio/pressure/bmp280-spi.c b/drivers/iio/pressure/bmp280-spi.c new file mode 100644 index 000000000000..17bc95586f9e --- /dev/null +++ b/drivers/iio/pressure/bmp280-spi.c @@ -0,0 +1,125 @@ +/* + * SPI interface for the BMP280 driver + * + * Inspired by the older BMP085 driver drivers/misc/bmp085-spi.c + */ +#include +#include +#include +#include + +#include "bmp280.h" + +static int bmp280_regmap_spi_write(void *context, const void *data, + size_t count) +{ + struct device *dev = context; + struct spi_device *spi = to_spi_device(dev); + u8 buf[2]; + + memcpy(buf, data, 2); + /* + * The SPI register address (= full register address without bit 7) and + * the write command (bit7 = RW = '0') + */ + buf[0] &= ~0x80; + + return spi_write_then_read(spi, buf, 2, NULL, 0); +} + +static int bmp280_regmap_spi_read(void *context, const void *reg, + size_t reg_size, void *val, size_t val_size) +{ + struct device *dev = context; + struct spi_device *spi = to_spi_device(dev); + + return spi_write_then_read(spi, reg, reg_size, val, val_size); +} + +static struct regmap_bus bmp280_regmap_bus = { + .write = bmp280_regmap_spi_write, + .read = bmp280_regmap_spi_read, + .reg_format_endian_default = REGMAP_ENDIAN_BIG, + .val_format_endian_default = REGMAP_ENDIAN_BIG, +}; + +static int bmp280_spi_probe(struct spi_device *spi) +{ + const struct spi_device_id *id = spi_get_device_id(spi); + struct regmap *regmap; + const struct regmap_config *regmap_config; + int ret; + + spi->bits_per_word = 8; + ret = spi_setup(spi); + if (ret < 0) { + dev_err(&spi->dev, "spi_setup failed!\n"); + return ret; + } + + switch (id->driver_data) { + case BMP180_CHIP_ID: + regmap_config = &bmp180_regmap_config; + break; + case BMP280_CHIP_ID: + case BME280_CHIP_ID: + regmap_config = &bmp280_regmap_config; + break; + default: + return -EINVAL; + } + + regmap = devm_regmap_init(&spi->dev, + &bmp280_regmap_bus, + &spi->dev, + regmap_config); + if (IS_ERR(regmap)) { + dev_err(&spi->dev, "failed to allocate register map\n"); + return PTR_ERR(regmap); + } + + return bmp280_common_probe(&spi->dev, + regmap, + id->driver_data, + id->name, + spi->irq); +} + +static int bmp280_spi_remove(struct spi_device *spi) +{ + return bmp280_common_remove(&spi->dev); +} + +static const struct of_device_id bmp280_of_spi_match[] = { + { .compatible = "bosch,bmp085", }, + { .compatible = "bosch,bmp180", }, + { .compatible = "bosch,bmp181", }, + { .compatible = "bosch,bmp280", }, + { .compatible = "bosch,bme280", }, + { }, +}; +MODULE_DEVICE_TABLE(of, bmp280_of_spi_match); + +static const struct spi_device_id bmp280_spi_id[] = { + { "bmp180", BMP180_CHIP_ID }, + { "bmp181", BMP180_CHIP_ID }, + { "bmp280", BMP280_CHIP_ID }, + { "bme280", BME280_CHIP_ID }, + { } +}; +MODULE_DEVICE_TABLE(spi, bmp280_spi_id); + +static struct spi_driver bmp280_spi_driver = { + .driver = { + .name = "bmp280", + .of_match_table = bmp280_of_spi_match, + .pm = &bmp280_dev_pm_ops, + }, + .id_table = bmp280_spi_id, + .probe = bmp280_spi_probe, + .remove = bmp280_spi_remove, +}; +module_spi_driver(bmp280_spi_driver); + +MODULE_DESCRIPTION("BMP280 SPI bus driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/iio/pressure/bmp280.h b/drivers/iio/pressure/bmp280.h new file mode 100644 index 000000000000..2c770e13be0e --- /dev/null +++ b/drivers/iio/pressure/bmp280.h @@ -0,0 +1,112 @@ +#include +#include +#include + +/* BMP280 specific registers */ +#define BMP280_REG_HUMIDITY_LSB 0xFE +#define BMP280_REG_HUMIDITY_MSB 0xFD +#define BMP280_REG_TEMP_XLSB 0xFC +#define BMP280_REG_TEMP_LSB 0xFB +#define BMP280_REG_TEMP_MSB 0xFA +#define BMP280_REG_PRESS_XLSB 0xF9 +#define BMP280_REG_PRESS_LSB 0xF8 +#define BMP280_REG_PRESS_MSB 0xF7 + +#define BMP280_REG_CONFIG 0xF5 +#define BMP280_REG_CTRL_MEAS 0xF4 +#define BMP280_REG_STATUS 0xF3 +#define BMP280_REG_CTRL_HUMIDITY 0xF2 + +/* Due to non linear mapping, and data sizes we can't do a bulk read */ +#define BMP280_REG_COMP_H1 0xA1 +#define BMP280_REG_COMP_H2 0xE1 +#define BMP280_REG_COMP_H3 0xE3 +#define BMP280_REG_COMP_H4 0xE4 +#define BMP280_REG_COMP_H5 0xE5 +#define BMP280_REG_COMP_H6 0xE7 + +#define BMP280_REG_COMP_TEMP_START 0x88 +#define BMP280_COMP_TEMP_REG_COUNT 6 + +#define BMP280_REG_COMP_PRESS_START 0x8E +#define BMP280_COMP_PRESS_REG_COUNT 18 + +#define BMP280_FILTER_MASK (BIT(4) | BIT(3) | BIT(2)) +#define BMP280_FILTER_OFF 0 +#define BMP280_FILTER_2X BIT(2) +#define BMP280_FILTER_4X BIT(3) +#define BMP280_FILTER_8X (BIT(3) | BIT(2)) +#define BMP280_FILTER_16X BIT(4) + +#define BMP280_OSRS_HUMIDITY_MASK (BIT(2) | BIT(1) | BIT(0)) +#define BMP280_OSRS_HUMIDITIY_X(osrs_h) ((osrs_h) << 0) +#define BMP280_OSRS_HUMIDITY_SKIP 0 +#define BMP280_OSRS_HUMIDITY_1X BMP280_OSRS_HUMIDITIY_X(1) +#define BMP280_OSRS_HUMIDITY_2X BMP280_OSRS_HUMIDITIY_X(2) +#define BMP280_OSRS_HUMIDITY_4X BMP280_OSRS_HUMIDITIY_X(3) +#define BMP280_OSRS_HUMIDITY_8X BMP280_OSRS_HUMIDITIY_X(4) +#define BMP280_OSRS_HUMIDITY_16X BMP280_OSRS_HUMIDITIY_X(5) + +#define BMP280_OSRS_TEMP_MASK (BIT(7) | BIT(6) | BIT(5)) +#define BMP280_OSRS_TEMP_SKIP 0 +#define BMP280_OSRS_TEMP_X(osrs_t) ((osrs_t) << 5) +#define BMP280_OSRS_TEMP_1X BMP280_OSRS_TEMP_X(1) +#define BMP280_OSRS_TEMP_2X BMP280_OSRS_TEMP_X(2) +#define BMP280_OSRS_TEMP_4X BMP280_OSRS_TEMP_X(3) +#define BMP280_OSRS_TEMP_8X BMP280_OSRS_TEMP_X(4) +#define BMP280_OSRS_TEMP_16X BMP280_OSRS_TEMP_X(5) + +#define BMP280_OSRS_PRESS_MASK (BIT(4) | BIT(3) | BIT(2)) +#define BMP280_OSRS_PRESS_SKIP 0 +#define BMP280_OSRS_PRESS_X(osrs_p) ((osrs_p) << 2) +#define BMP280_OSRS_PRESS_1X BMP280_OSRS_PRESS_X(1) +#define BMP280_OSRS_PRESS_2X BMP280_OSRS_PRESS_X(2) +#define BMP280_OSRS_PRESS_4X BMP280_OSRS_PRESS_X(3) +#define BMP280_OSRS_PRESS_8X BMP280_OSRS_PRESS_X(4) +#define BMP280_OSRS_PRESS_16X BMP280_OSRS_PRESS_X(5) + +#define BMP280_MODE_MASK (BIT(1) | BIT(0)) +#define BMP280_MODE_SLEEP 0 +#define BMP280_MODE_FORCED BIT(0) +#define BMP280_MODE_NORMAL (BIT(1) | BIT(0)) + +/* BMP180 specific registers */ +#define BMP180_REG_OUT_XLSB 0xF8 +#define BMP180_REG_OUT_LSB 0xF7 +#define BMP180_REG_OUT_MSB 0xF6 + +#define BMP180_REG_CALIB_START 0xAA +#define BMP180_REG_CALIB_COUNT 22 + +#define BMP180_MEAS_SCO BIT(5) +#define BMP180_MEAS_TEMP (0x0E | BMP180_MEAS_SCO) +#define BMP180_MEAS_PRESS_X(oss) ((oss) << 6 | 0x14 | BMP180_MEAS_SCO) +#define BMP180_MEAS_PRESS_1X BMP180_MEAS_PRESS_X(0) +#define BMP180_MEAS_PRESS_2X BMP180_MEAS_PRESS_X(1) +#define BMP180_MEAS_PRESS_4X BMP180_MEAS_PRESS_X(2) +#define BMP180_MEAS_PRESS_8X BMP180_MEAS_PRESS_X(3) + +/* BMP180 and BMP280 common registers */ +#define BMP280_REG_CTRL_MEAS 0xF4 +#define BMP280_REG_RESET 0xE0 +#define BMP280_REG_ID 0xD0 + +#define BMP180_CHIP_ID 0x55 +#define BMP280_CHIP_ID 0x58 +#define BME280_CHIP_ID 0x60 +#define BMP280_SOFT_RESET_VAL 0xB6 + +/* Regmap configurations */ +extern const struct regmap_config bmp180_regmap_config; +extern const struct regmap_config bmp280_regmap_config; + +/* Probe called from different transports */ +int bmp280_common_probe(struct device *dev, + struct regmap *regmap, + unsigned int chip, + const char *name, + int irq); +int bmp280_common_remove(struct device *dev); + +/* PM ops */ +extern const struct dev_pm_ops bmp280_dev_pm_ops; diff --git a/drivers/iio/pressure/hp206c.c b/drivers/iio/pressure/hp206c.c index 90f2b6e4a920..12f769e86355 100644 --- a/drivers/iio/pressure/hp206c.c +++ b/drivers/iio/pressure/hp206c.c @@ -401,6 +401,7 @@ static const struct i2c_device_id hp206c_id[] = { {"hp206c"}, {} }; +MODULE_DEVICE_TABLE(i2c, hp206c_id); #ifdef CONFIG_ACPI static const struct acpi_device_id hp206c_acpi_match[] = { diff --git a/drivers/iio/pressure/mpl3115.c b/drivers/iio/pressure/mpl3115.c index 01b2e0b18878..6392d7b62841 100644 --- a/drivers/iio/pressure/mpl3115.c +++ b/drivers/iio/pressure/mpl3115.c @@ -171,7 +171,7 @@ static irqreturn_t mpl3115_trigger_handler(int irq, void *p) mutex_unlock(&data->lock); iio_push_to_buffers_with_timestamp(indio_dev, buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); done: iio_trigger_notify_done(indio_dev->trig); diff --git a/drivers/iio/pressure/ms5611_core.c b/drivers/iio/pressure/ms5611_core.c index 76578b07bb6e..feb41f82c64a 100644 --- a/drivers/iio/pressure/ms5611_core.c +++ b/drivers/iio/pressure/ms5611_core.c @@ -224,7 +224,8 @@ static irqreturn_t ms5611_trigger_handler(int irq, void *p) if (ret < 0) goto err; - iio_push_to_buffers_with_timestamp(indio_dev, buf, iio_get_time_ns()); + iio_push_to_buffers_with_timestamp(indio_dev, buf, + iio_get_time_ns(indio_dev)); err: iio_trigger_notify_done(indio_dev->trig); diff --git a/drivers/iio/pressure/ms5637.c b/drivers/iio/pressure/ms5637.c index e68052c118e6..953ffbc0ef96 100644 --- a/drivers/iio/pressure/ms5637.c +++ b/drivers/iio/pressure/ms5637.c @@ -1,6 +1,6 @@ /* - * ms5637.c - Support for Measurement-Specialties ms5637 and ms8607 - * pressure & temperature sensor + * ms5637.c - Support for Measurement-Specialties MS5637, MS5805 + * MS5837 and MS8607 pressure & temperature sensor * * Copyright (c) 2015 Measurement-Specialties * @@ -11,6 +11,10 @@ * Datasheet: * http://www.meas-spec.com/downloads/MS5637-02BA03.pdf * Datasheet: + * http://www.meas-spec.com/downloads/MS5805-02BA01.pdf + * Datasheet: + * http://www.meas-spec.com/downloads/MS5837-30BA.pdf + * Datasheet: * http://www.meas-spec.com/downloads/MS8607-02BA01.pdf */ @@ -170,9 +174,12 @@ static int ms5637_probe(struct i2c_client *client, static const struct i2c_device_id ms5637_id[] = { {"ms5637", 0}, - {"ms8607-temppressure", 1}, + {"ms5805", 0}, + {"ms5837", 0}, + {"ms8607-temppressure", 0}, {} }; +MODULE_DEVICE_TABLE(i2c, ms5637_id); static struct i2c_driver ms5637_driver = { .probe = ms5637_probe, diff --git a/drivers/iio/pressure/st_pressure.h b/drivers/iio/pressure/st_pressure.h index f5f41490060b..903a21e46874 100644 --- a/drivers/iio/pressure/st_pressure.h +++ b/drivers/iio/pressure/st_pressure.h @@ -17,6 +17,7 @@ #define LPS001WP_PRESS_DEV_NAME "lps001wp" #define LPS25H_PRESS_DEV_NAME "lps25h" #define LPS331AP_PRESS_DEV_NAME "lps331ap" +#define LPS22HB_PRESS_DEV_NAME "lps22hb" /** * struct st_sensors_platform_data - default press platform data diff --git a/drivers/iio/pressure/st_pressure_core.c b/drivers/iio/pressure/st_pressure_core.c index 92a118c3c4ac..55df9a75eb3a 100644 --- a/drivers/iio/pressure/st_pressure_core.c +++ b/drivers/iio/pressure/st_pressure_core.c @@ -28,6 +28,72 @@ #include #include "st_pressure.h" +/* + * About determining pressure scaling factors + * ------------------------------------------ + * + * Datasheets specify typical pressure sensitivity so that pressure is computed + * according to the following equation : + * pressure[mBar] = raw / sensitivity + * where : + * raw the 24 bits long raw sampled pressure + * sensitivity a scaling factor specified by the datasheet in LSB/mBar + * + * IIO ABI expects pressure to be expressed as kPascal, hence pressure should be + * computed according to : + * pressure[kPascal] = pressure[mBar] / 10 + * = raw / (sensitivity * 10) (1) + * + * Finally, st_press_read_raw() returns pressure scaling factor as an + * IIO_VAL_INT_PLUS_NANO with a zero integral part and "gain" as decimal part. + * Therefore, from (1), "gain" becomes : + * gain = 10^9 / (sensitivity * 10) + * = 10^8 / sensitivity + * + * About determining temperature scaling factors and offsets + * --------------------------------------------------------- + * + * Datasheets specify typical temperature sensitivity and offset so that + * temperature is computed according to the following equation : + * temp[Celsius] = offset[Celsius] + (raw / sensitivity) + * where : + * raw the 16 bits long raw sampled temperature + * offset a constant specified by the datasheet in degree Celsius + * (sometimes zero) + * sensitivity a scaling factor specified by the datasheet in LSB/Celsius + * + * IIO ABI expects temperature to be expressed as milli degree Celsius such as + * user space should compute temperature according to : + * temp[mCelsius] = temp[Celsius] * 10^3 + * = (offset[Celsius] + (raw / sensitivity)) * 10^3 + * = ((offset[Celsius] * sensitivity) + raw) * + * (10^3 / sensitivity) (2) + * + * IIO ABI expects user space to apply offset and scaling factors to raw samples + * according to : + * temp[mCelsius] = (OFFSET + raw) * SCALE + * where : + * OFFSET an arbitrary constant exposed by device + * SCALE an arbitrary scaling factor exposed by device + * + * Matching OFFSET and SCALE with members of (2) gives : + * OFFSET = offset[Celsius] * sensitivity (3) + * SCALE = 10^3 / sensitivity (4) + * + * st_press_read_raw() returns temperature scaling factor as an + * IIO_VAL_FRACTIONAL with a 10^3 numerator and "gain2" as denominator. + * Therefore, from (3), "gain2" becomes : + * gain2 = sensitivity + * + * When declared within channel, i.e. for a non zero specified offset, + * st_press_read_raw() will return the latter as an IIO_VAL_FRACTIONAL such as : + * numerator = OFFSET * 10^3 + * denominator = 10^3 + * giving from (4): + * numerator = offset[Celsius] * 10^3 * sensitivity + * = offset[mCelsius] * gain2 + */ + #define MCELSIUS_PER_CELSIUS 1000 /* Default pressure sensitivity */ @@ -39,8 +105,6 @@ #define ST_PRESS_LSB_PER_CELSIUS 480UL #define ST_PRESS_MILLI_CELSIUS_OFFSET 42500UL -#define ST_PRESS_NUMBER_DATA_CHANNELS 1 - /* FULLSCALE */ #define ST_PRESS_FS_AVL_1100MB 1100 #define ST_PRESS_FS_AVL_1260MB 1260 @@ -48,7 +112,11 @@ #define ST_PRESS_1_OUT_XL_ADDR 0x28 #define ST_TEMP_1_OUT_L_ADDR 0x2b -/* CUSTOM VALUES FOR LPS331AP SENSOR */ +/* + * CUSTOM VALUES FOR LPS331AP SENSOR + * See LPS331AP datasheet: + * http://www2.st.com/resource/en/datasheet/lps331ap.pdf + */ #define ST_PRESS_LPS331AP_WAI_EXP 0xbb #define ST_PRESS_LPS331AP_ODR_ADDR 0x20 #define ST_PRESS_LPS331AP_ODR_MASK 0x70 @@ -71,7 +139,9 @@ #define ST_PRESS_LPS331AP_OD_IRQ_MASK 0x40 #define ST_PRESS_LPS331AP_MULTIREAD_BIT true -/* CUSTOM VALUES FOR LPS001WP SENSOR */ +/* + * CUSTOM VALUES FOR THE OBSOLETE LPS001WP SENSOR + */ /* LPS001WP pressure resolution */ #define ST_PRESS_LPS001WP_LSB_PER_MBAR 16UL @@ -94,7 +164,11 @@ #define ST_PRESS_LPS001WP_OUT_L_ADDR 0x28 #define ST_TEMP_LPS001WP_OUT_L_ADDR 0x2a -/* CUSTOM VALUES FOR LPS25H SENSOR */ +/* + * CUSTOM VALUES FOR LPS25H SENSOR + * See LPS25H datasheet: + * http://www2.st.com/resource/en/datasheet/lps25h.pdf + */ #define ST_PRESS_LPS25H_WAI_EXP 0xbd #define ST_PRESS_LPS25H_ODR_ADDR 0x20 #define ST_PRESS_LPS25H_ODR_MASK 0x70 @@ -117,27 +191,54 @@ #define ST_PRESS_LPS25H_OUT_XL_ADDR 0x28 #define ST_TEMP_LPS25H_OUT_L_ADDR 0x2b +/* + * CUSTOM VALUES FOR LPS22HB SENSOR + * See LPS22HB datasheet: + * http://www2.st.com/resource/en/datasheet/lps22hb.pdf + */ + +/* LPS22HB temperature sensitivity */ +#define ST_PRESS_LPS22HB_LSB_PER_CELSIUS 100UL + +#define ST_PRESS_LPS22HB_WAI_EXP 0xb1 +#define ST_PRESS_LPS22HB_ODR_ADDR 0x10 +#define ST_PRESS_LPS22HB_ODR_MASK 0x70 +#define ST_PRESS_LPS22HB_ODR_AVL_1HZ_VAL 0x01 +#define ST_PRESS_LPS22HB_ODR_AVL_10HZ_VAL 0x02 +#define ST_PRESS_LPS22HB_ODR_AVL_25HZ_VAL 0x03 +#define ST_PRESS_LPS22HB_ODR_AVL_50HZ_VAL 0x04 +#define ST_PRESS_LPS22HB_ODR_AVL_75HZ_VAL 0x05 +#define ST_PRESS_LPS22HB_PW_ADDR 0x10 +#define ST_PRESS_LPS22HB_PW_MASK 0x70 +#define ST_PRESS_LPS22HB_BDU_ADDR 0x10 +#define ST_PRESS_LPS22HB_BDU_MASK 0x02 +#define ST_PRESS_LPS22HB_DRDY_IRQ_ADDR 0x12 +#define ST_PRESS_LPS22HB_DRDY_IRQ_INT1_MASK 0x04 +#define ST_PRESS_LPS22HB_DRDY_IRQ_INT2_MASK 0x08 +#define ST_PRESS_LPS22HB_IHL_IRQ_ADDR 0x12 +#define ST_PRESS_LPS22HB_IHL_IRQ_MASK 0x80 +#define ST_PRESS_LPS22HB_OD_IRQ_ADDR 0x12 +#define ST_PRESS_LPS22HB_OD_IRQ_MASK 0x40 +#define ST_PRESS_LPS22HB_MULTIREAD_BIT true + static const struct iio_chan_spec st_press_1_channels[] = { { .type = IIO_PRESSURE, - .channel2 = IIO_NO_MOD, .address = ST_PRESS_1_OUT_XL_ADDR, - .scan_index = ST_SENSORS_SCAN_X, + .scan_index = 0, .scan_type = { .sign = 'u', .realbits = 24, - .storagebits = 24, + .storagebits = 32, .endianness = IIO_LE, }, .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), - .modified = 0, }, { .type = IIO_TEMP, - .channel2 = IIO_NO_MOD, .address = ST_TEMP_1_OUT_L_ADDR, - .scan_index = -1, + .scan_index = 1, .scan_type = { .sign = 'u', .realbits = 16, @@ -148,17 +249,15 @@ static const struct iio_chan_spec st_press_1_channels[] = { BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_OFFSET), - .modified = 0, }, - IIO_CHAN_SOFT_TIMESTAMP(1) + IIO_CHAN_SOFT_TIMESTAMP(2) }; static const struct iio_chan_spec st_press_lps001wp_channels[] = { { .type = IIO_PRESSURE, - .channel2 = IIO_NO_MOD, .address = ST_PRESS_LPS001WP_OUT_L_ADDR, - .scan_index = ST_SENSORS_SCAN_X, + .scan_index = 0, .scan_type = { .sign = 'u', .realbits = 16, @@ -168,13 +267,11 @@ static const struct iio_chan_spec st_press_lps001wp_channels[] = { .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), - .modified = 0, }, { .type = IIO_TEMP, - .channel2 = IIO_NO_MOD, .address = ST_TEMP_LPS001WP_OUT_L_ADDR, - .scan_index = -1, + .scan_index = 1, .scan_type = { .sign = 'u', .realbits = 16, @@ -184,9 +281,42 @@ static const struct iio_chan_spec st_press_lps001wp_channels[] = { .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), - .modified = 0, }, - IIO_CHAN_SOFT_TIMESTAMP(1) + IIO_CHAN_SOFT_TIMESTAMP(2) +}; + +static const struct iio_chan_spec st_press_lps22hb_channels[] = { + { + .type = IIO_PRESSURE, + .address = ST_PRESS_1_OUT_XL_ADDR, + .scan_index = 0, + .scan_type = { + .sign = 'u', + .realbits = 24, + .storagebits = 32, + .endianness = IIO_LE, + }, + .info_mask_separate = + BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), + }, + { + .type = IIO_TEMP, + .address = ST_TEMP_1_OUT_L_ADDR, + .scan_index = 1, + .scan_type = { + .sign = 's', + .realbits = 16, + .storagebits = 16, + .endianness = IIO_LE, + }, + .info_mask_separate = + BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), + }, + IIO_CHAN_SOFT_TIMESTAMP(2) }; static const struct st_sensor_settings st_press_sensors_settings[] = { @@ -346,6 +476,59 @@ static const struct st_sensor_settings st_press_sensors_settings[] = { .multi_read_bit = ST_PRESS_LPS25H_MULTIREAD_BIT, .bootime = 2, }, + { + .wai = ST_PRESS_LPS22HB_WAI_EXP, + .wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS, + .sensors_supported = { + [0] = LPS22HB_PRESS_DEV_NAME, + }, + .ch = (struct iio_chan_spec *)st_press_lps22hb_channels, + .num_ch = ARRAY_SIZE(st_press_lps22hb_channels), + .odr = { + .addr = ST_PRESS_LPS22HB_ODR_ADDR, + .mask = ST_PRESS_LPS22HB_ODR_MASK, + .odr_avl = { + { 1, ST_PRESS_LPS22HB_ODR_AVL_1HZ_VAL, }, + { 10, ST_PRESS_LPS22HB_ODR_AVL_10HZ_VAL, }, + { 25, ST_PRESS_LPS22HB_ODR_AVL_25HZ_VAL, }, + { 50, ST_PRESS_LPS22HB_ODR_AVL_50HZ_VAL, }, + { 75, ST_PRESS_LPS22HB_ODR_AVL_75HZ_VAL, }, + }, + }, + .pw = { + .addr = ST_PRESS_LPS22HB_PW_ADDR, + .mask = ST_PRESS_LPS22HB_PW_MASK, + .value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE, + }, + .fs = { + .fs_avl = { + /* + * Pressure and temperature sensitivity values + * as defined in table 3 of LPS22HB datasheet. + */ + [0] = { + .num = ST_PRESS_FS_AVL_1260MB, + .gain = ST_PRESS_KPASCAL_NANO_SCALE, + .gain2 = ST_PRESS_LPS22HB_LSB_PER_CELSIUS, + }, + }, + }, + .bdu = { + .addr = ST_PRESS_LPS22HB_BDU_ADDR, + .mask = ST_PRESS_LPS22HB_BDU_MASK, + }, + .drdy_irq = { + .addr = ST_PRESS_LPS22HB_DRDY_IRQ_ADDR, + .mask_int1 = ST_PRESS_LPS22HB_DRDY_IRQ_INT1_MASK, + .mask_int2 = ST_PRESS_LPS22HB_DRDY_IRQ_INT2_MASK, + .addr_ihl = ST_PRESS_LPS22HB_IHL_IRQ_ADDR, + .mask_ihl = ST_PRESS_LPS22HB_IHL_IRQ_MASK, + .addr_od = ST_PRESS_LPS22HB_OD_IRQ_ADDR, + .mask_od = ST_PRESS_LPS22HB_OD_IRQ_MASK, + .addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR, + }, + .multi_read_bit = ST_PRESS_LPS22HB_MULTIREAD_BIT, + }, }; static int st_press_write_raw(struct iio_dev *indio_dev, @@ -462,23 +645,30 @@ int st_press_common_probe(struct iio_dev *indio_dev) indio_dev->info = &press_info; mutex_init(&press_data->tb.buf_lock); - st_sensors_power_enable(indio_dev); + err = st_sensors_power_enable(indio_dev); + if (err) + return err; err = st_sensors_check_device_support(indio_dev, ARRAY_SIZE(st_press_sensors_settings), st_press_sensors_settings); if (err < 0) - return err; - - press_data->num_data_channels = ST_PRESS_NUMBER_DATA_CHANNELS; + goto st_press_power_off; + + /* + * Skip timestamping channel while declaring available channels to + * common st_sensor layer. Look at st_sensors_get_buffer_element() to + * see how timestamps are explicitly pushed as last samples block + * element. + */ + press_data->num_data_channels = press_data->sensor_settings->num_ch - 1; press_data->multiread_bit = press_data->sensor_settings->multi_read_bit; indio_dev->channels = press_data->sensor_settings->ch; indio_dev->num_channels = press_data->sensor_settings->num_ch; - if (press_data->sensor_settings->fs.addr != 0) - press_data->current_fullscale = - (struct st_sensor_fullscale_avl *) - &press_data->sensor_settings->fs.fs_avl[0]; + press_data->current_fullscale = + (struct st_sensor_fullscale_avl *) + &press_data->sensor_settings->fs.fs_avl[0]; press_data->odr = press_data->sensor_settings->odr.odr_avl[0].hz; @@ -490,11 +680,11 @@ int st_press_common_probe(struct iio_dev *indio_dev) err = st_sensors_init_sensor(indio_dev, press_data->dev->platform_data); if (err < 0) - return err; + goto st_press_power_off; err = st_press_allocate_ring(indio_dev); if (err < 0) - return err; + goto st_press_power_off; if (irq > 0) { err = st_sensors_allocate_trigger(indio_dev, @@ -517,6 +707,8 @@ st_press_device_register_error: st_sensors_deallocate_trigger(indio_dev); st_press_probe_trigger_error: st_press_deallocate_ring(indio_dev); +st_press_power_off: + st_sensors_power_disable(indio_dev); return err; } diff --git a/drivers/iio/pressure/st_pressure_i2c.c b/drivers/iio/pressure/st_pressure_i2c.c index 8fcf9766eaec..ed18701c68c9 100644 --- a/drivers/iio/pressure/st_pressure_i2c.c +++ b/drivers/iio/pressure/st_pressure_i2c.c @@ -32,6 +32,10 @@ static const struct of_device_id st_press_of_match[] = { .compatible = "st,lps331ap-press", .data = LPS331AP_PRESS_DEV_NAME, }, + { + .compatible = "st,lps22hb-press", + .data = LPS22HB_PRESS_DEV_NAME, + }, {}, }; MODULE_DEVICE_TABLE(of, st_press_of_match); diff --git a/drivers/iio/pressure/st_pressure_spi.c b/drivers/iio/pressure/st_pressure_spi.c index 40c0692ff1de..550508025af1 100644 --- a/drivers/iio/pressure/st_pressure_spi.c +++ b/drivers/iio/pressure/st_pressure_spi.c @@ -50,6 +50,7 @@ static const struct spi_device_id st_press_id_table[] = { { LPS001WP_PRESS_DEV_NAME }, { LPS25H_PRESS_DEV_NAME }, { LPS331AP_PRESS_DEV_NAME }, + { LPS22HB_PRESS_DEV_NAME }, {}, }; MODULE_DEVICE_TABLE(spi, st_press_id_table); diff --git a/drivers/iio/proximity/as3935.c b/drivers/iio/proximity/as3935.c index e2f926cdcad2..2e3a70e1b245 100644 --- a/drivers/iio/proximity/as3935.c +++ b/drivers/iio/proximity/as3935.c @@ -231,10 +231,16 @@ static void as3935_event_work(struct work_struct *work) { struct as3935_state *st; int val; + int ret; st = container_of(work, struct as3935_state, work.work); - as3935_read(st, AS3935_INT, &val); + ret = as3935_read(st, AS3935_INT, &val); + if (ret) { + dev_warn(&st->spi->dev, "read error\n"); + return; + } + val &= AS3935_INT_MASK; switch (val) { @@ -242,7 +248,7 @@ static void as3935_event_work(struct work_struct *work) iio_trigger_poll(st->trig); break; case AS3935_NOISE_INT: - dev_warn(&st->spi->dev, "noise level is too high"); + dev_warn(&st->spi->dev, "noise level is too high\n"); break; } } @@ -346,7 +352,6 @@ static int as3935_probe(struct spi_device *spi) st = iio_priv(indio_dev); st->spi = spi; - st->tune_cap = 0; spi_set_drvdata(spi, indio_dev); mutex_init(&st->lock); @@ -468,4 +473,3 @@ module_spi_driver(as3935_driver); MODULE_AUTHOR("Matt Ranostay "); MODULE_DESCRIPTION("AS3935 lightning sensor"); MODULE_LICENSE("GPL"); -MODULE_ALIAS("spi:as3935"); diff --git a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c index 4f502386aa86..3141c3c161bb 100644 --- a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c +++ b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c @@ -203,22 +203,19 @@ static int lidar_read_raw(struct iio_dev *indio_dev, struct lidar_data *data = iio_priv(indio_dev); int ret = -EINVAL; - mutex_lock(&indio_dev->mlock); - - if (iio_buffer_enabled(indio_dev) && mask == IIO_CHAN_INFO_RAW) { - ret = -EBUSY; - goto error_busy; - } - switch (mask) { case IIO_CHAN_INFO_RAW: { u16 reg; + if (iio_device_claim_direct_mode(indio_dev)) + return -EBUSY; + ret = lidar_get_measurement(data, ®); if (!ret) { *val = reg; ret = IIO_VAL_INT; } + iio_device_release_direct_mode(indio_dev); break; } case IIO_CHAN_INFO_SCALE: @@ -228,9 +225,6 @@ static int lidar_read_raw(struct iio_dev *indio_dev, break; } -error_busy: - mutex_unlock(&indio_dev->mlock); - return ret; } @@ -244,7 +238,7 @@ static irqreturn_t lidar_trigger_handler(int irq, void *private) ret = lidar_get_measurement(data, data->buffer); if (!ret) { iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); } else if (ret != -EINVAL) { dev_err(&data->client->dev, "cannot read LIDAR measurement"); } diff --git a/drivers/iio/proximity/sx9500.c b/drivers/iio/proximity/sx9500.c index 66cd09a18786..1d74b3aafeed 100644 --- a/drivers/iio/proximity/sx9500.c +++ b/drivers/iio/proximity/sx9500.c @@ -492,7 +492,7 @@ static void sx9500_push_events(struct iio_dev *indio_dev) dir = new_prox ? IIO_EV_DIR_FALLING : IIO_EV_DIR_RISING; ev = IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, chan, IIO_EV_TYPE_THRESH, dir); - iio_push_event(indio_dev, ev, iio_get_time_ns()); + iio_push_event(indio_dev, ev, iio_get_time_ns(indio_dev)); data->prox_stat[chan] = new_prox; } } @@ -669,7 +669,7 @@ static irqreturn_t sx9500_trigger_handler(int irq, void *private) } iio_push_to_buffers_with_timestamp(indio_dev, data->buffer, - iio_get_time_ns()); + iio_get_time_ns(indio_dev)); out: mutex_unlock(&data->mutex); diff --git a/drivers/iio/temperature/tsys02d.c b/drivers/iio/temperature/tsys02d.c index ab6fe8f6f2d1..c0a19a000387 100644 --- a/drivers/iio/temperature/tsys02d.c +++ b/drivers/iio/temperature/tsys02d.c @@ -174,6 +174,7 @@ static const struct i2c_device_id tsys02d_id[] = { {"tsys02d", 0}, {} }; +MODULE_DEVICE_TABLE(i2c, tsys02d_id); static struct i2c_driver tsys02d_driver = { .probe = tsys02d_probe, diff --git a/drivers/iio/trigger/Kconfig b/drivers/iio/trigger/Kconfig index 519e6772f6f5..809b2e7d58fa 100644 --- a/drivers/iio/trigger/Kconfig +++ b/drivers/iio/trigger/Kconfig @@ -24,6 +24,18 @@ config IIO_INTERRUPT_TRIGGER To compile this driver as a module, choose M here: the module will be called iio-trig-interrupt. +config IIO_TIGHTLOOP_TRIGGER + tristate "A kthread based hammering loop trigger" + depends on IIO_SW_TRIGGER + help + An experimental trigger, used to allow sensors to be sampled as fast + as possible under the limitations of whatever else is going on. + Uses a tight loop in a kthread. Will only work with lower half only + trigger consumers. + + To compile this driver as a module, choose M here: the + module will be called iio-trig-loop. + config IIO_SYSFS_TRIGGER tristate "SYSFS trigger" depends on SYSFS diff --git a/drivers/iio/trigger/Makefile b/drivers/iio/trigger/Makefile index fe06eb564367..aab4dc23303d 100644 --- a/drivers/iio/trigger/Makefile +++ b/drivers/iio/trigger/Makefile @@ -7,3 +7,4 @@ obj-$(CONFIG_IIO_HRTIMER_TRIGGER) += iio-trig-hrtimer.o obj-$(CONFIG_IIO_INTERRUPT_TRIGGER) += iio-trig-interrupt.o obj-$(CONFIG_IIO_SYSFS_TRIGGER) += iio-trig-sysfs.o +obj-$(CONFIG_IIO_TIGHTLOOP_TRIGGER) += iio-trig-loop.o diff --git a/drivers/iio/trigger/iio-trig-loop.c b/drivers/iio/trigger/iio-trig-loop.c new file mode 100644 index 000000000000..dc6be28f96fe --- /dev/null +++ b/drivers/iio/trigger/iio-trig-loop.c @@ -0,0 +1,143 @@ +/* + * Copyright 2016 Jonathan Cameron + * + * Licensed under the GPL-2. + * + * Based on a mashup of the hrtimer trigger and continuous sampling proposal of + * Gregor Boirie + * + * Note this is still rather experimental and may eat babies. + * + * Todo + * * Protect against connection of devices that 'need' the top half + * handler. + * * Work out how to run top half handlers in this context if it is + * safe to do so (timestamp grabbing for example) + * + * Tested against a max1363. Used about 33% cpu for the thread and 20% + * for generic_buffer piping to /dev/null. Watermark set at 64 on a 128 + * element kfifo buffer. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct iio_loop_info { + struct iio_sw_trigger swt; + struct task_struct *task; +}; + +static struct config_item_type iio_loop_type = { + .ct_owner = THIS_MODULE, +}; + +static int iio_loop_thread(void *data) +{ + struct iio_trigger *trig = data; + + set_freezable(); + + do { + iio_trigger_poll_chained(trig); + } while (likely(!kthread_freezable_should_stop(NULL))); + + return 0; +} + +static int iio_loop_trigger_set_state(struct iio_trigger *trig, bool state) +{ + struct iio_loop_info *loop_trig = iio_trigger_get_drvdata(trig); + + if (state) { + loop_trig->task = kthread_run(iio_loop_thread, + trig, trig->name); + if (unlikely(IS_ERR(loop_trig->task))) { + dev_err(&trig->dev, + "failed to create trigger loop thread\n"); + return PTR_ERR(loop_trig->task); + } + } else { + kthread_stop(loop_trig->task); + } + + return 0; +} + +static const struct iio_trigger_ops iio_loop_trigger_ops = { + .set_trigger_state = iio_loop_trigger_set_state, + .owner = THIS_MODULE, +}; + +static struct iio_sw_trigger *iio_trig_loop_probe(const char *name) +{ + struct iio_loop_info *trig_info; + int ret; + + trig_info = kzalloc(sizeof(*trig_info), GFP_KERNEL); + if (!trig_info) + return ERR_PTR(-ENOMEM); + + trig_info->swt.trigger = iio_trigger_alloc("%s", name); + if (!trig_info->swt.trigger) { + ret = -ENOMEM; + goto err_free_trig_info; + } + + iio_trigger_set_drvdata(trig_info->swt.trigger, trig_info); + trig_info->swt.trigger->ops = &iio_loop_trigger_ops; + + ret = iio_trigger_register(trig_info->swt.trigger); + if (ret) + goto err_free_trigger; + + iio_swt_group_init_type_name(&trig_info->swt, name, &iio_loop_type); + + return &trig_info->swt; + +err_free_trigger: + iio_trigger_free(trig_info->swt.trigger); +err_free_trig_info: + kfree(trig_info); + + return ERR_PTR(ret); +} + +static int iio_trig_loop_remove(struct iio_sw_trigger *swt) +{ + struct iio_loop_info *trig_info; + + trig_info = iio_trigger_get_drvdata(swt->trigger); + + iio_trigger_unregister(swt->trigger); + iio_trigger_free(swt->trigger); + kfree(trig_info); + + return 0; +} + +static const struct iio_sw_trigger_ops iio_trig_loop_ops = { + .probe = iio_trig_loop_probe, + .remove = iio_trig_loop_remove, +}; + +static struct iio_sw_trigger_type iio_trig_loop = { + .name = "loop", + .owner = THIS_MODULE, + .ops = &iio_trig_loop_ops, +}; + +module_iio_sw_trigger_driver(iio_trig_loop); + +MODULE_AUTHOR("Jonathan Cameron "); +MODULE_DESCRIPTION("Loop based trigger for the iio subsystem"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:iio-trig-loop"); diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 040966775f40..1a2984c28b95 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -411,7 +411,9 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port, for (ix = 0; ix < table->sz; ix++) if (table->data_vec[ix].attr.ndev == ndev) - if (!del_gid(ib_dev, port, table, ix, false)) + if (!del_gid(ib_dev, port, table, ix, + !!(table->data_vec[ix].props & + GID_TABLE_ENTRY_DEFAULT))) deleted = true; write_unlock_irq(&table->rwlock); diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index f0c91ba3178a..ad1b1adcf6f0 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -708,17 +708,6 @@ static void cma_deref_id(struct rdma_id_private *id_priv) complete(&id_priv->comp); } -static int cma_disable_callback(struct rdma_id_private *id_priv, - enum rdma_cm_state state) -{ - mutex_lock(&id_priv->handler_mutex); - if (id_priv->state != state) { - mutex_unlock(&id_priv->handler_mutex); - return -EINVAL; - } - return 0; -} - struct rdma_cm_id *rdma_create_id(struct net *net, rdma_cm_event_handler event_handler, void *context, enum rdma_port_space ps, @@ -1671,11 +1660,12 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) struct rdma_cm_event event; int ret = 0; + mutex_lock(&id_priv->handler_mutex); if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && - cma_disable_callback(id_priv, RDMA_CM_CONNECT)) || + id_priv->state != RDMA_CM_CONNECT) || (ib_event->event == IB_CM_TIMEWAIT_EXIT && - cma_disable_callback(id_priv, RDMA_CM_DISCONNECT))) - return 0; + id_priv->state != RDMA_CM_DISCONNECT)) + goto out; memset(&event, 0, sizeof event); switch (ib_event->event) { @@ -1870,7 +1860,7 @@ static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_e static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) { - struct rdma_id_private *listen_id, *conn_id; + struct rdma_id_private *listen_id, *conn_id = NULL; struct rdma_cm_event event; struct net_device *net_dev; int offset, ret; @@ -1884,9 +1874,10 @@ static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) goto net_dev_put; } - if (cma_disable_callback(listen_id, RDMA_CM_LISTEN)) { + mutex_lock(&listen_id->handler_mutex); + if (listen_id->state != RDMA_CM_LISTEN) { ret = -ECONNABORTED; - goto net_dev_put; + goto err1; } memset(&event, 0, sizeof event); @@ -1976,8 +1967,9 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; - if (cma_disable_callback(id_priv, RDMA_CM_CONNECT)) - return 0; + mutex_lock(&id_priv->handler_mutex); + if (id_priv->state != RDMA_CM_CONNECT) + goto out; memset(&event, 0, sizeof event); switch (iw_event->event) { @@ -2029,6 +2021,7 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) return ret; } +out: mutex_unlock(&id_priv->handler_mutex); return ret; } @@ -2039,13 +2032,15 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, struct rdma_cm_id *new_cm_id; struct rdma_id_private *listen_id, *conn_id; struct rdma_cm_event event; - int ret; + int ret = -ECONNABORTED; struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; listen_id = cm_id->context; - if (cma_disable_callback(listen_id, RDMA_CM_LISTEN)) - return -ECONNABORTED; + + mutex_lock(&listen_id->handler_mutex); + if (listen_id->state != RDMA_CM_LISTEN) + goto out; /* Create a new RDMA id for the new IW CM ID */ new_cm_id = rdma_create_id(listen_id->id.route.addr.dev_addr.net, @@ -3216,8 +3211,9 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd; int ret = 0; - if (cma_disable_callback(id_priv, RDMA_CM_CONNECT)) - return 0; + mutex_lock(&id_priv->handler_mutex); + if (id_priv->state != RDMA_CM_CONNECT) + goto out; memset(&event, 0, sizeof event); switch (ib_event->event) { @@ -3673,12 +3669,13 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) struct rdma_id_private *id_priv; struct cma_multicast *mc = multicast->context; struct rdma_cm_event event; - int ret; + int ret = 0; id_priv = mc->id_priv; - if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) && - cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED)) - return 0; + mutex_lock(&id_priv->handler_mutex); + if (id_priv->state != RDMA_CM_ADDR_BOUND && + id_priv->state != RDMA_CM_ADDR_RESOLVED) + goto out; if (!status) status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey)); @@ -3720,6 +3717,7 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) return 0; } +out: mutex_unlock(&id_priv->handler_mutex); return 0; } @@ -3878,12 +3876,12 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num - rdma_start_port(id_priv->cma_dev->device)]; if (addr->sa_family == AF_INET) { - if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) + if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT; err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, true); - if (!err) { - mc->igmp_joined = true; - mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT; + if (!err) + mc->igmp_joined = true; } } else { if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index a5793c8f1590..60df4f8e81be 100644 --- a/drivers/infiniband/core/sysfs.c +++ b/drivers/infiniband/core/sysfs.c @@ -530,6 +530,7 @@ static PORT_PMA_ATTR(port_xmit_data , 12, 32, 192); static PORT_PMA_ATTR(port_rcv_data , 13, 32, 224); static PORT_PMA_ATTR(port_xmit_packets , 14, 32, 256); static PORT_PMA_ATTR(port_rcv_packets , 15, 32, 288); +static PORT_PMA_ATTR(port_xmit_wait , 0, 32, 320); /* * Counters added by extended set @@ -560,6 +561,7 @@ static struct attribute *pma_attrs[] = { &port_pma_attr_port_rcv_data.attr.attr, &port_pma_attr_port_xmit_packets.attr.attr, &port_pma_attr_port_rcv_packets.attr.attr, + &port_pma_attr_port_xmit_wait.attr.attr, NULL }; @@ -579,6 +581,7 @@ static struct attribute *pma_attrs_ext[] = { &port_pma_attr_ext_port_xmit_data.attr.attr, &port_pma_attr_ext_port_rcv_data.attr.attr, &port_pma_attr_ext_port_xmit_packets.attr.attr, + &port_pma_attr_port_xmit_wait.attr.attr, &port_pma_attr_ext_port_rcv_packets.attr.attr, &port_pma_attr_ext_unicast_rcv_packets.attr.attr, &port_pma_attr_ext_unicast_xmit_packets.attr.attr, @@ -604,6 +607,7 @@ static struct attribute *pma_attrs_noietf[] = { &port_pma_attr_ext_port_rcv_data.attr.attr, &port_pma_attr_ext_port_xmit_packets.attr.attr, &port_pma_attr_ext_port_rcv_packets.attr.attr, + &port_pma_attr_port_xmit_wait.attr.attr, NULL }; diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 1a8babb8ee3c..825021d1008b 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1747,7 +1747,7 @@ static int create_qp(struct ib_uverbs_file *file, struct ib_srq *srq = NULL; struct ib_qp *qp; char *buf; - struct ib_qp_init_attr attr; + struct ib_qp_init_attr attr = {}; struct ib_uverbs_ex_create_qp_resp resp; int ret; diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 1d7d4cf442e3..6298f54b4137 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -511,12 +511,16 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, ah_attr->grh.dgid = sgid; if (!rdma_cap_eth_ah(device, port_num)) { - ret = ib_find_cached_gid_by_port(device, &dgid, - IB_GID_TYPE_IB, - port_num, NULL, - &gid_index); - if (ret) - return ret; + if (dgid.global.interface_id != cpu_to_be64(IB_SA_WELL_KNOWN_GUID)) { + ret = ib_find_cached_gid_by_port(device, &dgid, + IB_GID_TYPE_IB, + port_num, NULL, + &gid_index); + if (ret) + return ret; + } else { + gid_index = 0; + } } ah_attr->grh.sgid_index = (u8) gid_index; diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 81619fbb5842..dad4d0ebbdff 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -1037,7 +1037,7 @@ static void dc_shutdown(struct hfi1_devdata *); static void dc_start(struct hfi1_devdata *); static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, unsigned int *np); -static void remove_full_mgmt_pkey(struct hfi1_pportdata *ppd); +static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); /* * Error interrupt table entry. This is used as input to the interrupt @@ -6962,8 +6962,6 @@ void handle_link_down(struct work_struct *work) } reset_neighbor_info(ppd); - if (ppd->mgmt_allowed) - remove_full_mgmt_pkey(ppd); /* disable the port */ clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); @@ -7070,12 +7068,16 @@ static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd) __func__, ppd->pkeys[2], FULL_MGMT_P_KEY); ppd->pkeys[2] = FULL_MGMT_P_KEY; (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); + hfi1_event_pkey_change(ppd->dd, ppd->port); } -static void remove_full_mgmt_pkey(struct hfi1_pportdata *ppd) +static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd) { - ppd->pkeys[2] = 0; - (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); + if (ppd->pkeys[2] != 0) { + ppd->pkeys[2] = 0; + (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); + hfi1_event_pkey_change(ppd->dd, ppd->port); + } } /* @@ -9168,6 +9170,13 @@ int start_link(struct hfi1_pportdata *ppd) return 0; } + /* + * FULL_MGMT_P_KEY is cleared from the pkey table, so that the + * pkey table can be configured properly if the HFI unit is connected + * to switch port with MgmtAllowed=NO + */ + clear_full_mgmt_pkey(ppd); + return set_link_state(ppd, HLS_DN_POLL); } @@ -9777,7 +9786,7 @@ static void set_send_length(struct hfi1_pportdata *ppd) u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2) & SEND_LEN_CHECK1_LEN_VL15_MASK) << SEND_LEN_CHECK1_LEN_VL15_SHIFT; - int i; + int i, j; u32 thres; for (i = 0; i < ppd->vls_supported; i++) { @@ -9801,7 +9810,10 @@ static void set_send_length(struct hfi1_pportdata *ppd) sc_mtu_to_threshold(dd->vld[i].sc, dd->vld[i].mtu, dd->rcd[0]->rcvhdrqentsize)); - sc_set_cr_threshold(dd->vld[i].sc, thres); + for (j = 0; j < INIT_SC_PER_VL; j++) + sc_set_cr_threshold( + pio_select_send_context_vl(dd, j, i), + thres); } thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50), sc_mtu_to_threshold(dd->vld[15].sc, @@ -14101,8 +14113,14 @@ static int init_asic_data(struct hfi1_devdata *dd) { unsigned long flags; struct hfi1_devdata *tmp, *peer = NULL; + struct hfi1_asic_data *asic_data; int ret = 0; + /* pre-allocate the asic structure in case we are the first device */ + asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL); + if (!asic_data) + return -ENOMEM; + spin_lock_irqsave(&hfi1_devs_lock, flags); /* Find our peer device */ list_for_each_entry(tmp, &hfi1_dev_list, list) { @@ -14114,18 +14132,14 @@ static int init_asic_data(struct hfi1_devdata *dd) } if (peer) { + /* use already allocated structure */ dd->asic_data = peer->asic_data; + kfree(asic_data); } else { - dd->asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL); - if (!dd->asic_data) { - ret = -ENOMEM; - goto done; - } + dd->asic_data = asic_data; mutex_init(&dd->asic_data->asic_resource_mutex); } dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */ - -done: spin_unlock_irqrestore(&hfi1_devs_lock, flags); return ret; } diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 7a5b0e676cc7..c702a009608f 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -203,6 +203,9 @@ static long hfi1_file_ioctl(struct file *fp, unsigned int cmd, switch (cmd) { case HFI1_IOCTL_ASSIGN_CTXT: + if (uctxt) + return -EINVAL; + if (copy_from_user(&uinfo, (struct hfi1_user_info __user *)arg, sizeof(uinfo))) diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 0d28a5a40fae..eed971ccd2a1 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -1383,7 +1383,7 @@ static void postinit_cleanup(struct hfi1_devdata *dd) static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { int ret = 0, j, pidx, initfail; - struct hfi1_devdata *dd = NULL; + struct hfi1_devdata *dd = ERR_PTR(-EINVAL); struct hfi1_pportdata *ppd; /* First, lock the non-writable module parameters */ diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c index 219029576ba0..fca07a1d6c28 100644 --- a/drivers/infiniband/hw/hfi1/mad.c +++ b/drivers/infiniband/hw/hfi1/mad.c @@ -78,6 +78,16 @@ static inline void clear_opa_smp_data(struct opa_smp *smp) memset(data, 0, size); } +void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port) +{ + struct ib_event event; + + event.event = IB_EVENT_PKEY_CHANGE; + event.device = &dd->verbs_dev.rdi.ibdev; + event.element.port_num = port; + ib_dispatch_event(&event); +} + static void send_trap(struct hfi1_ibport *ibp, void *data, unsigned len) { struct ib_mad_send_buf *send_buf; @@ -1418,15 +1428,10 @@ static int set_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys) } if (changed) { - struct ib_event event; - (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); - - event.event = IB_EVENT_PKEY_CHANGE; - event.device = &dd->verbs_dev.rdi.ibdev; - event.element.port_num = port; - ib_dispatch_event(&event); + hfi1_event_pkey_change(dd, port); } + return 0; } diff --git a/drivers/infiniband/hw/hfi1/mad.h b/drivers/infiniband/hw/hfi1/mad.h index 55ee08675333..8b734aaae88a 100644 --- a/drivers/infiniband/hw/hfi1/mad.h +++ b/drivers/infiniband/hw/hfi1/mad.h @@ -434,4 +434,6 @@ struct sc2vlnt { COUNTER_MASK(1, 3) | \ COUNTER_MASK(1, 4)) +void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port); + #endif /* _HFI1_MAD_H */ diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c index d5edb1afbb8f..d4022450b73f 100644 --- a/drivers/infiniband/hw/hfi1/pio.c +++ b/drivers/infiniband/hw/hfi1/pio.c @@ -995,7 +995,7 @@ static void sc_wait_for_packet_egress(struct send_context *sc, int pause) /* counter is reset if occupancy count changes */ if (reg != reg_prev) loop = 0; - if (loop > 500) { + if (loop > 50000) { /* timed out - bounce the link */ dd_dev_err(dd, "%s: context %u(%u) timeout waiting for packets to egress, remaining count %u, bouncing link\n", @@ -1797,6 +1797,21 @@ static void pio_map_rcu_callback(struct rcu_head *list) pio_map_free(m); } +/* + * Set credit return threshold for the kernel send context + */ +static void set_threshold(struct hfi1_devdata *dd, int scontext, int i) +{ + u32 thres; + + thres = min(sc_percent_to_threshold(dd->kernel_send_context[scontext], + 50), + sc_mtu_to_threshold(dd->kernel_send_context[scontext], + dd->vld[i].mtu, + dd->rcd[0]->rcvhdrqentsize)); + sc_set_cr_threshold(dd->kernel_send_context[scontext], thres); +} + /* * pio_map_init - called when #vls change * @dd: hfi1_devdata @@ -1872,11 +1887,16 @@ int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_scontexts) if (!newmap->map[i]) goto bail; newmap->map[i]->mask = (1 << ilog2(sz)) - 1; - /* assign send contexts */ + /* + * assign send contexts and + * adjust credit return threshold + */ for (j = 0; j < sz; j++) { - if (dd->kernel_send_context[scontext]) + if (dd->kernel_send_context[scontext]) { newmap->map[i]->ksc[j] = dd->kernel_send_context[scontext]; + set_threshold(dd, scontext, i); + } if (++scontext >= first_scontext + vl_scontexts[i]) /* wrap back to first send context */ diff --git a/drivers/infiniband/hw/hfi1/qsfp.c b/drivers/infiniband/hw/hfi1/qsfp.c index 2441669f0817..9fb561682c66 100644 --- a/drivers/infiniband/hw/hfi1/qsfp.c +++ b/drivers/infiniband/hw/hfi1/qsfp.c @@ -579,7 +579,8 @@ int qsfp_dump(struct hfi1_pportdata *ppd, char *buf, int len) if (ppd->qsfp_info.cache_valid) { if (QSFP_IS_CU(cache[QSFP_MOD_TECH_OFFS])) - sprintf(lenstr, "%dM ", cache[QSFP_MOD_LEN_OFFS]); + snprintf(lenstr, sizeof(lenstr), "%dM ", + cache[QSFP_MOD_LEN_OFFS]); power_byte = cache[QSFP_MOD_PWR_OFFS]; sofar += scnprintf(buf + sofar, len - sofar, "PWR:%.3sW\n", diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c index 1e503ad0bebb..be91f6fa1c87 100644 --- a/drivers/infiniband/hw/hfi1/ud.c +++ b/drivers/infiniband/hw/hfi1/ud.c @@ -678,8 +678,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) u32 tlen = packet->tlen; struct rvt_qp *qp = packet->qp; bool has_grh = rcv_flags & HFI1_HAS_GRH; - bool sc4_bit = has_sc4_bit(packet); - u8 sc; + u8 sc5 = hdr2sc((struct hfi1_message_header *)hdr, packet->rhf); u32 bth1; int is_mcast; struct ib_grh *grh = NULL; @@ -697,10 +696,8 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) */ struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); u32 lqpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK; - u8 sl, sc5; + u8 sl; - sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf; - sc5 |= sc4_bit; sl = ibp->sc_to_sl[sc5]; process_becn(ppd, sl, 0, lqpn, 0, IB_CC_SVCTYPE_UD); @@ -717,10 +714,6 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) if (!is_mcast && (opcode != IB_OPCODE_CNP) && bth1 & HFI1_FECN_SMASK) { u16 slid = be16_to_cpu(hdr->lrh[3]); - u8 sc5; - - sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf; - sc5 |= sc4_bit; return_cnp(ibp, qp, src_qp, pkey, dlid, slid, sc5, grh); } @@ -745,10 +738,6 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) if (qp->ibqp.qp_num > 1) { struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); u16 slid; - u8 sc5; - - sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf; - sc5 |= sc4_bit; slid = be16_to_cpu(hdr->lrh[3]); if (unlikely(rcv_pkey_check(ppd, pkey, sc5, slid))) { @@ -790,10 +779,6 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) /* Received on QP0, and so by definition, this is an SMP */ struct opa_smp *smp = (struct opa_smp *)data; u16 slid = be16_to_cpu(hdr->lrh[3]); - u8 sc5; - - sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf; - sc5 |= sc4_bit; if (opa_smp_check(ibp, pkey, sc5, qp, slid, smp)) goto drop; @@ -890,9 +875,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) } wc.slid = be16_to_cpu(hdr->lrh[3]); - sc = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf; - sc |= sc4_bit; - wc.sl = ibp->sc_to_sl[sc]; + wc.sl = ibp->sc_to_sl[sc5]; /* * Save the LMC lower bits if the destination LID is a unicast LID. diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.c b/drivers/infiniband/hw/hfi1/verbs_txreq.c index bc95c4112c61..d8fb056526f8 100644 --- a/drivers/infiniband/hw/hfi1/verbs_txreq.c +++ b/drivers/infiniband/hw/hfi1/verbs_txreq.c @@ -92,11 +92,10 @@ void hfi1_put_txreq(struct verbs_txreq *tx) struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev, struct rvt_qp *qp) + __must_hold(&qp->s_lock) { struct verbs_txreq *tx = ERR_PTR(-EBUSY); - unsigned long flags; - spin_lock_irqsave(&qp->s_lock, flags); write_seqlock(&dev->iowait_lock); if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { struct hfi1_qp_priv *priv; @@ -116,7 +115,6 @@ struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev, } out: write_sequnlock(&dev->iowait_lock); - spin_unlock_irqrestore(&qp->s_lock, flags); return tx; } diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.h b/drivers/infiniband/hw/hfi1/verbs_txreq.h index 1cf69b2fe4a5..a1d6e0807f97 100644 --- a/drivers/infiniband/hw/hfi1/verbs_txreq.h +++ b/drivers/infiniband/hw/hfi1/verbs_txreq.h @@ -73,6 +73,7 @@ struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev, static inline struct verbs_txreq *get_txreq(struct hfi1_ibdev *dev, struct rvt_qp *qp) + __must_hold(&qp->slock) { struct verbs_txreq *tx; struct hfi1_qp_priv *priv = qp->priv; diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h index 8b9532034558..b738acdb9b02 100644 --- a/drivers/infiniband/hw/i40iw/i40iw.h +++ b/drivers/infiniband/hw/i40iw/i40iw.h @@ -113,6 +113,8 @@ #define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types) #define IW_CFG_FPM_QP_COUNT 32768 +#define I40IW_MAX_PAGES_PER_FMR 512 +#define I40IW_MIN_PAGES_PER_FMR 1 #define I40IW_MTU_TO_MSS 40 #define I40IW_DEFAULT_MSS 1460 diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c index c963cad92f5a..6e9081380a27 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_main.c +++ b/drivers/infiniband/hw/i40iw/i40iw_main.c @@ -600,8 +600,7 @@ static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev) cqp_init_info.scratch_array = cqp->scratch_array; status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info); if (status) { - i40iw_pr_err("cqp init status %d maj_err %d min_err %d\n", - status, maj_err, min_err); + i40iw_pr_err("cqp init status %d\n", status); goto exit; } status = dev->cqp_ops->cqp_create(dev->cqp, true, &maj_err, &min_err); diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 02a735b64208..283b64c942ee 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -79,6 +79,7 @@ static int i40iw_query_device(struct ib_device *ibdev, props->max_qp_init_rd_atom = props->max_qp_rd_atom; props->atomic_cap = IB_ATOMIC_NONE; props->max_map_per_fmr = 1; + props->max_fast_reg_page_list_len = I40IW_MAX_PAGES_PER_FMR; return 0; } @@ -1473,6 +1474,7 @@ static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT; info->pd_id = iwpd->sc_pd.pd_id; info->total_len = iwmr->length; + info->remote_access = true; cqp_info->cqp_cmd = OP_ALLOC_STAG; cqp_info->post_sq = 1; cqp_info->in.u.alloc_stag.dev = &iwdev->sc_dev; @@ -1527,7 +1529,7 @@ static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, mutex_lock(&iwdev->pbl_mutex); status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt); mutex_unlock(&iwdev->pbl_mutex); - if (!status) + if (status) goto err1; if (palloc->level != I40IW_LEVEL_1) @@ -2149,6 +2151,7 @@ static int i40iw_post_send(struct ib_qp *ibqp, struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev; struct i40iw_fast_reg_stag_info info; + memset(&info, 0, sizeof(info)); info.access_rights = I40IW_ACCESS_FLAGS_LOCALREAD; info.access_rights |= i40iw_get_user_access(flags); info.stag_key = reg_wr(ib_wr)->key & 0xff; @@ -2158,10 +2161,14 @@ static int i40iw_post_send(struct ib_qp *ibqp, info.addr_type = I40IW_ADDR_TYPE_VA_BASED; info.va = (void *)(uintptr_t)iwmr->ibmr.iova; info.total_len = iwmr->ibmr.length; + info.reg_addr_pa = *(u64 *)palloc->level1.addr; info.first_pm_pbl_index = palloc->level1.idx; info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; info.signaled = ib_wr->send_flags & IB_SEND_SIGNALED; + if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR) + info.chunk_size = 1; + if (page_shift == 21) info.page_size = 1; /* 2M page */ @@ -2327,13 +2334,16 @@ static int i40iw_req_notify_cq(struct ib_cq *ibcq, { struct i40iw_cq *iwcq; struct i40iw_cq_uk *ukcq; - enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_SOLICITED; + unsigned long flags; + enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_EVENT; iwcq = (struct i40iw_cq *)ibcq; ukcq = &iwcq->sc_cq.cq_uk; - if (notify_flags == IB_CQ_NEXT_COMP) - cq_notify = IW_CQ_COMPL_EVENT; + if (notify_flags == IB_CQ_SOLICITED) + cq_notify = IW_CQ_COMPL_SOLICITED; + spin_lock_irqsave(&iwcq->lock, flags); ukcq->ops.iw_cq_request_notification(ukcq, cq_notify); + spin_unlock_irqrestore(&iwcq->lock, flags); return 0; } diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c index 105246fba2e7..5fc623362731 100644 --- a/drivers/infiniband/hw/mlx4/ah.c +++ b/drivers/infiniband/hw/mlx4/ah.c @@ -47,6 +47,7 @@ static struct ib_ah *create_ib_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, ah->av.ib.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); ah->av.ib.g_slid = ah_attr->src_path_bits; + ah->av.ib.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); if (ah_attr->ah_flags & IB_AH_GRH) { ah->av.ib.g_slid |= 0x80; ah->av.ib.gid_index = ah_attr->grh.sgid_index; @@ -64,7 +65,6 @@ static struct ib_ah *create_ib_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, !(1 << ah->av.ib.stat_rate & dev->caps.stat_rate_support)) --ah->av.ib.stat_rate; } - ah->av.ib.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); return &ah->ibah; } diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c index d68f506c1922..9c2e53d28f98 100644 --- a/drivers/infiniband/hw/mlx4/mad.c +++ b/drivers/infiniband/hw/mlx4/mad.c @@ -527,7 +527,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, tun_tx_ix = (++tun_qp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1); spin_unlock(&tun_qp->tx_lock); if (ret) - goto out; + goto end; tun_mad = (struct mlx4_rcv_tunnel_mad *) (tun_qp->tx_ring[tun_tx_ix].buf.addr); if (tun_qp->tx_ring[tun_tx_ix].ah) @@ -596,9 +596,15 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port, wr.wr.send_flags = IB_SEND_SIGNALED; ret = ib_post_send(src_qp, &wr.wr, &bad_wr); -out: - if (ret) - ib_destroy_ah(ah); + if (!ret) + return 0; + out: + spin_lock(&tun_qp->tx_lock); + tun_qp->tx_ix_tail++; + spin_unlock(&tun_qp->tx_lock); + tun_qp->tx_ring[tun_tx_ix].ah = NULL; +end: + ib_destroy_ah(ah); return ret; } @@ -1326,9 +1332,15 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port, ret = ib_post_send(send_qp, &wr.wr, &bad_wr); + if (!ret) + return 0; + + spin_lock(&sqp->tx_lock); + sqp->tx_ix_tail++; + spin_unlock(&sqp->tx_lock); + sqp->tx_ring[wire_tx_ix].ah = NULL; out: - if (ret) - ib_destroy_ah(ah); + ib_destroy_ah(ah); return ret; } diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 0eb09e104542..42a46078d7d5 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1704,6 +1704,9 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, struct mlx4_dev *dev = (to_mdev(qp->device))->dev; int is_bonded = mlx4_is_bonded(dev); + if (flow_attr->port < 1 || flow_attr->port > qp->device->phys_port_cnt) + return ERR_PTR(-EINVAL); + if ((flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) && (flow_attr->type != IB_FLOW_ATTR_NORMAL)) return ERR_PTR(-EOPNOTSUPP); diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 6c5ac5d8f32f..29acda249612 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -139,7 +139,7 @@ struct mlx4_ib_mr { u32 max_pages; struct mlx4_mr mmr; struct ib_umem *umem; - void *pages_alloc; + size_t page_map_size; }; struct mlx4_ib_mw { diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 631272172a0b..5d73989d9771 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -277,20 +277,23 @@ mlx4_alloc_priv_pages(struct ib_device *device, struct mlx4_ib_mr *mr, int max_pages) { - int size = max_pages * sizeof(u64); - int add_size; int ret; - add_size = max_t(int, MLX4_MR_PAGES_ALIGN - ARCH_KMALLOC_MINALIGN, 0); + /* Ensure that size is aligned to DMA cacheline + * requirements. + * max_pages is limited to MLX4_MAX_FAST_REG_PAGES + * so page_map_size will never cross PAGE_SIZE. + */ + mr->page_map_size = roundup(max_pages * sizeof(u64), + MLX4_MR_PAGES_ALIGN); - mr->pages_alloc = kzalloc(size + add_size, GFP_KERNEL); - if (!mr->pages_alloc) + /* Prevent cross page boundary allocation. */ + mr->pages = (__be64 *)get_zeroed_page(GFP_KERNEL); + if (!mr->pages) return -ENOMEM; - mr->pages = PTR_ALIGN(mr->pages_alloc, MLX4_MR_PAGES_ALIGN); - mr->page_map = dma_map_single(device->dma_device, mr->pages, - size, DMA_TO_DEVICE); + mr->page_map_size, DMA_TO_DEVICE); if (dma_mapping_error(device->dma_device, mr->page_map)) { ret = -ENOMEM; @@ -298,9 +301,9 @@ mlx4_alloc_priv_pages(struct ib_device *device, } return 0; -err: - kfree(mr->pages_alloc); +err: + free_page((unsigned long)mr->pages); return ret; } @@ -309,11 +312,10 @@ mlx4_free_priv_pages(struct mlx4_ib_mr *mr) { if (mr->pages) { struct ib_device *device = mr->ibmr.device; - int size = mr->max_pages * sizeof(u64); dma_unmap_single(device->dma_device, mr->page_map, - size, DMA_TO_DEVICE); - kfree(mr->pages_alloc); + mr->page_map_size, DMA_TO_DEVICE); + free_page((unsigned long)mr->pages); mr->pages = NULL; } } @@ -537,14 +539,12 @@ int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, mr->npages = 0; ib_dma_sync_single_for_cpu(ibmr->device, mr->page_map, - sizeof(u64) * mr->max_pages, - DMA_TO_DEVICE); + mr->page_map_size, DMA_TO_DEVICE); rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, mlx4_set_page); ib_dma_sync_single_for_device(ibmr->device, mr->page_map, - sizeof(u64) * mr->max_pages, - DMA_TO_DEVICE); + mr->page_map_size, DMA_TO_DEVICE); return rc; } diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 81b0e1fbec1d..768085f59566 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -232,7 +232,7 @@ static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size) } } else { ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); - s = (ctrl->fence_size & 0x3f) << 4; + s = (ctrl->qpn_vlan.fence_size & 0x3f) << 4; for (i = 64; i < s; i += 64) { wqe = buf + i; *wqe = cpu_to_be32(0xffffffff); @@ -264,7 +264,7 @@ static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size) inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl)); } ctrl->srcrb_flags = 0; - ctrl->fence_size = size / 16; + ctrl->qpn_vlan.fence_size = size / 16; /* * Make sure descriptor is fully written before setting ownership bit * (because HW can start executing as soon as we do). @@ -362,7 +362,7 @@ static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags) sizeof (struct mlx4_wqe_raddr_seg); case MLX4_IB_QPT_RC: return sizeof (struct mlx4_wqe_ctrl_seg) + - sizeof (struct mlx4_wqe_atomic_seg) + + sizeof (struct mlx4_wqe_masked_atomic_seg) + sizeof (struct mlx4_wqe_raddr_seg); case MLX4_IB_QPT_SMI: case MLX4_IB_QPT_GSI: @@ -1191,8 +1191,10 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd, { err = create_qp_common(to_mdev(pd->device), pd, init_attr, udata, 0, &qp, gfp); - if (err) + if (err) { + kfree(qp); return ERR_PTR(err); + } qp->ibqp.qp_num = qp->mqp.qpn; qp->xrcdn = xrcdn; @@ -1990,7 +1992,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, ctrl = get_send_wqe(qp, i); ctrl->owner_opcode = cpu_to_be32(1 << 31); if (qp->sq_max_wqes_per_wr == 1) - ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4); + ctrl->qpn_vlan.fence_size = + 1 << (qp->sq.wqe_shift - 4); stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift); } @@ -3167,8 +3170,8 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, wmb(); *lso_wqe = lso_hdr_sz; - ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ? - MLX4_WQE_CTRL_FENCE : 0) | size; + ctrl->qpn_vlan.fence_size = (wr->send_flags & IB_SEND_FENCE ? + MLX4_WQE_CTRL_FENCE : 0) | size; /* * Make sure descriptor is fully written before diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c index 1534af113058..364aab9f3c9e 100644 --- a/drivers/infiniband/hw/mlx5/mad.c +++ b/drivers/infiniband/hw/mlx5/mad.c @@ -121,7 +121,7 @@ static void pma_cnt_ext_assign(struct ib_pma_portcounters_ext *pma_cnt_ext, pma_cnt_ext->port_xmit_data = cpu_to_be64(MLX5_SUM_CNT(out, transmitted_ib_unicast.octets, transmitted_ib_multicast.octets) >> 2); - pma_cnt_ext->port_xmit_data = + pma_cnt_ext->port_rcv_data = cpu_to_be64(MLX5_SUM_CNT(out, received_ib_unicast.octets, received_ib_multicast.octets) >> 2); pma_cnt_ext->port_xmit_packets = diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index b48ad85315dc..dad63f038bb8 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1528,21 +1528,18 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, { struct mlx5_flow_table *ft = ft_prio->flow_table; struct mlx5_ib_flow_handler *handler; + struct mlx5_flow_spec *spec; void *ib_flow = flow_attr + 1; - u8 match_criteria_enable = 0; unsigned int spec_index; - u32 *match_c; - u32 *match_v; u32 action; int err = 0; if (!is_valid_attr(flow_attr)) return ERR_PTR(-EINVAL); - match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); - match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL); + spec = mlx5_vzalloc(sizeof(*spec)); handler = kzalloc(sizeof(*handler), GFP_KERNEL); - if (!handler || !match_c || !match_v) { + if (!handler || !spec) { err = -ENOMEM; goto free; } @@ -1550,7 +1547,8 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, INIT_LIST_HEAD(&handler->list); for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { - err = parse_flow_attr(match_c, match_v, ib_flow); + err = parse_flow_attr(spec->match_criteria, + spec->match_value, ib_flow); if (err < 0) goto free; @@ -1558,11 +1556,11 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, } /* Outer header support only */ - match_criteria_enable = (!outer_header_zero(match_c)) << 0; + spec->match_criteria_enable = (!outer_header_zero(spec->match_criteria)) + << 0; action = dst ? MLX5_FLOW_CONTEXT_ACTION_FWD_DEST : MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; - handler->rule = mlx5_add_flow_rule(ft, match_criteria_enable, - match_c, match_v, + handler->rule = mlx5_add_flow_rule(ft, spec, action, MLX5_FS_DEFAULT_FLOW_TAG, dst); @@ -1578,8 +1576,7 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, free: if (err) kfree(handler); - kfree(match_c); - kfree(match_v); + kvfree(spec); return err ? ERR_PTR(err) : handler; } diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index ce434228a5ea..ce0a7ab35a22 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3332,10 +3332,11 @@ static u8 get_fence(u8 fence, struct ib_send_wr *wr) return MLX5_FENCE_MODE_SMALL_AND_FENCE; else return fence; - - } else { - return 0; + } else if (unlikely(wr->send_flags & IB_SEND_FENCE)) { + return MLX5_FENCE_MODE_FENCE; } + + return 0; } static int begin_wqe(struct mlx5_ib_qp *qp, void **seg, diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c index ff946d5f59e4..382466a90da7 100644 --- a/drivers/infiniband/hw/qib/qib_file_ops.c +++ b/drivers/infiniband/hw/qib/qib_file_ops.c @@ -2178,6 +2178,11 @@ static ssize_t qib_write(struct file *fp, const char __user *data, switch (cmd.type) { case QIB_CMD_ASSIGN_CTXT: + if (rcd) { + ret = -EINVAL; + goto bail; + } + ret = qib_assign_ctxt(fp, &cmd.cmd.user_info); if (ret) goto bail; diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index 7de5134bec85..41ba7e9cadaa 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -369,8 +369,8 @@ static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, /* wrap to first map page, invert bit 0 */ offset = qpt->incr | ((offset & 1) ^ 1); } - /* there can be no bits at shift and below */ - WARN_ON(offset & (rdi->dparms.qos_shift - 1)); + /* there can be no set bits in low-order QoS bits */ + WARN_ON(offset & (BIT(rdi->dparms.qos_shift) - 1)); qpn = mk_qpn(qpt, map, offset); } @@ -576,12 +576,6 @@ static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, qp->s_ssn = 1; qp->s_lsn = 0; qp->s_mig_state = IB_MIG_MIGRATED; - if (qp->s_ack_queue) - memset( - qp->s_ack_queue, - 0, - rvt_max_atomic(rdi) * - sizeof(*qp->s_ack_queue)); qp->r_head_ack_queue = 0; qp->s_tail_ack_queue = 0; qp->s_num_rd_atomic = 0; @@ -705,8 +699,10 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, * initialization that is needed. */ priv = rdi->driver_f.qp_priv_alloc(rdi, qp, gfp); - if (!priv) + if (IS_ERR(priv)) { + ret = priv; goto bail_qp; + } qp->priv = priv; qp->timeout_jiffies = usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index e1cc2cc42f25..30c4fda7a05a 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -501,9 +501,7 @@ static noinline int check_support(struct rvt_dev_info *rdi, int verb) !rdi->driver_f.quiesce_qp || !rdi->driver_f.notify_error_qp || !rdi->driver_f.mtu_from_qp || - !rdi->driver_f.mtu_to_path_mtu || - !rdi->driver_f.shut_down_port || - !rdi->driver_f.cap_mask_chg) + !rdi->driver_f.mtu_to_path_mtu) return -EINVAL; break; diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index e68b20cba70b..4a4155640d51 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -1638,8 +1638,7 @@ retry: */ qp_init->cap.max_send_wr = srp_sq_size / 2; qp_init->cap.max_rdma_ctxs = srp_sq_size / 2; - qp_init->cap.max_send_sge = max(sdev->device->attrs.max_sge_rd, - sdev->device->attrs.max_sge); + qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE; qp_init->port_num = ch->sport->port; ch->qp = ib_create_qp(sdev->pd, qp_init); diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h index fee6bfd7ca21..389030487da7 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.h +++ b/drivers/infiniband/ulp/srpt/ib_srpt.h @@ -106,6 +106,7 @@ enum { SRP_LOGIN_RSP_MULTICHAN_MAINTAINED = 0x2, SRPT_DEF_SG_TABLESIZE = 128, + SRPT_DEF_SG_PER_WQE = 16, MIN_SRPT_SQ_SIZE = 16, DEF_SRPT_SQ_SIZE = 4096, diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 804dbcc37d3f..a529a4535457 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -1031,17 +1031,17 @@ static int xpad_play_effect(struct input_dev *dev, void *data, struct ff_effect case XTYPE_XBOXONE: packet->data[0] = 0x09; /* activate rumble */ - packet->data[1] = 0x08; + packet->data[1] = 0x00; packet->data[2] = xpad->odata_serial++; - packet->data[3] = 0x08; /* continuous effect */ - packet->data[4] = 0x00; /* simple rumble mode */ - packet->data[5] = 0x03; /* L and R actuator only */ - packet->data[6] = 0x00; /* TODO: LT actuator */ - packet->data[7] = 0x00; /* TODO: RT actuator */ + packet->data[3] = 0x09; + packet->data[4] = 0x00; + packet->data[5] = 0x0F; + packet->data[6] = 0x00; + packet->data[7] = 0x00; packet->data[8] = strong / 512; /* left actuator */ packet->data[9] = weak / 512; /* right actuator */ - packet->data[10] = 0x80; /* length of pulse */ - packet->data[11] = 0x00; /* stop period of pulse */ + packet->data[10] = 0xFF; + packet->data[11] = 0x00; packet->data[12] = 0x00; packet->len = 13; packet->pending = true; @@ -1431,22 +1431,15 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id int ep_irq_in_idx; int i, error; + if (intf->cur_altsetting->desc.bNumEndpoints != 2) + return -ENODEV; + for (i = 0; xpad_device[i].idVendor; i++) { if ((le16_to_cpu(udev->descriptor.idVendor) == xpad_device[i].idVendor) && (le16_to_cpu(udev->descriptor.idProduct) == xpad_device[i].idProduct)) break; } - if (xpad_device[i].xtype == XTYPE_XBOXONE && - intf->cur_altsetting->desc.bInterfaceNumber != 0) { - /* - * The Xbox One controller lists three interfaces all with the - * same interface class, subclass and protocol. Differentiate by - * interface number. - */ - return -ENODEV; - } - xpad = kzalloc(sizeof(struct usb_xpad), GFP_KERNEL); if (!xpad) return -ENOMEM; @@ -1478,6 +1471,8 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id if (intf->cur_altsetting->desc.bInterfaceClass == USB_CLASS_VENDOR_SPEC) { if (intf->cur_altsetting->desc.bInterfaceProtocol == 129) xpad->xtype = XTYPE_XBOX360W; + else if (intf->cur_altsetting->desc.bInterfaceProtocol == 208) + xpad->xtype = XTYPE_XBOXONE; else xpad->xtype = XTYPE_XBOX360; } else { @@ -1492,6 +1487,17 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id xpad->mapping |= MAP_STICKS_TO_NULL; } + if (xpad->xtype == XTYPE_XBOXONE && + intf->cur_altsetting->desc.bInterfaceNumber != 0) { + /* + * The Xbox One controller lists three interfaces all with the + * same interface class, subclass and protocol. Differentiate by + * interface number. + */ + error = -ENODEV; + goto err_free_in_urb; + } + error = xpad_init_output(intf, xpad); if (error) goto err_free_in_urb; diff --git a/drivers/input/mouse/elantech.c b/drivers/input/mouse/elantech.c index 78f93cf68840..be5b399da5d3 100644 --- a/drivers/input/mouse/elantech.c +++ b/drivers/input/mouse/elantech.c @@ -1568,13 +1568,7 @@ static int elantech_set_properties(struct elantech_data *etd) case 5: etd->hw_version = 3; break; - case 6: - case 7: - case 8: - case 9: - case 10: - case 13: - case 14: + case 6 ... 14: etd->hw_version = 4; break; default: diff --git a/drivers/input/mouse/vmmouse.c b/drivers/input/mouse/vmmouse.c index a3f0f5a47490..0f586780ceb4 100644 --- a/drivers/input/mouse/vmmouse.c +++ b/drivers/input/mouse/vmmouse.c @@ -355,18 +355,11 @@ int vmmouse_detect(struct psmouse *psmouse, bool set_properties) return -ENXIO; } - if (!request_region(VMMOUSE_PROTO_PORT, 4, "vmmouse")) { - psmouse_dbg(psmouse, "VMMouse port in use.\n"); - return -EBUSY; - } - /* Check if the device is present */ response = ~VMMOUSE_PROTO_MAGIC; VMMOUSE_CMD(GETVERSION, 0, version, response, dummy1, dummy2); - if (response != VMMOUSE_PROTO_MAGIC || version == 0xffffffffU) { - release_region(VMMOUSE_PROTO_PORT, 4); + if (response != VMMOUSE_PROTO_MAGIC || version == 0xffffffffU) return -ENXIO; - } if (set_properties) { psmouse->vendor = VMMOUSE_VENDOR; @@ -374,8 +367,6 @@ int vmmouse_detect(struct psmouse *psmouse, bool set_properties) psmouse->model = version; } - release_region(VMMOUSE_PROTO_PORT, 4); - return 0; } @@ -394,7 +385,6 @@ static void vmmouse_disconnect(struct psmouse *psmouse) psmouse_reset(psmouse); input_unregister_device(priv->abs_dev); kfree(priv); - release_region(VMMOUSE_PROTO_PORT, 4); } /** @@ -438,15 +428,10 @@ int vmmouse_init(struct psmouse *psmouse) struct input_dev *rel_dev = psmouse->dev, *abs_dev; int error; - if (!request_region(VMMOUSE_PROTO_PORT, 4, "vmmouse")) { - psmouse_dbg(psmouse, "VMMouse port in use.\n"); - return -EBUSY; - } - psmouse_reset(psmouse); error = vmmouse_enable(psmouse); if (error) - goto release_region; + return error; priv = kzalloc(sizeof(*priv), GFP_KERNEL); abs_dev = input_allocate_device(); @@ -502,8 +487,5 @@ init_fail: kfree(priv); psmouse->private = NULL; -release_region: - release_region(VMMOUSE_PROTO_PORT, 4); - return error; } diff --git a/drivers/input/rmi4/rmi_bus.c b/drivers/input/rmi4/rmi_bus.c index b368b0515c5a..253df96be427 100644 --- a/drivers/input/rmi4/rmi_bus.c +++ b/drivers/input/rmi4/rmi_bus.c @@ -157,11 +157,11 @@ static int rmi_function_match(struct device *dev, struct device_driver *drv) static void rmi_function_of_probe(struct rmi_function *fn) { char of_name[9]; + struct device_node *node = fn->rmi_dev->xport->dev->of_node; snprintf(of_name, sizeof(of_name), "rmi4-f%02x", fn->fd.function_number); - fn->dev.of_node = of_find_node_by_name( - fn->rmi_dev->xport->dev->of_node, of_name); + fn->dev.of_node = of_get_child_by_name(node, of_name); } #else static inline void rmi_function_of_probe(struct rmi_function *fn) diff --git a/drivers/input/rmi4/rmi_f12.c b/drivers/input/rmi4/rmi_f12.c index 8dd3fb5e1f94..88e91559c84e 100644 --- a/drivers/input/rmi4/rmi_f12.c +++ b/drivers/input/rmi4/rmi_f12.c @@ -66,7 +66,7 @@ static int rmi_f12_read_sensor_tuning(struct f12_data *f12) struct rmi_device *rmi_dev = fn->rmi_dev; int ret; int offset; - u8 buf[14]; + u8 buf[15]; int pitch_x = 0; int pitch_y = 0; int clip_x_low = 0; @@ -86,9 +86,10 @@ static int rmi_f12_read_sensor_tuning(struct f12_data *f12) offset = rmi_register_desc_calc_reg_offset(&f12->control_reg_desc, 8); - if (item->reg_size > 14) { - dev_err(&fn->dev, "F12 control8 should be 14 bytes, not: %ld\n", - item->reg_size); + if (item->reg_size > sizeof(buf)) { + dev_err(&fn->dev, + "F12 control8 should be no bigger than %zd bytes, not: %ld\n", + sizeof(buf), item->reg_size); return -ENODEV; } diff --git a/drivers/input/touchscreen/sur40.c b/drivers/input/touchscreen/sur40.c index 880c40b23f66..4ea475775d58 100644 --- a/drivers/input/touchscreen/sur40.c +++ b/drivers/input/touchscreen/sur40.c @@ -126,7 +126,7 @@ struct sur40_image_header { #define VIDEO_PACKET_SIZE 16384 /* polling interval (ms) */ -#define POLL_INTERVAL 4 +#define POLL_INTERVAL 1 /* maximum number of contacts FIXME: this is a guess? */ #define MAX_CONTACTS 64 @@ -151,7 +151,6 @@ struct sur40_state { struct mutex lock; struct vb2_queue queue; - struct vb2_alloc_ctx *alloc_ctx; struct list_head buf_list; spinlock_t qlock; int sequence; @@ -448,7 +447,7 @@ static void sur40_process_video(struct sur40_state *sur40) /* return error if streaming was stopped in the meantime */ if (sur40->sequence == -1) - goto err_poll; + return; /* mark as finished */ new_buf->vb.vb2_buf.timestamp = ktime_get_ns(); @@ -580,19 +579,13 @@ static int sur40_probe(struct usb_interface *interface, sur40->queue = sur40_queue; sur40->queue.drv_priv = sur40; sur40->queue.lock = &sur40->lock; + sur40->queue.dev = sur40->dev; /* initialize the queue */ error = vb2_queue_init(&sur40->queue); if (error) goto err_unreg_v4l2; - sur40->alloc_ctx = vb2_dma_sg_init_ctx(sur40->dev); - if (IS_ERR(sur40->alloc_ctx)) { - dev_err(sur40->dev, "Can't allocate buffer context"); - error = PTR_ERR(sur40->alloc_ctx); - goto err_unreg_v4l2; - } - sur40->vdev = sur40_video_device; sur40->vdev.v4l2_dev = &sur40->v4l2; sur40->vdev.lock = &sur40->lock; @@ -633,7 +626,6 @@ static void sur40_disconnect(struct usb_interface *interface) video_unregister_device(&sur40->vdev); v4l2_device_unregister(&sur40->v4l2); - vb2_dma_sg_cleanup_ctx(sur40->alloc_ctx); input_unregister_polled_device(sur40->input); input_free_polled_device(sur40->input); @@ -653,13 +645,10 @@ static void sur40_disconnect(struct usb_interface *interface) */ static int sur40_queue_setup(struct vb2_queue *q, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { - struct sur40_state *sur40 = vb2_get_drv_priv(q); - if (q->num_buffers + *nbuffers < 3) *nbuffers = 3 - q->num_buffers; - alloc_ctxs[0] = sur40->alloc_ctx; if (*nplanes) return sizes[0] < sur40_video_format.sizeimage ? -EINVAL : 0; @@ -736,6 +725,7 @@ static int sur40_start_streaming(struct vb2_queue *vq, unsigned int count) static void sur40_stop_streaming(struct vb2_queue *vq) { struct sur40_state *sur40 = vb2_get_drv_priv(vq); + vb2_wait_for_all_buffers(vq); sur40->sequence = -1; /* Release all active buffers */ @@ -793,7 +783,6 @@ static int sur40_vidioc_enum_fmt(struct file *file, void *priv, { if (f->index != 0) return -EINVAL; - strlcpy(f->description, "8-bit greyscale", sizeof(f->description)); f->pixelformat = V4L2_PIX_FMT_GREY; f->flags = 0; return 0; diff --git a/drivers/input/touchscreen/ts4800-ts.c b/drivers/input/touchscreen/ts4800-ts.c index 3c3dd78303be..fed73eeb47b3 100644 --- a/drivers/input/touchscreen/ts4800-ts.c +++ b/drivers/input/touchscreen/ts4800-ts.c @@ -118,6 +118,13 @@ static int ts4800_parse_dt(struct platform_device *pdev, return -ENODEV; } + ts->regmap = syscon_node_to_regmap(syscon_np); + of_node_put(syscon_np); + if (IS_ERR(ts->regmap)) { + dev_err(dev, "cannot get parent's regmap\n"); + return PTR_ERR(ts->regmap); + } + error = of_property_read_u32_index(np, "syscon", 1, ®); if (error < 0) { dev_err(dev, "no offset in syscon\n"); @@ -134,12 +141,6 @@ static int ts4800_parse_dt(struct platform_device *pdev, ts->bit = BIT(bit); - ts->regmap = syscon_node_to_regmap(syscon_np); - if (IS_ERR(ts->regmap)) { - dev_err(dev, "cannot get parent's regmap\n"); - return PTR_ERR(ts->regmap); - } - return 0; } diff --git a/drivers/input/touchscreen/tsc2004.c b/drivers/input/touchscreen/tsc2004.c index 7295c198aa08..6fe55d598fac 100644 --- a/drivers/input/touchscreen/tsc2004.c +++ b/drivers/input/touchscreen/tsc2004.c @@ -22,6 +22,11 @@ #include #include "tsc200x-core.h" +static const struct input_id tsc2004_input_id = { + .bustype = BUS_I2C, + .product = 2004, +}; + static int tsc2004_cmd(struct device *dev, u8 cmd) { u8 tx = TSC200X_CMD | TSC200X_CMD_12BIT | cmd; @@ -42,7 +47,7 @@ static int tsc2004_probe(struct i2c_client *i2c, const struct i2c_device_id *id) { - return tsc200x_probe(&i2c->dev, i2c->irq, BUS_I2C, + return tsc200x_probe(&i2c->dev, i2c->irq, &tsc2004_input_id, devm_regmap_init_i2c(i2c, &tsc200x_regmap_config), tsc2004_cmd); } diff --git a/drivers/input/touchscreen/tsc2005.c b/drivers/input/touchscreen/tsc2005.c index b9f593dfd2ef..f2c5f0e47f77 100644 --- a/drivers/input/touchscreen/tsc2005.c +++ b/drivers/input/touchscreen/tsc2005.c @@ -24,6 +24,11 @@ #include #include "tsc200x-core.h" +static const struct input_id tsc2005_input_id = { + .bustype = BUS_SPI, + .product = 2005, +}; + static int tsc2005_cmd(struct device *dev, u8 cmd) { u8 tx = TSC200X_CMD | TSC200X_CMD_12BIT | cmd; @@ -62,7 +67,7 @@ static int tsc2005_probe(struct spi_device *spi) if (error) return error; - return tsc200x_probe(&spi->dev, spi->irq, BUS_SPI, + return tsc200x_probe(&spi->dev, spi->irq, &tsc2005_input_id, devm_regmap_init_spi(spi, &tsc200x_regmap_config), tsc2005_cmd); } diff --git a/drivers/input/touchscreen/tsc200x-core.c b/drivers/input/touchscreen/tsc200x-core.c index 15240c1ee850..dfa7f1c4f545 100644 --- a/drivers/input/touchscreen/tsc200x-core.c +++ b/drivers/input/touchscreen/tsc200x-core.c @@ -450,7 +450,7 @@ static void tsc200x_close(struct input_dev *input) mutex_unlock(&ts->mutex); } -int tsc200x_probe(struct device *dev, int irq, __u16 bustype, +int tsc200x_probe(struct device *dev, int irq, const struct input_id *tsc_id, struct regmap *regmap, int (*tsc200x_cmd)(struct device *dev, u8 cmd)) { @@ -547,9 +547,18 @@ int tsc200x_probe(struct device *dev, int irq, __u16 bustype, snprintf(ts->phys, sizeof(ts->phys), "%s/input-ts", dev_name(dev)); - input_dev->name = "TSC200X touchscreen"; + if (tsc_id->product == 2004) { + input_dev->name = "TSC200X touchscreen"; + } else { + input_dev->name = devm_kasprintf(dev, GFP_KERNEL, + "TSC%04d touchscreen", + tsc_id->product); + if (!input_dev->name) + return -ENOMEM; + } + input_dev->phys = ts->phys; - input_dev->id.bustype = bustype; + input_dev->id = *tsc_id; input_dev->dev.parent = dev; input_dev->evbit[0] = BIT(EV_ABS) | BIT(EV_KEY); input_dev->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH); diff --git a/drivers/input/touchscreen/tsc200x-core.h b/drivers/input/touchscreen/tsc200x-core.h index 7a482d102614..49a63a3c6840 100644 --- a/drivers/input/touchscreen/tsc200x-core.h +++ b/drivers/input/touchscreen/tsc200x-core.h @@ -70,7 +70,7 @@ extern const struct regmap_config tsc200x_regmap_config; extern const struct dev_pm_ops tsc200x_pm_ops; -int tsc200x_probe(struct device *dev, int irq, __u16 bustype, +int tsc200x_probe(struct device *dev, int irq, const struct input_id *tsc_id, struct regmap *regmap, int (*tsc200x_cmd)(struct device *dev, u8 cmd)); int tsc200x_remove(struct device *dev); diff --git a/drivers/input/touchscreen/wacom_w8001.c b/drivers/input/touchscreen/wacom_w8001.c index bab3c6acf6a2..b6fc4bde79de 100644 --- a/drivers/input/touchscreen/wacom_w8001.c +++ b/drivers/input/touchscreen/wacom_w8001.c @@ -27,7 +27,7 @@ MODULE_AUTHOR("Jaya Kumar "); MODULE_DESCRIPTION(DRIVER_DESC); MODULE_LICENSE("GPL"); -#define W8001_MAX_LENGTH 11 +#define W8001_MAX_LENGTH 13 #define W8001_LEAD_MASK 0x80 #define W8001_LEAD_BYTE 0x80 #define W8001_TAB_MASK 0x40 @@ -155,6 +155,7 @@ static void parse_multi_touch(struct w8001 *w8001) bool touch = data[0] & (1 << i); input_mt_slot(dev, i); + input_mt_report_slot_state(dev, MT_TOOL_FINGER, touch); if (touch) { x = (data[6 * i + 1] << 7) | data[6 * i + 2]; y = (data[6 * i + 3] << 7) | data[6 * i + 4]; @@ -339,6 +340,15 @@ static irqreturn_t w8001_interrupt(struct serio *serio, w8001->idx = 0; parse_multi_touch(w8001); break; + + default: + /* + * ThinkPad X60 Tablet PC (pen only device) sometimes + * sends invalid data packets that are larger than + * W8001_PKTLEN_TPCPEN. Let's start over again. + */ + if (!w8001->touch_dev && w8001->idx > W8001_PKTLEN_TPCPEN - 1) + w8001->idx = 0; } return IRQ_HANDLED; @@ -513,6 +523,8 @@ static int w8001_setup_touch(struct w8001 *w8001, char *basename, 0, touch.x, 0, 0); input_set_abs_params(dev, ABS_MT_POSITION_Y, 0, touch.y, 0, 0); + input_set_abs_params(dev, ABS_MT_TOOL_TYPE, + 0, MT_TOOL_MAX, 0, 0); strlcat(basename, " 2FG", basename_sz); if (w8001->max_pen_x && w8001->max_pen_y) diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 9e0034196e10..59741ead7e15 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -1107,13 +1107,13 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; } + devid = e->devid; DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %02x:%02x.%x\n", hid, uid, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid)); - devid = e->devid; flags = e->flags; ret = add_acpi_hid_device(hid, uid, &devid, false); @@ -1568,13 +1568,23 @@ static int __init amd_iommu_init_pci(void) break; } + /* + * Order is important here to make sure any unity map requirements are + * fulfilled. The unity mappings are created and written to the device + * table during the amd_iommu_init_api() call. + * + * After that we call init_device_table_dma() to make sure any + * uninitialized DTE will block DMA, and in the end we flush the caches + * of all IOMMUs to make sure the changes to the device table are + * active. + */ + ret = amd_iommu_init_api(); + init_device_table_dma(); for_each_iommu(iommu) iommu_flush_all_caches(iommu); - ret = amd_iommu_init_api(); - if (!ret) print_iommu_info(); diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 56999d2fac07..fbdaf81ae925 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -538,8 +538,7 @@ static void do_fault(struct work_struct *work) if (access_error(vma, fault)) goto out; - ret = handle_mm_fault(mm, vma, address, flags); - + ret = handle_mm_fault(vma, address, flags); out: up_read(&mm->mmap_sem); diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 10700945994e..323dac9900ba 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4602,13 +4602,13 @@ static void free_all_cpu_cached_iovas(unsigned int cpu) for (i = 0; i < g_num_of_iommus; i++) { struct intel_iommu *iommu = g_iommus[i]; struct dmar_domain *domain; - u16 did; + int did; if (!iommu) continue; - for (did = 0; did < 0xffff; did++) { - domain = get_iommu_domain(iommu, did); + for (did = 0; did < cap_ndoms(iommu->cap); did++) { + domain = get_iommu_domain(iommu, (u16)did); if (!domain) continue; diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index d9939fa9b588..8ebb3530afa7 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -583,7 +583,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) if (access_error(vma, req)) goto invalid; - ret = handle_mm_fault(svm->mm, vma, address, + ret = handle_mm_fault(vma, address, req->wr_req ? FAULT_FLAG_WRITE : 0); if (ret & VM_FAULT_ERROR) goto invalid; diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index ba764a0835d3..e23001bfcfee 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -420,8 +420,10 @@ retry: /* Try replenishing IOVAs by flushing rcache. */ flushed_rcache = true; + preempt_disable(); for_each_online_cpu(cpu) free_cpu_cached_iovas(cpu, iovad); + preempt_enable(); goto retry; } @@ -749,7 +751,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, bool can_insert = false; unsigned long flags; - cpu_rcache = this_cpu_ptr(rcache->cpu_rcaches); + cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches); spin_lock_irqsave(&cpu_rcache->lock, flags); if (!iova_magazine_full(cpu_rcache->loaded)) { @@ -779,6 +781,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, iova_magazine_push(cpu_rcache->loaded, iova_pfn); spin_unlock_irqrestore(&cpu_rcache->lock, flags); + put_cpu_ptr(rcache->cpu_rcaches); if (mag_to_free) { iova_magazine_free_pfns(mag_to_free, iovad); @@ -812,7 +815,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, bool has_pfn = false; unsigned long flags; - cpu_rcache = this_cpu_ptr(rcache->cpu_rcaches); + cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches); spin_lock_irqsave(&cpu_rcache->lock, flags); if (!iova_magazine_empty(cpu_rcache->loaded)) { @@ -834,6 +837,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn); spin_unlock_irqrestore(&cpu_rcache->lock, flags); + put_cpu_ptr(rcache->cpu_rcaches); return iova_pfn; } diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index fa33c50b0e5a..5495a5ba8039 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -8,6 +8,12 @@ config ARM_GIC select IRQ_DOMAIN_HIERARCHY select MULTI_IRQ_HANDLER +config ARM_GIC_PM + bool + depends on PM + select ARM_GIC + select PM_CLK + config ARM_GIC_MAX_NR int default 2 if ARCH_REALVIEW diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 38853a187607..4c203b6b8163 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_ARCH_SUNXI) += irq-sun4i.o obj-$(CONFIG_ARCH_SUNXI) += irq-sunxi-nmi.o obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o +obj-$(CONFIG_ARM_GIC_PM) += irq-gic-pm.o obj-$(CONFIG_REALVIEW_DT) += irq-gic-realview.o obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-common.o @@ -69,3 +70,4 @@ obj-$(CONFIG_PIC32_EVIC) += irq-pic32-evic.o obj-$(CONFIG_MVEBU_ODMI) += irq-mvebu-odmi.o obj-$(CONFIG_LS_SCFG_MSI) += irq-ls-scfg-msi.o obj-$(CONFIG_EZNPS_GIC) += irq-eznps.o +obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-vic.o diff --git a/drivers/irqchip/exynos-combiner.c b/drivers/irqchip/exynos-combiner.c index ead15be2d20a..b78a169c9c83 100644 --- a/drivers/irqchip/exynos-combiner.c +++ b/drivers/irqchip/exynos-combiner.c @@ -55,14 +55,14 @@ static void combiner_mask_irq(struct irq_data *data) { u32 mask = 1 << (data->hwirq % 32); - __raw_writel(mask, combiner_base(data) + COMBINER_ENABLE_CLEAR); + writel_relaxed(mask, combiner_base(data) + COMBINER_ENABLE_CLEAR); } static void combiner_unmask_irq(struct irq_data *data) { u32 mask = 1 << (data->hwirq % 32); - __raw_writel(mask, combiner_base(data) + COMBINER_ENABLE_SET); + writel_relaxed(mask, combiner_base(data) + COMBINER_ENABLE_SET); } static void combiner_handle_cascade_irq(struct irq_desc *desc) @@ -75,7 +75,7 @@ static void combiner_handle_cascade_irq(struct irq_desc *desc) chained_irq_enter(chip, desc); spin_lock(&irq_controller_lock); - status = __raw_readl(chip_data->base + COMBINER_INT_STATUS); + status = readl_relaxed(chip_data->base + COMBINER_INT_STATUS); spin_unlock(&irq_controller_lock); status &= chip_data->irq_mask; @@ -135,7 +135,7 @@ static void __init combiner_init_one(struct combiner_chip_data *combiner_data, combiner_data->parent_irq = irq; /* Disable all interrupts */ - __raw_writel(combiner_data->irq_mask, base + COMBINER_ENABLE_CLEAR); + writel_relaxed(combiner_data->irq_mask, base + COMBINER_ENABLE_CLEAR); } static int combiner_irq_domain_xlate(struct irq_domain *d, @@ -218,7 +218,7 @@ static int combiner_suspend(void) for (i = 0; i < max_nr; i++) combiner_data[i].pm_save = - __raw_readl(combiner_data[i].base + COMBINER_ENABLE_SET); + readl_relaxed(combiner_data[i].base + COMBINER_ENABLE_SET); return 0; } @@ -235,9 +235,9 @@ static void combiner_resume(void) int i; for (i = 0; i < max_nr; i++) { - __raw_writel(combiner_data[i].irq_mask, + writel_relaxed(combiner_data[i].irq_mask, combiner_data[i].base + COMBINER_ENABLE_CLEAR); - __raw_writel(combiner_data[i].pm_save, + writel_relaxed(combiner_data[i].pm_save, combiner_data[i].base + COMBINER_ENABLE_SET); } } diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index e7dc6cbda2a1..7c42b1d13faf 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -541,7 +541,7 @@ static void armada_370_xp_mpic_resume(void) writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); } -struct syscore_ops armada_370_xp_mpic_syscore_ops = { +static struct syscore_ops armada_370_xp_mpic_syscore_ops = { .suspend = armada_370_xp_mpic_suspend, .resume = armada_370_xp_mpic_resume, }; diff --git a/drivers/irqchip/irq-aspeed-vic.c b/drivers/irqchip/irq-aspeed-vic.c new file mode 100644 index 000000000000..d24451d5bf8a --- /dev/null +++ b/drivers/irqchip/irq-aspeed-vic.c @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2015 - Ben Herrenschmidt, IBM Corp. + * + * Driver for Aspeed "new" VIC as found in SoC generation 3 and later + * + * Based on irq-vic.c: + * + * Copyright (C) 1999 - 2003 ARM Limited + * Copyright (C) 2000 Deep Blue Solutions Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* These definitions correspond to the "new mapping" of the + * register set that interleaves "high" and "low". The offsets + * below are for the "low" register, add 4 to get to the high one + */ +#define AVIC_IRQ_STATUS 0x00 +#define AVIC_FIQ_STATUS 0x08 +#define AVIC_RAW_STATUS 0x10 +#define AVIC_INT_SELECT 0x18 +#define AVIC_INT_ENABLE 0x20 +#define AVIC_INT_ENABLE_CLR 0x28 +#define AVIC_INT_TRIGGER 0x30 +#define AVIC_INT_TRIGGER_CLR 0x38 +#define AVIC_INT_SENSE 0x40 +#define AVIC_INT_DUAL_EDGE 0x48 +#define AVIC_INT_EVENT 0x50 +#define AVIC_EDGE_CLR 0x58 +#define AVIC_EDGE_STATUS 0x60 + +#define NUM_IRQS 64 + +struct aspeed_vic { + void __iomem *base; + u32 edge_sources[2]; + struct irq_domain *dom; +}; +static struct aspeed_vic *system_avic; + +static void vic_init_hw(struct aspeed_vic *vic) +{ + u32 sense; + + /* Disable all interrupts */ + writel(0xffffffff, vic->base + AVIC_INT_ENABLE_CLR); + writel(0xffffffff, vic->base + AVIC_INT_ENABLE_CLR + 4); + + /* Make sure no soft trigger is on */ + writel(0xffffffff, vic->base + AVIC_INT_TRIGGER_CLR); + writel(0xffffffff, vic->base + AVIC_INT_TRIGGER_CLR + 4); + + /* Set everything to be IRQ */ + writel(0, vic->base + AVIC_INT_SELECT); + writel(0, vic->base + AVIC_INT_SELECT + 4); + + /* Some interrupts have a programable high/low level trigger + * (4 GPIO direct inputs), for now we assume this was configured + * by firmware. We read which ones are edge now. + */ + sense = readl(vic->base + AVIC_INT_SENSE); + vic->edge_sources[0] = ~sense; + sense = readl(vic->base + AVIC_INT_SENSE + 4); + vic->edge_sources[1] = ~sense; + + /* Clear edge detection latches */ + writel(0xffffffff, vic->base + AVIC_EDGE_CLR); + writel(0xffffffff, vic->base + AVIC_EDGE_CLR + 4); +} + +static void __exception_irq_entry avic_handle_irq(struct pt_regs *regs) +{ + struct aspeed_vic *vic = system_avic; + u32 stat, irq; + + for (;;) { + irq = 0; + stat = readl_relaxed(vic->base + AVIC_IRQ_STATUS); + if (!stat) { + stat = readl_relaxed(vic->base + AVIC_IRQ_STATUS + 4); + irq = 32; + } + if (stat == 0) + break; + irq += ffs(stat) - 1; + handle_domain_irq(vic->dom, irq, regs); + } +} + +static void avic_ack_irq(struct irq_data *d) +{ + struct aspeed_vic *vic = irq_data_get_irq_chip_data(d); + unsigned int sidx = d->hwirq >> 5; + unsigned int sbit = 1u << (d->hwirq & 0x1f); + + /* Clear edge latch for edge interrupts, nop for level */ + if (vic->edge_sources[sidx] & sbit) + writel(sbit, vic->base + AVIC_EDGE_CLR + sidx * 4); +} + +static void avic_mask_irq(struct irq_data *d) +{ + struct aspeed_vic *vic = irq_data_get_irq_chip_data(d); + unsigned int sidx = d->hwirq >> 5; + unsigned int sbit = 1u << (d->hwirq & 0x1f); + + writel(sbit, vic->base + AVIC_INT_ENABLE_CLR + sidx * 4); +} + +static void avic_unmask_irq(struct irq_data *d) +{ + struct aspeed_vic *vic = irq_data_get_irq_chip_data(d); + unsigned int sidx = d->hwirq >> 5; + unsigned int sbit = 1u << (d->hwirq & 0x1f); + + writel(sbit, vic->base + AVIC_INT_ENABLE + sidx * 4); +} + +/* For level irq, faster than going through a nop "ack" and mask */ +static void avic_mask_ack_irq(struct irq_data *d) +{ + struct aspeed_vic *vic = irq_data_get_irq_chip_data(d); + unsigned int sidx = d->hwirq >> 5; + unsigned int sbit = 1u << (d->hwirq & 0x1f); + + /* First mask */ + writel(sbit, vic->base + AVIC_INT_ENABLE_CLR + sidx * 4); + + /* Then clear edge latch for edge interrupts */ + if (vic->edge_sources[sidx] & sbit) + writel(sbit, vic->base + AVIC_EDGE_CLR + sidx * 4); +} + +static struct irq_chip avic_chip = { + .name = "AVIC", + .irq_ack = avic_ack_irq, + .irq_mask = avic_mask_irq, + .irq_unmask = avic_unmask_irq, + .irq_mask_ack = avic_mask_ack_irq, +}; + +static int avic_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) +{ + struct aspeed_vic *vic = d->host_data; + unsigned int sidx = hwirq >> 5; + unsigned int sbit = 1u << (hwirq & 0x1f); + + /* Check if interrupt exists */ + if (sidx > 1) + return -EPERM; + + if (vic->edge_sources[sidx] & sbit) + irq_set_chip_and_handler(irq, &avic_chip, handle_edge_irq); + else + irq_set_chip_and_handler(irq, &avic_chip, handle_level_irq); + irq_set_chip_data(irq, vic); + irq_set_probe(irq); + return 0; +} + +static struct irq_domain_ops avic_dom_ops = { + .map = avic_map, + .xlate = irq_domain_xlate_onetwocell, +}; + +static int __init avic_of_init(struct device_node *node, + struct device_node *parent) +{ + void __iomem *regs; + struct aspeed_vic *vic; + + if (WARN(parent, "non-root Aspeed VIC not supported")) + return -EINVAL; + if (WARN(system_avic, "duplicate Aspeed VIC not supported")) + return -EINVAL; + + regs = of_iomap(node, 0); + if (WARN_ON(!regs)) + return -EIO; + + vic = kzalloc(sizeof(struct aspeed_vic), GFP_KERNEL); + if (WARN_ON(!vic)) { + iounmap(regs); + return -ENOMEM; + } + vic->base = regs; + + /* Initialize soures, all masked */ + vic_init_hw(vic); + + /* Ready to receive interrupts */ + system_avic = vic; + set_handle_irq(avic_handle_irq); + + /* Register our domain */ + vic->dom = irq_domain_add_simple(node, NUM_IRQS, 0, + &avic_dom_ops, vic); + + return 0; +} + +IRQCHIP_DECLARE(aspeed_new_vic, "aspeed,ast2400-vic", avic_of_init); diff --git a/drivers/irqchip/irq-bcm2835.c b/drivers/irqchip/irq-bcm2835.c index bf9cc5f2e839..44d7c38dde47 100644 --- a/drivers/irqchip/irq-bcm2835.c +++ b/drivers/irqchip/irq-bcm2835.c @@ -52,7 +52,6 @@ #include #include -#include /* Put the bank and irq (32 bits) into the hwirq */ #define MAKE_HWIRQ(b, n) ((b << 5) | (n)) @@ -242,7 +241,7 @@ static void __exception_irq_entry bcm2835_handle_irq( u32 hwirq; while ((hwirq = get_next_armctrl_hwirq()) != ~0) - handle_IRQ(irq_linear_revmap(intc.domain, hwirq), regs); + handle_domain_irq(intc.domain, hwirq, regs); } static void bcm2836_chained_handle_irq(struct irq_desc *desc) diff --git a/drivers/irqchip/irq-bcm2836.c b/drivers/irqchip/irq-bcm2836.c index 72ff1d5c5de6..df1949c0aa23 100644 --- a/drivers/irqchip/irq-bcm2836.c +++ b/drivers/irqchip/irq-bcm2836.c @@ -180,7 +180,7 @@ __exception_irq_entry bcm2836_arm_irqchip_handle_irq(struct pt_regs *regs) } else if (stat) { u32 hwirq = ffs(stat) - 1; - handle_IRQ(irq_linear_revmap(intc.domain, hwirq), regs); + handle_domain_irq(intc.domain, hwirq, regs); } } @@ -224,8 +224,8 @@ static struct notifier_block bcm2836_arm_irqchip_cpu_notifier = { }; #ifdef CONFIG_ARM -int __init bcm2836_smp_boot_secondary(unsigned int cpu, - struct task_struct *idle) +static int __init bcm2836_smp_boot_secondary(unsigned int cpu, + struct task_struct *idle) { unsigned long secondary_startup_phys = (unsigned long)virt_to_phys((void *)secondary_startup); diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c index 61b18ab33ad9..0ec92631e23c 100644 --- a/drivers/irqchip/irq-bcm7120-l2.c +++ b/drivers/irqchip/irq-bcm7120-l2.c @@ -215,7 +215,7 @@ static int __init bcm7120_l2_intc_iomap_3380(struct device_node *dn, return 0; } -int __init bcm7120_l2_intc_probe(struct device_node *dn, +static int __init bcm7120_l2_intc_probe(struct device_node *dn, struct device_node *parent, int (*iomap_regs_fn)(struct device_node *, struct bcm7120_l2_intc_data *), @@ -339,15 +339,15 @@ out_unmap: return ret; } -int __init bcm7120_l2_intc_probe_7120(struct device_node *dn, - struct device_node *parent) +static int __init bcm7120_l2_intc_probe_7120(struct device_node *dn, + struct device_node *parent) { return bcm7120_l2_intc_probe(dn, parent, bcm7120_l2_intc_iomap_7120, "BCM7120 L2"); } -int __init bcm7120_l2_intc_probe_3380(struct device_node *dn, - struct device_node *parent) +static int __init bcm7120_l2_intc_probe_3380(struct device_node *dn, + struct device_node *parent) { return bcm7120_l2_intc_probe(dn, parent, bcm7120_l2_intc_iomap_3380, "BCM3380 L2"); diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index 65cd341f331a..1d4a5b46d9ae 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -112,8 +112,8 @@ static void brcmstb_l2_intc_resume(struct irq_data *d) irq_gc_unlock(gc); } -int __init brcmstb_l2_intc_of_init(struct device_node *np, - struct device_node *parent) +static int __init brcmstb_l2_intc_of_init(struct device_node *np, + struct device_node *parent) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; struct brcmstb_l2_intc_data *data; diff --git a/drivers/irqchip/irq-gic-common.c b/drivers/irqchip/irq-gic-common.c index 89e7423f0ebb..9ae71804b5dd 100644 --- a/drivers/irqchip/irq-gic-common.c +++ b/drivers/irqchip/irq-gic-common.c @@ -90,8 +90,8 @@ int gic_configure_irq(unsigned int irq, unsigned int type, return ret; } -void __init gic_dist_config(void __iomem *base, int gic_irqs, - void (*sync_access)(void)) +void gic_dist_config(void __iomem *base, int gic_irqs, + void (*sync_access)(void)) { unsigned int i; diff --git a/drivers/irqchip/irq-gic-pm.c b/drivers/irqchip/irq-gic-pm.c new file mode 100644 index 000000000000..4cbffba3ff13 --- /dev/null +++ b/drivers/irqchip/irq-gic-pm.c @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2016 NVIDIA CORPORATION, All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct gic_clk_data { + unsigned int num_clocks; + const char *const *clocks; +}; + +static int gic_runtime_resume(struct device *dev) +{ + struct gic_chip_data *gic = dev_get_drvdata(dev); + int ret; + + ret = pm_clk_resume(dev); + if (ret) + return ret; + + /* + * On the very first resume, the pointer to the driver data + * will be NULL and this is intentional, because we do not + * want to restore the GIC on the very first resume. So if + * the pointer is not valid just return. + */ + if (!gic) + return 0; + + gic_dist_restore(gic); + gic_cpu_restore(gic); + + return 0; +} + +static int gic_runtime_suspend(struct device *dev) +{ + struct gic_chip_data *gic = dev_get_drvdata(dev); + + gic_dist_save(gic); + gic_cpu_save(gic); + + return pm_clk_suspend(dev); +} + +static int gic_get_clocks(struct device *dev, const struct gic_clk_data *data) +{ + struct clk *clk; + unsigned int i; + int ret; + + if (!dev || !data) + return -EINVAL; + + ret = pm_clk_create(dev); + if (ret) + return ret; + + for (i = 0; i < data->num_clocks; i++) { + clk = of_clk_get_by_name(dev->of_node, data->clocks[i]); + if (IS_ERR(clk)) { + dev_err(dev, "failed to get clock %s\n", + data->clocks[i]); + ret = PTR_ERR(clk); + goto error; + } + + ret = pm_clk_add_clk(dev, clk); + if (ret) { + dev_err(dev, "failed to add clock at index %d\n", i); + clk_put(clk); + goto error; + } + } + + return 0; + +error: + pm_clk_destroy(dev); + + return ret; +} + +static int gic_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct gic_clk_data *data; + struct gic_chip_data *gic; + int ret, irq; + + data = of_device_get_match_data(&pdev->dev); + if (!data) { + dev_err(&pdev->dev, "no device match found\n"); + return -ENODEV; + } + + irq = irq_of_parse_and_map(dev->of_node, 0); + if (!irq) { + dev_err(dev, "no parent interrupt found!\n"); + return -EINVAL; + } + + ret = gic_get_clocks(dev, data); + if (ret) + goto irq_dispose; + + pm_runtime_enable(dev); + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + goto rpm_disable; + + ret = gic_of_init_child(dev, &gic, irq); + if (ret) + goto rpm_put; + + platform_set_drvdata(pdev, gic); + + pm_runtime_put(dev); + + dev_info(dev, "GIC IRQ controller registered\n"); + + return 0; + +rpm_put: + pm_runtime_put_sync(dev); +rpm_disable: + pm_runtime_disable(dev); + pm_clk_destroy(dev); +irq_dispose: + irq_dispose_mapping(irq); + + return ret; +} + +static const struct dev_pm_ops gic_pm_ops = { + SET_RUNTIME_PM_OPS(gic_runtime_suspend, + gic_runtime_resume, NULL) +}; + +static const char * const gic400_clocks[] = { + "clk", +}; + +static const struct gic_clk_data gic400_data = { + .num_clocks = ARRAY_SIZE(gic400_clocks), + .clocks = gic400_clocks, +}; + +static const struct of_device_id gic_match[] = { + { .compatible = "nvidia,tegra210-agic", .data = &gic400_data }, + {}, +}; +MODULE_DEVICE_TABLE(of, gic_match); + +static struct platform_driver gic_driver = { + .probe = gic_probe, + .driver = { + .name = "gic", + .of_match_table = gic_match, + .pm = &gic_pm_ops, + } +}; + +builtin_platform_driver(gic_driver); diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c index ad0d2960b664..35eb7ac5d21f 100644 --- a/drivers/irqchip/irq-gic-v2m.c +++ b/drivers/irqchip/irq-gic-v2m.c @@ -24,6 +24,7 @@ #include #include #include +#include /* * MSI_TYPER: diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 5eb1f9e17a98..7ceaba81efb4 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -56,13 +56,14 @@ struct its_collection { }; /* - * The ITS_BASER structure - contains memory information and cached - * value of BASER register configuration. + * The ITS_BASER structure - contains memory information, cached + * value of BASER register configuration and ITS page size. */ struct its_baser { void *base; u64 val; u32 order; + u32 psz; }; /* @@ -824,180 +825,241 @@ static const char *its_base_type_string[] = { [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)", }; -static void its_free_tables(struct its_node *its) +static u64 its_read_baser(struct its_node *its, struct its_baser *baser) { - int i; + u32 idx = baser - its->tables; - for (i = 0; i < GITS_BASER_NR_REGS; i++) { - if (its->tables[i].base) { - free_pages((unsigned long)its->tables[i].base, - its->tables[i].order); - its->tables[i].base = NULL; - } - } + return readq_relaxed(its->base + GITS_BASER + (idx << 3)); } -static int its_alloc_tables(const char *node_name, struct its_node *its) +static void its_write_baser(struct its_node *its, struct its_baser *baser, + u64 val) { - int err; - int i; - int psz = SZ_64K; - u64 shr = GITS_BASER_InnerShareable; - u64 cache; - u64 typer; - u32 ids; + u32 idx = baser - its->tables; - if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) { - /* - * erratum 22375: only alloc 8MB table size - * erratum 24313: ignore memory access type - */ - cache = 0; - ids = 0x14; /* 20 bits, 8MB */ - } else { - cache = GITS_BASER_WaWb; - typer = readq_relaxed(its->base + GITS_TYPER); - ids = GITS_TYPER_DEVBITS(typer); + writeq_relaxed(val, its->base + GITS_BASER + (idx << 3)); + baser->val = its_read_baser(its, baser); +} + +static int its_setup_baser(struct its_node *its, struct its_baser *baser, + u64 cache, u64 shr, u32 psz, u32 order, + bool indirect) +{ + u64 val = its_read_baser(its, baser); + u64 esz = GITS_BASER_ENTRY_SIZE(val); + u64 type = GITS_BASER_TYPE(val); + u32 alloc_pages; + void *base; + u64 tmp; + +retry_alloc_baser: + alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz); + if (alloc_pages > GITS_BASER_PAGES_MAX) { + pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n", + &its->phys_base, its_base_type_string[type], + alloc_pages, GITS_BASER_PAGES_MAX); + alloc_pages = GITS_BASER_PAGES_MAX; + order = get_order(GITS_BASER_PAGES_MAX * psz); } - its->device_ids = ids; + base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!base) + return -ENOMEM; - for (i = 0; i < GITS_BASER_NR_REGS; i++) { - u64 val = readq_relaxed(its->base + GITS_BASER + i * 8); - u64 type = GITS_BASER_TYPE(val); - u64 entry_size = GITS_BASER_ENTRY_SIZE(val); - int order = get_order(psz); - int alloc_pages; - u64 tmp; - void *base; +retry_baser: + val = (virt_to_phys(base) | + (type << GITS_BASER_TYPE_SHIFT) | + ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | + ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) | + cache | + shr | + GITS_BASER_VALID); + + val |= indirect ? GITS_BASER_INDIRECT : 0x0; + + switch (psz) { + case SZ_4K: + val |= GITS_BASER_PAGE_SIZE_4K; + break; + case SZ_16K: + val |= GITS_BASER_PAGE_SIZE_16K; + break; + case SZ_64K: + val |= GITS_BASER_PAGE_SIZE_64K; + break; + } - if (type == GITS_BASER_TYPE_NONE) - continue; + its_write_baser(its, baser, val); + tmp = baser->val; + if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { /* - * Allocate as many entries as required to fit the - * range of device IDs that the ITS can grok... The ID - * space being incredibly sparse, this results in a - * massive waste of memory. - * - * For other tables, only allocate a single page. + * Shareability didn't stick. Just use + * whatever the read reported, which is likely + * to be the only thing this redistributor + * supports. If that's zero, make it + * non-cacheable as well. */ - if (type == GITS_BASER_TYPE_DEVICE) { - /* - * 'order' was initialized earlier to the default page - * granule of the the ITS. We can't have an allocation - * smaller than that. If the requested allocation - * is smaller, round up to the default page granule. - */ - order = max(get_order((1UL << ids) * entry_size), - order); - if (order >= MAX_ORDER) { - order = MAX_ORDER - 1; - pr_warn("%s: Device Table too large, reduce its page order to %u\n", - node_name, order); - } - } - -retry_alloc_baser: - alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz); - if (alloc_pages > GITS_BASER_PAGES_MAX) { - alloc_pages = GITS_BASER_PAGES_MAX; - order = get_order(GITS_BASER_PAGES_MAX * psz); - pr_warn("%s: Device Table too large, reduce its page order to %u (%u pages)\n", - node_name, order, alloc_pages); + shr = tmp & GITS_BASER_SHAREABILITY_MASK; + if (!shr) { + cache = GITS_BASER_nC; + __flush_dcache_area(base, PAGE_ORDER_TO_SIZE(order)); } + goto retry_baser; + } - base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); - if (!base) { - err = -ENOMEM; - goto out_free; - } - - its->tables[i].base = base; - its->tables[i].order = order; - -retry_baser: - val = (virt_to_phys(base) | - (type << GITS_BASER_TYPE_SHIFT) | - ((entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | - cache | - shr | - GITS_BASER_VALID); + if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) { + /* + * Page size didn't stick. Let's try a smaller + * size and retry. If we reach 4K, then + * something is horribly wrong... + */ + free_pages((unsigned long)base, order); + baser->base = NULL; switch (psz) { - case SZ_4K: - val |= GITS_BASER_PAGE_SIZE_4K; - break; case SZ_16K: - val |= GITS_BASER_PAGE_SIZE_16K; - break; + psz = SZ_4K; + goto retry_alloc_baser; case SZ_64K: - val |= GITS_BASER_PAGE_SIZE_64K; - break; + psz = SZ_16K; + goto retry_alloc_baser; } + } + + if (val != tmp) { + pr_err("ITS@%pa: %s doesn't stick: %lx %lx\n", + &its->phys_base, its_base_type_string[type], + (unsigned long) val, (unsigned long) tmp); + free_pages((unsigned long)base, order); + return -ENXIO; + } - val |= alloc_pages - 1; - its->tables[i].val = val; + baser->order = order; + baser->base = base; + baser->psz = psz; + tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz; - writeq_relaxed(val, its->base + GITS_BASER + i * 8); - tmp = readq_relaxed(its->base + GITS_BASER + i * 8); + pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n", + &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / tmp), + its_base_type_string[type], + (unsigned long)virt_to_phys(base), + indirect ? "indirect" : "flat", (int)esz, + psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); - if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { + return 0; +} + +static bool its_parse_baser_device(struct its_node *its, struct its_baser *baser, + u32 psz, u32 *order) +{ + u64 esz = GITS_BASER_ENTRY_SIZE(its_read_baser(its, baser)); + u64 val = GITS_BASER_InnerShareable | GITS_BASER_WaWb; + u32 ids = its->device_ids; + u32 new_order = *order; + bool indirect = false; + + /* No need to enable Indirection if memory requirement < (psz*2)bytes */ + if ((esz << ids) > (psz * 2)) { + /* + * Find out whether hw supports a single or two-level table by + * table by reading bit at offset '62' after writing '1' to it. + */ + its_write_baser(its, baser, val | GITS_BASER_INDIRECT); + indirect = !!(baser->val & GITS_BASER_INDIRECT); + + if (indirect) { /* - * Shareability didn't stick. Just use - * whatever the read reported, which is likely - * to be the only thing this redistributor - * supports. If that's zero, make it - * non-cacheable as well. + * The size of the lvl2 table is equal to ITS page size + * which is 'psz'. For computing lvl1 table size, + * subtract ID bits that sparse lvl2 table from 'ids' + * which is reported by ITS hardware times lvl1 table + * entry size. */ - shr = tmp & GITS_BASER_SHAREABILITY_MASK; - if (!shr) { - cache = GITS_BASER_nC; - __flush_dcache_area(base, PAGE_ORDER_TO_SIZE(order)); - } - goto retry_baser; + ids -= ilog2(psz / esz); + esz = GITS_LVL1_ENTRY_SIZE; } + } - if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) { - /* - * Page size didn't stick. Let's try a smaller - * size and retry. If we reach 4K, then - * something is horribly wrong... - */ - free_pages((unsigned long)base, order); - its->tables[i].base = NULL; + /* + * Allocate as many entries as required to fit the + * range of device IDs that the ITS can grok... The ID + * space being incredibly sparse, this results in a + * massive waste of memory if two-level device table + * feature is not supported by hardware. + */ + new_order = max_t(u32, get_order(esz << ids), new_order); + if (new_order >= MAX_ORDER) { + new_order = MAX_ORDER - 1; + ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / esz); + pr_warn("ITS@%pa: Device Table too large, reduce ids %u->%u\n", + &its->phys_base, its->device_ids, ids); + } - switch (psz) { - case SZ_16K: - psz = SZ_4K; - goto retry_alloc_baser; - case SZ_64K: - psz = SZ_16K; - goto retry_alloc_baser; - } - } + *order = new_order; + + return indirect; +} - if (val != tmp) { - pr_err("ITS: %s: GITS_BASER%d doesn't stick: %lx %lx\n", - node_name, i, - (unsigned long) val, (unsigned long) tmp); - err = -ENXIO; - goto out_free; +static void its_free_tables(struct its_node *its) +{ + int i; + + for (i = 0; i < GITS_BASER_NR_REGS; i++) { + if (its->tables[i].base) { + free_pages((unsigned long)its->tables[i].base, + its->tables[i].order); + its->tables[i].base = NULL; } + } +} + +static int its_alloc_tables(struct its_node *its) +{ + u64 typer = readq_relaxed(its->base + GITS_TYPER); + u32 ids = GITS_TYPER_DEVBITS(typer); + u64 shr = GITS_BASER_InnerShareable; + u64 cache = GITS_BASER_WaWb; + u32 psz = SZ_64K; + int err, i; - pr_info("ITS: allocated %d %s @%lx (psz %dK, shr %d)\n", - (int)(PAGE_ORDER_TO_SIZE(order) / entry_size), - its_base_type_string[type], - (unsigned long)virt_to_phys(base), - psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); + if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) { + /* + * erratum 22375: only alloc 8MB table size + * erratum 24313: ignore memory access type + */ + cache = GITS_BASER_nCnB; + ids = 0x14; /* 20 bits, 8MB */ } - return 0; + its->device_ids = ids; -out_free: - its_free_tables(its); + for (i = 0; i < GITS_BASER_NR_REGS; i++) { + struct its_baser *baser = its->tables + i; + u64 val = its_read_baser(its, baser); + u64 type = GITS_BASER_TYPE(val); + u32 order = get_order(psz); + bool indirect = false; - return err; + if (type == GITS_BASER_TYPE_NONE) + continue; + + if (type == GITS_BASER_TYPE_DEVICE) + indirect = its_parse_baser_device(its, baser, psz, &order); + + err = its_setup_baser(its, baser, cache, shr, psz, order, indirect); + if (err < 0) { + its_free_tables(its); + return err; + } + + /* Update settings which will be used for next BASERn */ + psz = baser->psz; + cache = baser->val & GITS_BASER_CACHEABILITY_MASK; + shr = baser->val & GITS_BASER_SHAREABILITY_MASK; + } + + return 0; } static int its_alloc_collections(struct its_node *its) @@ -1185,10 +1247,57 @@ static struct its_baser *its_get_baser(struct its_node *its, u32 type) return NULL; } +static bool its_alloc_device_table(struct its_node *its, u32 dev_id) +{ + struct its_baser *baser; + struct page *page; + u32 esz, idx; + __le64 *table; + + baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); + + /* Don't allow device id that exceeds ITS hardware limit */ + if (!baser) + return (ilog2(dev_id) < its->device_ids); + + /* Don't allow device id that exceeds single, flat table limit */ + esz = GITS_BASER_ENTRY_SIZE(baser->val); + if (!(baser->val & GITS_BASER_INDIRECT)) + return (dev_id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); + + /* Compute 1st level table index & check if that exceeds table limit */ + idx = dev_id >> ilog2(baser->psz / esz); + if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE)) + return false; + + table = baser->base; + + /* Allocate memory for 2nd level table */ + if (!table[idx]) { + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(baser->psz)); + if (!page) + return false; + + /* Flush Lvl2 table to PoC if hw doesn't support coherency */ + if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) + __flush_dcache_area(page_address(page), baser->psz); + + table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); + + /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ + if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) + __flush_dcache_area(table + idx, GITS_LVL1_ENTRY_SIZE); + + /* Ensure updated table contents are visible to ITS hardware */ + dsb(sy); + } + + return true; +} + static struct its_device *its_create_device(struct its_node *its, u32 dev_id, int nvecs) { - struct its_baser *baser; struct its_device *dev; unsigned long *lpi_map; unsigned long flags; @@ -1199,14 +1308,7 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, int nr_ites; int sz; - baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); - - /* Don't allow 'dev_id' that exceeds single, flat table limit */ - if (baser) { - if (dev_id >= (PAGE_ORDER_TO_SIZE(baser->order) / - GITS_BASER_ENTRY_SIZE(baser->val))) - return NULL; - } else if (ilog2(dev_id) >= its->device_ids) + if (!its_alloc_device_table(its, dev_id)) return NULL; dev = kzalloc(sizeof(*dev), GFP_KERNEL); @@ -1569,7 +1671,7 @@ static int __init its_probe(struct device_node *node, its_enable_quirks(its); - err = its_alloc_tables(node->full_name, its); + err = its_alloc_tables(its); if (err) goto out_free_cmd; diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index fbc4ae2afd29..1de07eb5839c 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -75,7 +75,7 @@ struct gic_chip_data { void __iomem *raw_dist_base; void __iomem *raw_cpu_base; u32 percpu_offset; -#ifdef CONFIG_CPU_PM +#if defined(CONFIG_CPU_PM) || defined(CONFIG_ARM_GIC_PM) u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)]; u32 saved_spi_active[DIV_ROUND_UP(1020, 32)]; u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)]; @@ -449,7 +449,7 @@ static void gic_cpu_if_up(struct gic_chip_data *gic) } -static void __init gic_dist_init(struct gic_chip_data *gic) +static void gic_dist_init(struct gic_chip_data *gic) { unsigned int i; u32 cpumask; @@ -528,14 +528,14 @@ int gic_cpu_if_down(unsigned int gic_nr) return 0; } -#ifdef CONFIG_CPU_PM +#if defined(CONFIG_CPU_PM) || defined(CONFIG_ARM_GIC_PM) /* * Saves the GIC distributor registers during suspend or idle. Must be called * with interrupts disabled but before powering down the GIC. After calling * this function, no interrupts will be delivered by the GIC, and another * platform-specific wakeup source must be enabled. */ -static void gic_dist_save(struct gic_chip_data *gic) +void gic_dist_save(struct gic_chip_data *gic) { unsigned int gic_irqs; void __iomem *dist_base; @@ -574,7 +574,7 @@ static void gic_dist_save(struct gic_chip_data *gic) * handled normally, but any edge interrupts that occured will not be seen by * the GIC and need to be handled by the platform-specific wakeup source. */ -static void gic_dist_restore(struct gic_chip_data *gic) +void gic_dist_restore(struct gic_chip_data *gic) { unsigned int gic_irqs; unsigned int i; @@ -620,7 +620,7 @@ static void gic_dist_restore(struct gic_chip_data *gic) writel_relaxed(GICD_ENABLE, dist_base + GIC_DIST_CTRL); } -static void gic_cpu_save(struct gic_chip_data *gic) +void gic_cpu_save(struct gic_chip_data *gic) { int i; u32 *ptr; @@ -650,7 +650,7 @@ static void gic_cpu_save(struct gic_chip_data *gic) } -static void gic_cpu_restore(struct gic_chip_data *gic) +void gic_cpu_restore(struct gic_chip_data *gic) { int i; u32 *ptr; @@ -727,7 +727,7 @@ static struct notifier_block gic_notifier_block = { .notifier_call = gic_notifier, }; -static int __init gic_pm_init(struct gic_chip_data *gic) +static int gic_pm_init(struct gic_chip_data *gic) { gic->saved_ppi_enable = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4, sizeof(u32)); @@ -757,7 +757,7 @@ free_ppi_enable: return -ENOMEM; } #else -static int __init gic_pm_init(struct gic_chip_data *gic) +static int gic_pm_init(struct gic_chip_data *gic) { return 0; } @@ -1032,32 +1032,31 @@ static const struct irq_domain_ops gic_irq_domain_ops = { .unmap = gic_irq_domain_unmap, }; -static int __init __gic_init_bases(struct gic_chip_data *gic, int irq_start, - struct fwnode_handle *handle) +static void gic_init_chip(struct gic_chip_data *gic, struct device *dev, + const char *name, bool use_eoimode1) { - irq_hw_number_t hwirq_base; - int gic_irqs, irq_base, i, ret; - - if (WARN_ON(!gic || gic->domain)) - return -EINVAL; - /* Initialize irq_chip */ gic->chip = gic_chip; + gic->chip.name = name; + gic->chip.parent_device = dev; - if (static_key_true(&supports_deactivate) && gic == &gic_data[0]) { + if (use_eoimode1) { gic->chip.irq_mask = gic_eoimode1_mask_irq; gic->chip.irq_eoi = gic_eoimode1_eoi_irq; gic->chip.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity; - gic->chip.name = kasprintf(GFP_KERNEL, "GICv2"); - } else { - gic->chip.name = kasprintf(GFP_KERNEL, "GIC-%d", - (int)(gic - &gic_data[0])); } #ifdef CONFIG_SMP if (gic == &gic_data[0]) gic->chip.irq_set_affinity = gic_set_affinity; #endif +} + +static int gic_init_bases(struct gic_chip_data *gic, int irq_start, + struct fwnode_handle *handle) +{ + irq_hw_number_t hwirq_base; + int gic_irqs, irq_base, ret; if (IS_ENABLED(CONFIG_GIC_NON_BANKED) && gic->percpu_offset) { /* Frankein-GIC without banked registers... */ @@ -1138,6 +1137,36 @@ static int __init __gic_init_bases(struct gic_chip_data *gic, int irq_start, goto error; } + gic_dist_init(gic); + ret = gic_cpu_init(gic); + if (ret) + goto error; + + ret = gic_pm_init(gic); + if (ret) + goto error; + + return 0; + +error: + if (IS_ENABLED(CONFIG_GIC_NON_BANKED) && gic->percpu_offset) { + free_percpu(gic->dist_base.percpu_base); + free_percpu(gic->cpu_base.percpu_base); + } + + return ret; +} + +static int __init __gic_init_bases(struct gic_chip_data *gic, + int irq_start, + struct fwnode_handle *handle) +{ + char *name; + int i, ret; + + if (WARN_ON(!gic || gic->domain)) + return -EINVAL; + if (gic == &gic_data[0]) { /* * Initialize the CPU interface map to all CPUs. @@ -1155,24 +1184,17 @@ static int __init __gic_init_bases(struct gic_chip_data *gic, int irq_start, pr_info("GIC: Using split EOI/Deactivate mode\n"); } - gic_dist_init(gic); - ret = gic_cpu_init(gic); - if (ret) - goto error; - - ret = gic_pm_init(gic); - if (ret) - goto error; - - return 0; - -error: - if (IS_ENABLED(CONFIG_GIC_NON_BANKED) && gic->percpu_offset) { - free_percpu(gic->dist_base.percpu_base); - free_percpu(gic->cpu_base.percpu_base); + if (static_key_true(&supports_deactivate) && gic == &gic_data[0]) { + name = kasprintf(GFP_KERNEL, "GICv2"); + gic_init_chip(gic, NULL, name, true); + } else { + name = kasprintf(GFP_KERNEL, "GIC-%d", (int)(gic-&gic_data[0])); + gic_init_chip(gic, NULL, name, false); } - kfree(gic->chip.name); + ret = gic_init_bases(gic, irq_start, handle); + if (ret) + kfree(name); return ret; } @@ -1250,7 +1272,7 @@ static bool gic_check_eoimode(struct device_node *node, void __iomem **base) return true; } -static int __init gic_of_setup(struct gic_chip_data *gic, struct device_node *node) +static int gic_of_setup(struct gic_chip_data *gic, struct device_node *node) { if (!gic || !node) return -EINVAL; @@ -1274,6 +1296,34 @@ error: return -ENOMEM; } +int gic_of_init_child(struct device *dev, struct gic_chip_data **gic, int irq) +{ + int ret; + + if (!dev || !dev->of_node || !gic || !irq) + return -EINVAL; + + *gic = devm_kzalloc(dev, sizeof(**gic), GFP_KERNEL); + if (!*gic) + return -ENOMEM; + + gic_init_chip(*gic, dev, dev->of_node->name, false); + + ret = gic_of_setup(*gic, dev->of_node); + if (ret) + return ret; + + ret = gic_init_bases(*gic, -1, &dev->of_node->fwnode); + if (ret) { + gic_teardown(*gic); + return ret; + } + + irq_set_chained_handler_and_data(irq, gic_handle_cascade_irq, *gic); + + return 0; +} + static void __init gic_of_setup_kvm_info(struct device_node *node) { int ret; @@ -1353,7 +1403,11 @@ IRQCHIP_DECLARE(cortex_a7_gic, "arm,cortex-a7-gic", gic_of_init); IRQCHIP_DECLARE(msm_8660_qgic, "qcom,msm-8660-qgic", gic_of_init); IRQCHIP_DECLARE(msm_qgic2, "qcom,msm-qgic2", gic_of_init); IRQCHIP_DECLARE(pl390, "arm,pl390", gic_of_init); - +#else +int gic_of_init_child(struct device *dev, struct gic_chip_data **gic, int irq) +{ + return -ENOTSUPP; +} #endif #ifdef CONFIG_ACPI diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c index 3b5e10aa48ab..3786d0f21972 100644 --- a/drivers/irqchip/irq-mips-gic.c +++ b/drivers/irqchip/irq-mips-gic.c @@ -718,7 +718,7 @@ static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq, spin_lock_irqsave(&gic_lock, flags); gic_map_to_pin(intr, gic_cpu_pin); - gic_map_to_vpe(intr, vpe); + gic_map_to_vpe(intr, mips_cm_vp_id(vpe)); for (i = 0; i < min(gic_vpes, NR_CPUS); i++) clear_bit(intr, pcpu_masks[i].pcpu_mask); set_bit(intr, pcpu_masks[vpe].pcpu_mask); @@ -746,6 +746,12 @@ static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq, /* verify that it doesn't conflict with an IPI irq */ if (test_bit(spec->hwirq, ipi_resrv)) return -EBUSY; + + hwirq = GIC_SHARED_TO_HWIRQ(spec->hwirq); + + return irq_domain_set_hwirq_and_chip(d, virq, hwirq, + &gic_level_irq_controller, + NULL); } else { base_hwirq = find_first_bit(ipi_resrv, gic_shared_intrs); if (base_hwirq == gic_shared_intrs) { @@ -867,10 +873,14 @@ static int gic_dev_domain_alloc(struct irq_domain *d, unsigned int virq, &gic_level_irq_controller, NULL); if (ret) - return ret; + goto error; } return 0; + +error: + irq_domain_free_irqs_parent(d, virq, nr_irqs); + return ret; } void gic_dev_domain_free(struct irq_domain *d, unsigned int virq, @@ -949,7 +959,7 @@ int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node, switch (bus_token) { case DOMAIN_BUS_IPI: is_ipi = d->bus_token == bus_token; - return to_of_node(d->fwnode) == node && is_ipi; + return (!node || to_of_node(d->fwnode) == node) && is_ipi; break; default: return 0; @@ -1032,12 +1042,14 @@ static void __init __gic_init(unsigned long gic_base_addr, &gic_irq_domain_ops, NULL); if (!gic_irq_domain) panic("Failed to add GIC IRQ domain"); + gic_irq_domain->name = "mips-gic-irq"; gic_dev_domain = irq_domain_add_hierarchy(gic_irq_domain, 0, GIC_NUM_LOCAL_INTRS + gic_shared_intrs, node, &gic_dev_domain_ops, NULL); if (!gic_dev_domain) panic("Failed to add GIC DEV domain"); + gic_dev_domain->name = "mips-gic-dev"; gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain, IRQ_DOMAIN_FLAG_IPI_PER_CPU, @@ -1046,6 +1058,7 @@ static void __init __gic_init(unsigned long gic_base_addr, if (!gic_ipi_domain) panic("Failed to add GIC IPI domain"); + gic_ipi_domain->name = "mips-gic-ipi"; gic_ipi_domain->bus_token = DOMAIN_BUS_IPI; if (node && diff --git a/drivers/irqchip/irq-omap-intc.c b/drivers/irqchip/irq-omap-intc.c index 9d1bcfc33e4c..b04a8ac6e744 100644 --- a/drivers/irqchip/irq-omap-intc.c +++ b/drivers/irqchip/irq-omap-intc.c @@ -23,6 +23,8 @@ #include #include +#include + /* Define these here for now until we drop all board-files */ #define OMAP24XX_IC_BASE 0x480fe000 #define OMAP34XX_IC_BASE 0x48200000 diff --git a/drivers/irqchip/irq-s3c24xx.c b/drivers/irqchip/irq-s3c24xx.c index 5dc5a760c723..c25ce5af091a 100644 --- a/drivers/irqchip/irq-s3c24xx.c +++ b/drivers/irqchip/irq-s3c24xx.c @@ -92,9 +92,9 @@ static void s3c_irq_mask(struct irq_data *data) unsigned long mask; unsigned int irqno; - mask = __raw_readl(intc->reg_mask); + mask = readl_relaxed(intc->reg_mask); mask |= (1UL << irq_data->offset); - __raw_writel(mask, intc->reg_mask); + writel_relaxed(mask, intc->reg_mask); if (parent_intc) { parent_data = &parent_intc->irqs[irq_data->parent_irq]; @@ -119,9 +119,9 @@ static void s3c_irq_unmask(struct irq_data *data) unsigned long mask; unsigned int irqno; - mask = __raw_readl(intc->reg_mask); + mask = readl_relaxed(intc->reg_mask); mask &= ~(1UL << irq_data->offset); - __raw_writel(mask, intc->reg_mask); + writel_relaxed(mask, intc->reg_mask); if (parent_intc) { irqno = irq_find_mapping(parent_intc->domain, @@ -136,9 +136,9 @@ static inline void s3c_irq_ack(struct irq_data *data) struct s3c_irq_intc *intc = irq_data->intc; unsigned long bitval = 1UL << irq_data->offset; - __raw_writel(bitval, intc->reg_pending); + writel_relaxed(bitval, intc->reg_pending); if (intc->reg_intpnd) - __raw_writel(bitval, intc->reg_intpnd); + writel_relaxed(bitval, intc->reg_intpnd); } static int s3c_irq_type(struct irq_data *data, unsigned int type) @@ -172,9 +172,9 @@ static int s3c_irqext_type_set(void __iomem *gpcon_reg, unsigned long newvalue = 0, value; /* Set the GPIO to external interrupt mode */ - value = __raw_readl(gpcon_reg); + value = readl_relaxed(gpcon_reg); value = (value & ~(3 << gpcon_offset)) | (0x02 << gpcon_offset); - __raw_writel(value, gpcon_reg); + writel_relaxed(value, gpcon_reg); /* Set the external interrupt to pointed trigger type */ switch (type) @@ -208,9 +208,9 @@ static int s3c_irqext_type_set(void __iomem *gpcon_reg, return -EINVAL; } - value = __raw_readl(extint_reg); + value = readl_relaxed(extint_reg); value = (value & ~(7 << extint_offset)) | (newvalue << extint_offset); - __raw_writel(value, extint_reg); + writel_relaxed(value, extint_reg); return 0; } @@ -315,8 +315,8 @@ static void s3c_irq_demux(struct irq_desc *desc) chained_irq_enter(chip, desc); - src = __raw_readl(sub_intc->reg_pending); - msk = __raw_readl(sub_intc->reg_mask); + src = readl_relaxed(sub_intc->reg_pending); + msk = readl_relaxed(sub_intc->reg_mask); src &= ~msk; src &= irq_data->sub_bits; @@ -337,7 +337,7 @@ static inline int s3c24xx_handle_intc(struct s3c_irq_intc *intc, int pnd; int offset; - pnd = __raw_readl(intc->reg_intpnd); + pnd = readl_relaxed(intc->reg_intpnd); if (!pnd) return false; @@ -352,7 +352,7 @@ static inline int s3c24xx_handle_intc(struct s3c_irq_intc *intc, * * Thanks to Klaus, Shannon, et al for helping to debug this problem */ - offset = __raw_readl(intc->reg_intpnd + 4); + offset = readl_relaxed(intc->reg_intpnd + 4); /* Find the bit manually, when the offset is wrong. * The pending register only ever contains the one bit of the next @@ -406,7 +406,7 @@ int s3c24xx_set_fiq(unsigned int irq, bool on) intmod = 0; } - __raw_writel(intmod, S3C2410_INTMOD); + writel_relaxed(intmod, S3C2410_INTMOD); return 0; } @@ -508,14 +508,14 @@ static void s3c24xx_clear_intc(struct s3c_irq_intc *intc) last = 0; for (i = 0; i < 4; i++) { - pend = __raw_readl(reg_source); + pend = readl_relaxed(reg_source); if (pend == 0 || pend == last) break; - __raw_writel(pend, intc->reg_pending); + writel_relaxed(pend, intc->reg_pending); if (intc->reg_intpnd) - __raw_writel(pend, intc->reg_intpnd); + writel_relaxed(pend, intc->reg_intpnd); pr_info("irq: clearing pending status %08x\n", (int)pend); last = pend; diff --git a/drivers/irqchip/irq-sirfsoc.c b/drivers/irqchip/irq-sirfsoc.c index 10cb21b9ba3d..e1336848affa 100644 --- a/drivers/irqchip/irq-sirfsoc.c +++ b/drivers/irqchip/irq-sirfsoc.c @@ -29,6 +29,11 @@ static struct irq_domain *sirfsoc_irqdomain; +static void __iomem *sirfsoc_irq_get_regbase(void) +{ + return (void __iomem __force *)sirfsoc_irqdomain->host_data; +} + static __init void sirfsoc_alloc_gc(void __iomem *base) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; @@ -53,7 +58,7 @@ static __init void sirfsoc_alloc_gc(void __iomem *base) static void __exception_irq_entry sirfsoc_handle_irq(struct pt_regs *regs) { - void __iomem *base = sirfsoc_irqdomain->host_data; + void __iomem *base = sirfsoc_irq_get_regbase(); u32 irqstat; irqstat = readl_relaxed(base + SIRFSOC_INIT_IRQ_ID); @@ -94,7 +99,7 @@ static struct sirfsoc_irq_status sirfsoc_irq_st; static int sirfsoc_irq_suspend(void) { - void __iomem *base = sirfsoc_irqdomain->host_data; + void __iomem *base = sirfsoc_irq_get_regbase(); sirfsoc_irq_st.mask0 = readl_relaxed(base + SIRFSOC_INT_RISC_MASK0); sirfsoc_irq_st.mask1 = readl_relaxed(base + SIRFSOC_INT_RISC_MASK1); @@ -106,7 +111,7 @@ static int sirfsoc_irq_suspend(void) static void sirfsoc_irq_resume(void) { - void __iomem *base = sirfsoc_irqdomain->host_data; + void __iomem *base = sirfsoc_irq_get_regbase(); writel_relaxed(sirfsoc_irq_st.mask0, base + SIRFSOC_INT_RISC_MASK0); writel_relaxed(sirfsoc_irq_st.mask1, base + SIRFSOC_INT_RISC_MASK1); diff --git a/drivers/irqchip/irq-tegra.c b/drivers/irqchip/irq-tegra.c index e902f081e16c..3973a14bb15b 100644 --- a/drivers/irqchip/irq-tegra.c +++ b/drivers/irqchip/irq-tegra.c @@ -90,7 +90,7 @@ static struct tegra_ictlr_info *lic; static inline void tegra_ictlr_write_mask(struct irq_data *d, unsigned long reg) { - void __iomem *base = d->chip_data; + void __iomem *base = (void __iomem __force *)d->chip_data; u32 mask; mask = BIT(d->hwirq % 32); @@ -266,7 +266,7 @@ static int tegra_ictlr_domain_alloc(struct irq_domain *domain, irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, &tegra_ictlr_chip, - info->base[ictlr]); + (void __force *)info->base[ictlr]); } parent_fwspec = *fwspec; diff --git a/drivers/irqchip/irq-vic.c b/drivers/irqchip/irq-vic.c index b956dfffe78c..f811a7de5857 100644 --- a/drivers/irqchip/irq-vic.c +++ b/drivers/irqchip/irq-vic.c @@ -167,7 +167,7 @@ static int vic_suspend(void) return 0; } -struct syscore_ops vic_syscore_ops = { +static struct syscore_ops vic_syscore_ops = { .suspend = vic_suspend, .resume = vic_resume, }; @@ -517,7 +517,8 @@ int __init vic_init_cascaded(void __iomem *base, unsigned int parent_irq, EXPORT_SYMBOL_GPL(vic_init_cascaded); #ifdef CONFIG_OF -int __init vic_of_init(struct device_node *node, struct device_node *parent) +static int __init vic_of_init(struct device_node *node, + struct device_node *parent) { void __iomem *regs; u32 interrupt_mask = ~0; diff --git a/drivers/isdn/hardware/eicon/divasmain.c b/drivers/isdn/hardware/eicon/divasmain.c index a2e0ed6c9a4d..32f34511c416 100644 --- a/drivers/isdn/hardware/eicon/divasmain.c +++ b/drivers/isdn/hardware/eicon/divasmain.c @@ -445,32 +445,32 @@ void divasa_unmap_pci_bar(void __iomem *bar) /********************************************************* ** I/O port access *********************************************************/ -byte __inline__ inpp(void __iomem *addr) +inline byte inpp(void __iomem *addr) { return (inb((unsigned long) addr)); } -word __inline__ inppw(void __iomem *addr) +inline word inppw(void __iomem *addr) { return (inw((unsigned long) addr)); } -void __inline__ inppw_buffer(void __iomem *addr, void *P, int length) +inline void inppw_buffer(void __iomem *addr, void *P, int length) { insw((unsigned long) addr, (word *) P, length >> 1); } -void __inline__ outppw_buffer(void __iomem *addr, void *P, int length) +inline void outppw_buffer(void __iomem *addr, void *P, int length) { outsw((unsigned long) addr, (word *) P, length >> 1); } -void __inline__ outppw(void __iomem *addr, word w) +inline void outppw(void __iomem *addr, word w) { outw(w, (unsigned long) addr); } -void __inline__ outpp(void __iomem *addr, word p) +inline void outpp(void __iomem *addr, word p) { outb(p, (unsigned long) addr); } diff --git a/drivers/isdn/hardware/eicon/platform.h b/drivers/isdn/hardware/eicon/platform.h index b2edb7590dda..62e2073c3690 100644 --- a/drivers/isdn/hardware/eicon/platform.h +++ b/drivers/isdn/hardware/eicon/platform.h @@ -203,7 +203,7 @@ void PCIread(byte bus, byte func, int offset, void *data, int length, void *pci_ /* ** I/O Port utilities */ -int diva_os_register_io_port(void *adapter, int register, unsigned long port, +int diva_os_register_io_port(void *adapter, int reg, unsigned long port, unsigned long length, const char *name, int id); /* ** I/O port access abstraction @@ -271,13 +271,13 @@ void diva_os_get_time(dword *sec, dword *usec); ** atomic operation, fake because we use threads */ typedef int diva_os_atomic_t; -static diva_os_atomic_t __inline__ +static inline diva_os_atomic_t diva_os_atomic_increment(diva_os_atomic_t *pv) { *pv += 1; return (*pv); } -static diva_os_atomic_t __inline__ +static inline diva_os_atomic_t diva_os_atomic_decrement(diva_os_atomic_t *pv) { *pv -= 1; diff --git a/drivers/lightnvm/Kconfig b/drivers/lightnvm/Kconfig index 85a339030e4b..61c68a1f054a 100644 --- a/drivers/lightnvm/Kconfig +++ b/drivers/lightnvm/Kconfig @@ -27,11 +27,13 @@ config NVM_DEBUG It is required to create/remove targets without IOCTLs. config NVM_GENNVM - tristate "Generic NVM manager for Open-Channel SSDs" + tristate "General Non-Volatile Memory Manager for Open-Channel SSDs" ---help--- - NVM media manager for Open-Channel SSDs that offload management - functionality to device, while keeping data placement and garbage - collection decisions on the host. + Non-volatile memory media manager for Open-Channel SSDs that implements + physical media metadata management and block provisioning API. + + This is the standard media manager for using Open-Channel SSDs, and + required for targets to be instantiated. config NVM_RRPC tristate "Round-robin Hybrid Open-Channel SSD target" diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c index 160c1a6838e1..9ebd2cfbd849 100644 --- a/drivers/lightnvm/core.c +++ b/drivers/lightnvm/core.c @@ -18,8 +18,6 @@ * */ -#include -#include #include #include #include @@ -28,46 +26,42 @@ #include #include #include -#include static LIST_HEAD(nvm_tgt_types); +static DECLARE_RWSEM(nvm_tgtt_lock); static LIST_HEAD(nvm_mgrs); static LIST_HEAD(nvm_devices); -static LIST_HEAD(nvm_targets); static DECLARE_RWSEM(nvm_lock); -static struct nvm_target *nvm_find_target(const char *name) +struct nvm_tgt_type *nvm_find_target_type(const char *name, int lock) { - struct nvm_target *tgt; + struct nvm_tgt_type *tmp, *tt = NULL; - list_for_each_entry(tgt, &nvm_targets, list) - if (!strcmp(name, tgt->disk->disk_name)) - return tgt; + if (lock) + down_write(&nvm_tgtt_lock); - return NULL; -} - -static struct nvm_tgt_type *nvm_find_target_type(const char *name) -{ - struct nvm_tgt_type *tt; - - list_for_each_entry(tt, &nvm_tgt_types, list) - if (!strcmp(name, tt->name)) - return tt; + list_for_each_entry(tmp, &nvm_tgt_types, list) + if (!strcmp(name, tmp->name)) { + tt = tmp; + break; + } - return NULL; + if (lock) + up_write(&nvm_tgtt_lock); + return tt; } +EXPORT_SYMBOL(nvm_find_target_type); int nvm_register_tgt_type(struct nvm_tgt_type *tt) { int ret = 0; - down_write(&nvm_lock); - if (nvm_find_target_type(tt->name)) + down_write(&nvm_tgtt_lock); + if (nvm_find_target_type(tt->name, 0)) ret = -EEXIST; else list_add(&tt->list, &nvm_tgt_types); - up_write(&nvm_lock); + up_write(&nvm_tgtt_lock); return ret; } @@ -110,7 +104,7 @@ static struct nvmm_type *nvm_find_mgr_type(const char *name) return NULL; } -struct nvmm_type *nvm_init_mgr(struct nvm_dev *dev) +static struct nvmm_type *nvm_init_mgr(struct nvm_dev *dev) { struct nvmm_type *mt; int ret; @@ -182,20 +176,6 @@ static struct nvm_dev *nvm_find_nvm_dev(const char *name) return NULL; } -struct nvm_block *nvm_get_blk_unlocked(struct nvm_dev *dev, struct nvm_lun *lun, - unsigned long flags) -{ - return dev->mt->get_blk_unlocked(dev, lun, flags); -} -EXPORT_SYMBOL(nvm_get_blk_unlocked); - -/* Assumes that all valid pages have already been moved on release to bm */ -void nvm_put_blk_unlocked(struct nvm_dev *dev, struct nvm_block *blk) -{ - return dev->mt->put_blk_unlocked(dev, blk); -} -EXPORT_SYMBOL(nvm_put_blk_unlocked); - struct nvm_block *nvm_get_blk(struct nvm_dev *dev, struct nvm_lun *lun, unsigned long flags) { @@ -210,6 +190,12 @@ void nvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk) } EXPORT_SYMBOL(nvm_put_blk); +void nvm_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type) +{ + return dev->mt->mark_blk(dev, ppa, type); +} +EXPORT_SYMBOL(nvm_mark_blk); + int nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) { return dev->mt->submit_io(dev, rqd); @@ -251,9 +237,10 @@ void nvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd) EXPORT_SYMBOL(nvm_generic_to_addr_mode); int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd, - struct ppa_addr *ppas, int nr_ppas, int vblk) + const struct ppa_addr *ppas, int nr_ppas, int vblk) { int i, plane_cnt, pl_idx; + struct ppa_addr ppa; if ((!vblk || dev->plane_mode == NVM_PLANE_SINGLE) && nr_ppas == 1) { rqd->nr_ppas = nr_ppas; @@ -278,8 +265,9 @@ int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd, for (i = 0; i < nr_ppas; i++) { for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) { - ppas[i].g.pl = pl_idx; - rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppas[i]; + ppa = ppas[i]; + ppa.g.pl = pl_idx; + rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppa; } } } @@ -337,7 +325,7 @@ static void nvm_end_io_sync(struct nvm_rq *rqd) complete(waiting); } -int __nvm_submit_ppa(struct nvm_dev *dev, struct nvm_rq *rqd, int opcode, +static int __nvm_submit_ppa(struct nvm_dev *dev, struct nvm_rq *rqd, int opcode, int flags, void *buf, int len) { DECLARE_COMPLETION_ONSTACK(wait); @@ -367,7 +355,9 @@ int __nvm_submit_ppa(struct nvm_dev *dev, struct nvm_rq *rqd, int opcode, /* Prevent hang_check timer from firing at us during very long I/O */ hang_check = sysctl_hung_task_timeout_secs; if (hang_check) - while (!wait_for_completion_io_timeout(&wait, hang_check * (HZ/2))); + while (!wait_for_completion_io_timeout(&wait, + hang_check * (HZ/2))) + ; else wait_for_completion_io(&wait); @@ -510,7 +500,8 @@ static int nvm_init_mlc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp) /* The lower page table encoding consists of a list of bytes, where each * has a lower and an upper half. The first half byte maintains the * increment value and every value after is an offset added to the - * previous incrementation value */ + * previous incrementation value + */ dev->lptbl[0] = mlc->pairs[0] & 0xF; for (i = 1; i < dev->lps_per_blk; i++) { p = mlc->pairs[i >> 1]; @@ -596,42 +587,11 @@ err_fmtype: return ret; } -static void nvm_remove_target(struct nvm_target *t) -{ - struct nvm_tgt_type *tt = t->type; - struct gendisk *tdisk = t->disk; - struct request_queue *q = tdisk->queue; - - lockdep_assert_held(&nvm_lock); - - del_gendisk(tdisk); - blk_cleanup_queue(q); - - if (tt->exit) - tt->exit(tdisk->private_data); - - put_disk(tdisk); - - list_del(&t->list); - kfree(t); -} - static void nvm_free_mgr(struct nvm_dev *dev) { - struct nvm_target *tgt, *tmp; - if (!dev->mt) return; - down_write(&nvm_lock); - list_for_each_entry_safe(tgt, tmp, &nvm_targets, list) { - if (tgt->dev != dev) - continue; - - nvm_remove_target(tgt); - } - up_write(&nvm_lock); - dev->mt->unregister_mgr(dev); dev->mt = NULL; } @@ -778,91 +738,6 @@ void nvm_unregister(char *disk_name) } EXPORT_SYMBOL(nvm_unregister); -static const struct block_device_operations nvm_fops = { - .owner = THIS_MODULE, -}; - -static int nvm_create_target(struct nvm_dev *dev, - struct nvm_ioctl_create *create) -{ - struct nvm_ioctl_create_simple *s = &create->conf.s; - struct request_queue *tqueue; - struct gendisk *tdisk; - struct nvm_tgt_type *tt; - struct nvm_target *t; - void *targetdata; - - if (!dev->mt) { - pr_info("nvm: device has no media manager registered.\n"); - return -ENODEV; - } - - down_write(&nvm_lock); - tt = nvm_find_target_type(create->tgttype); - if (!tt) { - pr_err("nvm: target type %s not found\n", create->tgttype); - up_write(&nvm_lock); - return -EINVAL; - } - - t = nvm_find_target(create->tgtname); - if (t) { - pr_err("nvm: target name already exists.\n"); - up_write(&nvm_lock); - return -EINVAL; - } - up_write(&nvm_lock); - - t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL); - if (!t) - return -ENOMEM; - - tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node); - if (!tqueue) - goto err_t; - blk_queue_make_request(tqueue, tt->make_rq); - - tdisk = alloc_disk(0); - if (!tdisk) - goto err_queue; - - sprintf(tdisk->disk_name, "%s", create->tgtname); - tdisk->flags = GENHD_FL_EXT_DEVT; - tdisk->major = 0; - tdisk->first_minor = 0; - tdisk->fops = &nvm_fops; - tdisk->queue = tqueue; - - targetdata = tt->init(dev, tdisk, s->lun_begin, s->lun_end); - if (IS_ERR(targetdata)) - goto err_init; - - tdisk->private_data = targetdata; - tqueue->queuedata = targetdata; - - blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect); - - set_capacity(tdisk, tt->capacity(targetdata)); - add_disk(tdisk); - - t->type = tt; - t->disk = tdisk; - t->dev = dev; - - down_write(&nvm_lock); - list_add_tail(&t->list, &nvm_targets); - up_write(&nvm_lock); - - return 0; -err_init: - put_disk(tdisk); -err_queue: - blk_cleanup_queue(tqueue); -err_t: - kfree(t); - return -ENOMEM; -} - static int __nvm_configure_create(struct nvm_ioctl_create *create) { struct nvm_dev *dev; @@ -871,11 +746,17 @@ static int __nvm_configure_create(struct nvm_ioctl_create *create) down_write(&nvm_lock); dev = nvm_find_nvm_dev(create->dev); up_write(&nvm_lock); + if (!dev) { pr_err("nvm: device not found\n"); return -EINVAL; } + if (!dev->mt) { + pr_info("nvm: device has no media manager registered.\n"); + return -ENODEV; + } + if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) { pr_err("nvm: config type not valid\n"); return -EINVAL; @@ -888,25 +769,7 @@ static int __nvm_configure_create(struct nvm_ioctl_create *create) return -EINVAL; } - return nvm_create_target(dev, create); -} - -static int __nvm_configure_remove(struct nvm_ioctl_remove *remove) -{ - struct nvm_target *t; - - down_write(&nvm_lock); - t = nvm_find_target(remove->tgtname); - if (!t) { - pr_err("nvm: target \"%s\" doesn't exist.\n", remove->tgtname); - up_write(&nvm_lock); - return -EINVAL; - } - - nvm_remove_target(t); - up_write(&nvm_lock); - - return 0; + return dev->mt->create_tgt(dev, create); } #ifdef CONFIG_NVM_DEBUG @@ -941,8 +804,9 @@ static int nvm_configure_show(const char *val) static int nvm_configure_remove(const char *val) { struct nvm_ioctl_remove remove; + struct nvm_dev *dev; char opcode; - int ret; + int ret = 0; ret = sscanf(val, "%c %256s", &opcode, remove.tgtname); if (ret != 2) { @@ -952,7 +816,13 @@ static int nvm_configure_remove(const char *val) remove.flags = 0; - return __nvm_configure_remove(&remove); + list_for_each_entry(dev, &nvm_devices, devices) { + ret = dev->mt->remove_tgt(dev, &remove); + if (!ret) + break; + } + + return ret; } static int nvm_configure_create(const char *val) @@ -1149,6 +1019,8 @@ static long nvm_ioctl_dev_create(struct file *file, void __user *arg) static long nvm_ioctl_dev_remove(struct file *file, void __user *arg) { struct nvm_ioctl_remove remove; + struct nvm_dev *dev; + int ret = 0; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -1163,7 +1035,13 @@ static long nvm_ioctl_dev_remove(struct file *file, void __user *arg) return -EINVAL; } - return __nvm_configure_remove(&remove); + list_for_each_entry(dev, &nvm_devices, devices) { + ret = dev->mt->remove_tgt(dev, &remove); + if (!ret) + break; + } + + return ret; } static void nvm_setup_nvm_sb_info(struct nvm_sb_info *info) diff --git a/drivers/lightnvm/gennvm.c b/drivers/lightnvm/gennvm.c index ec9fb6876e38..b74174c6d021 100644 --- a/drivers/lightnvm/gennvm.c +++ b/drivers/lightnvm/gennvm.c @@ -15,22 +15,160 @@ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, * USA. * - * Implementation of a generic nvm manager for Open-Channel SSDs. + * Implementation of a general nvm manager for Open-Channel SSDs. */ #include "gennvm.h" -static int gennvm_get_area(struct nvm_dev *dev, sector_t *lba, sector_t len) +static struct nvm_target *gen_find_target(struct gen_dev *gn, const char *name) { - struct gen_nvm *gn = dev->mp; - struct gennvm_area *area, *prev, *next; + struct nvm_target *tgt; + + list_for_each_entry(tgt, &gn->targets, list) + if (!strcmp(name, tgt->disk->disk_name)) + return tgt; + + return NULL; +} + +static const struct block_device_operations gen_fops = { + .owner = THIS_MODULE, +}; + +static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create) +{ + struct gen_dev *gn = dev->mp; + struct nvm_ioctl_create_simple *s = &create->conf.s; + struct request_queue *tqueue; + struct gendisk *tdisk; + struct nvm_tgt_type *tt; + struct nvm_target *t; + void *targetdata; + + tt = nvm_find_target_type(create->tgttype, 1); + if (!tt) { + pr_err("nvm: target type %s not found\n", create->tgttype); + return -EINVAL; + } + + mutex_lock(&gn->lock); + t = gen_find_target(gn, create->tgtname); + if (t) { + pr_err("nvm: target name already exists.\n"); + mutex_unlock(&gn->lock); + return -EINVAL; + } + mutex_unlock(&gn->lock); + + t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL); + if (!t) + return -ENOMEM; + + tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node); + if (!tqueue) + goto err_t; + blk_queue_make_request(tqueue, tt->make_rq); + + tdisk = alloc_disk(0); + if (!tdisk) + goto err_queue; + + sprintf(tdisk->disk_name, "%s", create->tgtname); + tdisk->flags = GENHD_FL_EXT_DEVT; + tdisk->major = 0; + tdisk->first_minor = 0; + tdisk->fops = &gen_fops; + tdisk->queue = tqueue; + + targetdata = tt->init(dev, tdisk, s->lun_begin, s->lun_end); + if (IS_ERR(targetdata)) + goto err_init; + + tdisk->private_data = targetdata; + tqueue->queuedata = targetdata; + + blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect); + + set_capacity(tdisk, tt->capacity(targetdata)); + add_disk(tdisk); + + t->type = tt; + t->disk = tdisk; + t->dev = dev; + + mutex_lock(&gn->lock); + list_add_tail(&t->list, &gn->targets); + mutex_unlock(&gn->lock); + + return 0; +err_init: + put_disk(tdisk); +err_queue: + blk_cleanup_queue(tqueue); +err_t: + kfree(t); + return -ENOMEM; +} + +static void __gen_remove_target(struct nvm_target *t) +{ + struct nvm_tgt_type *tt = t->type; + struct gendisk *tdisk = t->disk; + struct request_queue *q = tdisk->queue; + + del_gendisk(tdisk); + blk_cleanup_queue(q); + + if (tt->exit) + tt->exit(tdisk->private_data); + + put_disk(tdisk); + + list_del(&t->list); + kfree(t); +} + +/** + * gen_remove_tgt - Removes a target from the media manager + * @dev: device + * @remove: ioctl structure with target name to remove. + * + * Returns: + * 0: on success + * 1: on not found + * <0: on error + */ +static int gen_remove_tgt(struct nvm_dev *dev, struct nvm_ioctl_remove *remove) +{ + struct gen_dev *gn = dev->mp; + struct nvm_target *t; + + if (!gn) + return 1; + + mutex_lock(&gn->lock); + t = gen_find_target(gn, remove->tgtname); + if (!t) { + mutex_unlock(&gn->lock); + return 1; + } + __gen_remove_target(t); + mutex_unlock(&gn->lock); + + return 0; +} + +static int gen_get_area(struct nvm_dev *dev, sector_t *lba, sector_t len) +{ + struct gen_dev *gn = dev->mp; + struct gen_area *area, *prev, *next; sector_t begin = 0; sector_t max_sectors = (dev->sec_size * dev->total_secs) >> 9; if (len > max_sectors) return -EINVAL; - area = kmalloc(sizeof(struct gennvm_area), GFP_KERNEL); + area = kmalloc(sizeof(struct gen_area), GFP_KERNEL); if (!area) return -ENOMEM; @@ -64,10 +202,10 @@ static int gennvm_get_area(struct nvm_dev *dev, sector_t *lba, sector_t len) return 0; } -static void gennvm_put_area(struct nvm_dev *dev, sector_t begin) +static void gen_put_area(struct nvm_dev *dev, sector_t begin) { - struct gen_nvm *gn = dev->mp; - struct gennvm_area *area; + struct gen_dev *gn = dev->mp; + struct gen_area *area; spin_lock(&dev->lock); list_for_each_entry(area, &gn->area_list, list) { @@ -82,27 +220,27 @@ static void gennvm_put_area(struct nvm_dev *dev, sector_t begin) spin_unlock(&dev->lock); } -static void gennvm_blocks_free(struct nvm_dev *dev) +static void gen_blocks_free(struct nvm_dev *dev) { - struct gen_nvm *gn = dev->mp; + struct gen_dev *gn = dev->mp; struct gen_lun *lun; int i; - gennvm_for_each_lun(gn, lun, i) { + gen_for_each_lun(gn, lun, i) { if (!lun->vlun.blocks) break; vfree(lun->vlun.blocks); } } -static void gennvm_luns_free(struct nvm_dev *dev) +static void gen_luns_free(struct nvm_dev *dev) { - struct gen_nvm *gn = dev->mp; + struct gen_dev *gn = dev->mp; kfree(gn->luns); } -static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn) +static int gen_luns_init(struct nvm_dev *dev, struct gen_dev *gn) { struct gen_lun *lun; int i; @@ -111,7 +249,7 @@ static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn) if (!gn->luns) return -ENOMEM; - gennvm_for_each_lun(gn, lun, i) { + gen_for_each_lun(gn, lun, i) { spin_lock_init(&lun->vlun.lock); INIT_LIST_HEAD(&lun->free_list); INIT_LIST_HEAD(&lun->used_list); @@ -122,14 +260,11 @@ static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn) lun->vlun.lun_id = i % dev->luns_per_chnl; lun->vlun.chnl_id = i / dev->luns_per_chnl; lun->vlun.nr_free_blocks = dev->blks_per_lun; - lun->vlun.nr_open_blocks = 0; - lun->vlun.nr_closed_blocks = 0; - lun->vlun.nr_bad_blocks = 0; } return 0; } -static int gennvm_block_bb(struct gen_nvm *gn, struct ppa_addr ppa, +static int gen_block_bb(struct gen_dev *gn, struct ppa_addr ppa, u8 *blks, int nr_blks) { struct nvm_dev *dev = gn->dev; @@ -149,17 +284,16 @@ static int gennvm_block_bb(struct gen_nvm *gn, struct ppa_addr ppa, blk = &lun->vlun.blocks[i]; list_move_tail(&blk->list, &lun->bb_list); - lun->vlun.nr_bad_blocks++; lun->vlun.nr_free_blocks--; } return 0; } -static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) +static int gen_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) { struct nvm_dev *dev = private; - struct gen_nvm *gn = dev->mp; + struct gen_dev *gn = dev->mp; u64 elba = slba + nlb; struct gen_lun *lun; struct nvm_block *blk; @@ -167,7 +301,7 @@ static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) int lun_id; if (unlikely(elba > dev->total_secs)) { - pr_err("gennvm: L2P data from device is out of bounds!\n"); + pr_err("gen: L2P data from device is out of bounds!\n"); return -EINVAL; } @@ -175,7 +309,7 @@ static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) u64 pba = le64_to_cpu(entries[i]); if (unlikely(pba >= dev->total_secs && pba != U64_MAX)) { - pr_err("gennvm: L2P data entry is out of bounds!\n"); + pr_err("gen: L2P data entry is out of bounds!\n"); return -EINVAL; } @@ -200,16 +334,15 @@ static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) * block state. The block is assumed to be open. */ list_move_tail(&blk->list, &lun->used_list); - blk->state = NVM_BLK_ST_OPEN; + blk->state = NVM_BLK_ST_TGT; lun->vlun.nr_free_blocks--; - lun->vlun.nr_open_blocks++; } } return 0; } -static int gennvm_blocks_init(struct nvm_dev *dev, struct gen_nvm *gn) +static int gen_blocks_init(struct nvm_dev *dev, struct gen_dev *gn) { struct gen_lun *lun; struct nvm_block *block; @@ -222,7 +355,7 @@ static int gennvm_blocks_init(struct nvm_dev *dev, struct gen_nvm *gn) if (!blks) return -ENOMEM; - gennvm_for_each_lun(gn, lun, lun_iter) { + gen_for_each_lun(gn, lun, lun_iter) { lun->vlun.blocks = vzalloc(sizeof(struct nvm_block) * dev->blks_per_lun); if (!lun->vlun.blocks) { @@ -256,20 +389,20 @@ static int gennvm_blocks_init(struct nvm_dev *dev, struct gen_nvm *gn) ret = nvm_get_bb_tbl(dev, ppa, blks); if (ret) - pr_err("gennvm: could not get BB table\n"); + pr_err("gen: could not get BB table\n"); - ret = gennvm_block_bb(gn, ppa, blks, nr_blks); + ret = gen_block_bb(gn, ppa, blks, nr_blks); if (ret) - pr_err("gennvm: BB table map failed\n"); + pr_err("gen: BB table map failed\n"); } } if ((dev->identity.dom & NVM_RSP_L2P) && dev->ops->get_l2p_tbl) { ret = dev->ops->get_l2p_tbl(dev, 0, dev->total_secs, - gennvm_block_map, dev); + gen_block_map, dev); if (ret) { - pr_err("gennvm: could not read L2P table.\n"); - pr_warn("gennvm: default block initialization"); + pr_err("gen: could not read L2P table.\n"); + pr_warn("gen: default block initialization"); } } @@ -277,67 +410,79 @@ static int gennvm_blocks_init(struct nvm_dev *dev, struct gen_nvm *gn) return 0; } -static void gennvm_free(struct nvm_dev *dev) +static void gen_free(struct nvm_dev *dev) { - gennvm_blocks_free(dev); - gennvm_luns_free(dev); + gen_blocks_free(dev); + gen_luns_free(dev); kfree(dev->mp); dev->mp = NULL; } -static int gennvm_register(struct nvm_dev *dev) +static int gen_register(struct nvm_dev *dev) { - struct gen_nvm *gn; + struct gen_dev *gn; int ret; if (!try_module_get(THIS_MODULE)) return -ENODEV; - gn = kzalloc(sizeof(struct gen_nvm), GFP_KERNEL); + gn = kzalloc(sizeof(struct gen_dev), GFP_KERNEL); if (!gn) return -ENOMEM; gn->dev = dev; gn->nr_luns = dev->nr_luns; INIT_LIST_HEAD(&gn->area_list); + mutex_init(&gn->lock); + INIT_LIST_HEAD(&gn->targets); dev->mp = gn; - ret = gennvm_luns_init(dev, gn); + ret = gen_luns_init(dev, gn); if (ret) { - pr_err("gennvm: could not initialize luns\n"); + pr_err("gen: could not initialize luns\n"); goto err; } - ret = gennvm_blocks_init(dev, gn); + ret = gen_blocks_init(dev, gn); if (ret) { - pr_err("gennvm: could not initialize blocks\n"); + pr_err("gen: could not initialize blocks\n"); goto err; } return 1; err: - gennvm_free(dev); + gen_free(dev); module_put(THIS_MODULE); return ret; } -static void gennvm_unregister(struct nvm_dev *dev) +static void gen_unregister(struct nvm_dev *dev) { - gennvm_free(dev); + struct gen_dev *gn = dev->mp; + struct nvm_target *t, *tmp; + + mutex_lock(&gn->lock); + list_for_each_entry_safe(t, tmp, &gn->targets, list) { + if (t->dev != dev) + continue; + __gen_remove_target(t); + } + mutex_unlock(&gn->lock); + + gen_free(dev); module_put(THIS_MODULE); } -static struct nvm_block *gennvm_get_blk_unlocked(struct nvm_dev *dev, +static struct nvm_block *gen_get_blk(struct nvm_dev *dev, struct nvm_lun *vlun, unsigned long flags) { struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); struct nvm_block *blk = NULL; int is_gc = flags & NVM_IOTYPE_GC; - assert_spin_locked(&vlun->lock); - + spin_lock(&vlun->lock); if (list_empty(&lun->free_list)) { - pr_err_ratelimited("gennvm: lun %u have no free pages available", + pr_err_ratelimited("gen: lun %u have no free pages available", lun->vlun.id); goto out; } @@ -346,88 +491,58 @@ static struct nvm_block *gennvm_get_blk_unlocked(struct nvm_dev *dev, goto out; blk = list_first_entry(&lun->free_list, struct nvm_block, list); - list_move_tail(&blk->list, &lun->used_list); - blk->state = NVM_BLK_ST_OPEN; + list_move_tail(&blk->list, &lun->used_list); + blk->state = NVM_BLK_ST_TGT; lun->vlun.nr_free_blocks--; - lun->vlun.nr_open_blocks++; - out: - return blk; -} - -static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev, - struct nvm_lun *vlun, unsigned long flags) -{ - struct nvm_block *blk; - - spin_lock(&vlun->lock); - blk = gennvm_get_blk_unlocked(dev, vlun, flags); spin_unlock(&vlun->lock); return blk; } -static void gennvm_put_blk_unlocked(struct nvm_dev *dev, struct nvm_block *blk) +static void gen_put_blk(struct nvm_dev *dev, struct nvm_block *blk) { struct nvm_lun *vlun = blk->lun; struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); - assert_spin_locked(&vlun->lock); - - if (blk->state & NVM_BLK_ST_OPEN) { - list_move_tail(&blk->list, &lun->free_list); - lun->vlun.nr_open_blocks--; - lun->vlun.nr_free_blocks++; - blk->state = NVM_BLK_ST_FREE; - } else if (blk->state & NVM_BLK_ST_CLOSED) { + spin_lock(&vlun->lock); + if (blk->state & NVM_BLK_ST_TGT) { list_move_tail(&blk->list, &lun->free_list); - lun->vlun.nr_closed_blocks--; lun->vlun.nr_free_blocks++; blk->state = NVM_BLK_ST_FREE; } else if (blk->state & NVM_BLK_ST_BAD) { list_move_tail(&blk->list, &lun->bb_list); - lun->vlun.nr_bad_blocks++; blk->state = NVM_BLK_ST_BAD; } else { WARN_ON_ONCE(1); - pr_err("gennvm: erroneous block type (%lu -> %u)\n", + pr_err("gen: erroneous block type (%lu -> %u)\n", blk->id, blk->state); list_move_tail(&blk->list, &lun->bb_list); - lun->vlun.nr_bad_blocks++; - blk->state = NVM_BLK_ST_BAD; } -} - -static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk) -{ - struct nvm_lun *vlun = blk->lun; - - spin_lock(&vlun->lock); - gennvm_put_blk_unlocked(dev, blk); spin_unlock(&vlun->lock); } -static void gennvm_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type) +static void gen_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type) { - struct gen_nvm *gn = dev->mp; + struct gen_dev *gn = dev->mp; struct gen_lun *lun; struct nvm_block *blk; - pr_debug("gennvm: ppa (ch: %u lun: %u blk: %u pg: %u) -> %u\n", + pr_debug("gen: ppa (ch: %u lun: %u blk: %u pg: %u) -> %u\n", ppa.g.ch, ppa.g.lun, ppa.g.blk, ppa.g.pg, type); if (unlikely(ppa.g.ch > dev->nr_chnls || ppa.g.lun > dev->luns_per_chnl || ppa.g.blk > dev->blks_per_lun)) { WARN_ON_ONCE(1); - pr_err("gennvm: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u", + pr_err("gen: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u", ppa.g.ch, dev->nr_chnls, ppa.g.lun, dev->luns_per_chnl, ppa.g.blk, dev->blks_per_lun); return; } - lun = &gn->luns[ppa.g.lun * ppa.g.ch]; + lun = &gn->luns[(dev->luns_per_chnl * ppa.g.ch) + ppa.g.lun]; blk = &lun->vlun.blocks[ppa.g.blk]; /* will be moved to bb list on put_blk from target */ @@ -435,9 +550,9 @@ static void gennvm_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type) } /* - * mark block bad in gennvm. It is expected that the target recovers separately + * mark block bad in gen. It is expected that the target recovers separately */ -static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd) +static void gen_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd) { int bit = -1; int max_secs = dev->ops->max_phys_sect; @@ -447,25 +562,25 @@ static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd) /* look up blocks and mark them as bad */ if (rqd->nr_ppas == 1) { - gennvm_mark_blk(dev, rqd->ppa_addr, NVM_BLK_ST_BAD); + gen_mark_blk(dev, rqd->ppa_addr, NVM_BLK_ST_BAD); return; } while ((bit = find_next_bit(comp_bits, max_secs, bit + 1)) < max_secs) - gennvm_mark_blk(dev, rqd->ppa_list[bit], NVM_BLK_ST_BAD); + gen_mark_blk(dev, rqd->ppa_list[bit], NVM_BLK_ST_BAD); } -static void gennvm_end_io(struct nvm_rq *rqd) +static void gen_end_io(struct nvm_rq *rqd) { struct nvm_tgt_instance *ins = rqd->ins; if (rqd->error == NVM_RSP_ERR_FAILWRITE) - gennvm_mark_blk_bad(rqd->dev, rqd); + gen_mark_blk_bad(rqd->dev, rqd); ins->tt->end_io(rqd); } -static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) +static int gen_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) { if (!dev->ops->submit_io) return -ENODEV; @@ -474,11 +589,11 @@ static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) nvm_generic_to_addr_mode(dev, rqd); rqd->dev = dev; - rqd->end_io = gennvm_end_io; + rqd->end_io = gen_end_io; return dev->ops->submit_io(dev, rqd); } -static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, +static int gen_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, unsigned long flags) { struct ppa_addr addr = block_to_ppa(dev, blk); @@ -486,19 +601,19 @@ static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, return nvm_erase_ppa(dev, &addr, 1); } -static int gennvm_reserve_lun(struct nvm_dev *dev, int lunid) +static int gen_reserve_lun(struct nvm_dev *dev, int lunid) { return test_and_set_bit(lunid, dev->lun_map); } -static void gennvm_release_lun(struct nvm_dev *dev, int lunid) +static void gen_release_lun(struct nvm_dev *dev, int lunid) { WARN_ON(!test_and_clear_bit(lunid, dev->lun_map)); } -static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid) +static struct nvm_lun *gen_get_lun(struct nvm_dev *dev, int lunid) { - struct gen_nvm *gn = dev->mp; + struct gen_dev *gn = dev->mp; if (unlikely(lunid >= dev->nr_luns)) return NULL; @@ -506,66 +621,62 @@ static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid) return &gn->luns[lunid].vlun; } -static void gennvm_lun_info_print(struct nvm_dev *dev) +static void gen_lun_info_print(struct nvm_dev *dev) { - struct gen_nvm *gn = dev->mp; + struct gen_dev *gn = dev->mp; struct gen_lun *lun; unsigned int i; - gennvm_for_each_lun(gn, lun, i) { + gen_for_each_lun(gn, lun, i) { spin_lock(&lun->vlun.lock); - pr_info("%s: lun%8u\t%u\t%u\t%u\t%u\n", - dev->name, i, - lun->vlun.nr_free_blocks, - lun->vlun.nr_open_blocks, - lun->vlun.nr_closed_blocks, - lun->vlun.nr_bad_blocks); + pr_info("%s: lun%8u\t%u\n", dev->name, i, + lun->vlun.nr_free_blocks); spin_unlock(&lun->vlun.lock); } } -static struct nvmm_type gennvm = { +static struct nvmm_type gen = { .name = "gennvm", .version = {0, 1, 0}, - .register_mgr = gennvm_register, - .unregister_mgr = gennvm_unregister, + .register_mgr = gen_register, + .unregister_mgr = gen_unregister, - .get_blk_unlocked = gennvm_get_blk_unlocked, - .put_blk_unlocked = gennvm_put_blk_unlocked, + .create_tgt = gen_create_tgt, + .remove_tgt = gen_remove_tgt, - .get_blk = gennvm_get_blk, - .put_blk = gennvm_put_blk, + .get_blk = gen_get_blk, + .put_blk = gen_put_blk, - .submit_io = gennvm_submit_io, - .erase_blk = gennvm_erase_blk, + .submit_io = gen_submit_io, + .erase_blk = gen_erase_blk, - .mark_blk = gennvm_mark_blk, + .mark_blk = gen_mark_blk, - .get_lun = gennvm_get_lun, - .reserve_lun = gennvm_reserve_lun, - .release_lun = gennvm_release_lun, - .lun_info_print = gennvm_lun_info_print, + .get_lun = gen_get_lun, + .reserve_lun = gen_reserve_lun, + .release_lun = gen_release_lun, + .lun_info_print = gen_lun_info_print, - .get_area = gennvm_get_area, - .put_area = gennvm_put_area, + .get_area = gen_get_area, + .put_area = gen_put_area, }; -static int __init gennvm_module_init(void) +static int __init gen_module_init(void) { - return nvm_register_mgr(&gennvm); + return nvm_register_mgr(&gen); } -static void gennvm_module_exit(void) +static void gen_module_exit(void) { - nvm_unregister_mgr(&gennvm); + nvm_unregister_mgr(&gen); } -module_init(gennvm_module_init); -module_exit(gennvm_module_exit); +module_init(gen_module_init); +module_exit(gen_module_exit); MODULE_LICENSE("GPL v2"); -MODULE_DESCRIPTION("Generic media manager for Open-Channel SSDs"); +MODULE_DESCRIPTION("General media manager for Open-Channel SSDs"); diff --git a/drivers/lightnvm/gennvm.h b/drivers/lightnvm/gennvm.h index 04d7c23cfc61..8ecfa817d21d 100644 --- a/drivers/lightnvm/gennvm.h +++ b/drivers/lightnvm/gennvm.h @@ -34,20 +34,24 @@ struct gen_lun { */ }; -struct gen_nvm { +struct gen_dev { struct nvm_dev *dev; int nr_luns; struct gen_lun *luns; struct list_head area_list; + + struct mutex lock; + struct list_head targets; }; -struct gennvm_area { +struct gen_area { struct list_head list; sector_t begin; sector_t end; /* end is excluded */ }; -#define gennvm_for_each_lun(bm, lun, i) \ + +#define gen_for_each_lun(bm, lun, i) \ for ((i) = 0, lun = &(bm)->luns[0]; \ (i) < (bm)->nr_luns; (i)++, lun = &(bm)->luns[(i)]) diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c index 2103e97a974f..37fcaadbf80c 100644 --- a/drivers/lightnvm/rrpc.c +++ b/drivers/lightnvm/rrpc.c @@ -48,7 +48,7 @@ static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a) } static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba, - unsigned len) + unsigned int len) { sector_t i; @@ -96,10 +96,13 @@ static void rrpc_discard(struct rrpc *rrpc, struct bio *bio) sector_t len = bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE; struct nvm_rq *rqd; - do { + while (1) { rqd = rrpc_inflight_laddr_acquire(rrpc, slba, len); + if (rqd) + break; + schedule(); - } while (!rqd); + } if (IS_ERR(rqd)) { pr_err("rrpc: unable to acquire inflight IO\n"); @@ -172,39 +175,32 @@ static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_dev *dev, u64 addr) } /* requires lun->lock taken */ -static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *rblk) +static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *new_rblk, + struct rrpc_block **cur_rblk) { struct rrpc *rrpc = rlun->rrpc; - BUG_ON(!rblk); - - if (rlun->cur) { - spin_lock(&rlun->cur->lock); - WARN_ON(!block_is_full(rrpc, rlun->cur)); - spin_unlock(&rlun->cur->lock); + if (*cur_rblk) { + spin_lock(&(*cur_rblk)->lock); + WARN_ON(!block_is_full(rrpc, *cur_rblk)); + spin_unlock(&(*cur_rblk)->lock); } - rlun->cur = rblk; + *cur_rblk = new_rblk; } static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun, unsigned long flags) { - struct nvm_lun *lun = rlun->parent; struct nvm_block *blk; struct rrpc_block *rblk; - spin_lock(&lun->lock); - blk = nvm_get_blk_unlocked(rrpc->dev, rlun->parent, flags); + blk = nvm_get_blk(rrpc->dev, rlun->parent, flags); if (!blk) { pr_err("nvm: rrpc: cannot get new block from media manager\n"); - spin_unlock(&lun->lock); return NULL; } rblk = rrpc_get_rblk(rlun, blk->id); - list_add_tail(&rblk->list, &rlun->open_list); - spin_unlock(&lun->lock); - blk->priv = rblk; bitmap_zero(rblk->invalid_pages, rrpc->dev->sec_per_blk); rblk->next_page = 0; @@ -216,13 +212,7 @@ static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun, static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk) { - struct rrpc_lun *rlun = rblk->rlun; - struct nvm_lun *lun = rlun->parent; - - spin_lock(&lun->lock); - nvm_put_blk_unlocked(rrpc->dev, rblk->parent); - list_del(&rblk->list); - spin_unlock(&lun->lock); + nvm_put_blk(rrpc->dev, rblk->parent); } static void rrpc_put_blks(struct rrpc *rrpc) @@ -342,7 +332,7 @@ try: /* Perform read to do GC */ bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr); - bio->bi_rw = READ; + bio_set_op_attrs(bio, REQ_OP_READ, 0); bio->bi_private = &wait; bio->bi_end_io = rrpc_end_sync_bio; @@ -364,7 +354,7 @@ try: reinit_completion(&wait); bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr); - bio->bi_rw = WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio->bi_private = &wait; bio->bi_end_io = rrpc_end_sync_bio; @@ -508,21 +498,11 @@ static void rrpc_gc_queue(struct work_struct *work) struct rrpc *rrpc = gcb->rrpc; struct rrpc_block *rblk = gcb->rblk; struct rrpc_lun *rlun = rblk->rlun; - struct nvm_lun *lun = rblk->parent->lun; - struct nvm_block *blk = rblk->parent; spin_lock(&rlun->lock); list_add_tail(&rblk->prio, &rlun->prio_list); spin_unlock(&rlun->lock); - spin_lock(&lun->lock); - lun->nr_open_blocks--; - lun->nr_closed_blocks++; - blk->state &= ~NVM_BLK_ST_OPEN; - blk->state |= NVM_BLK_ST_CLOSED; - list_move_tail(&rblk->list, &rlun->closed_list); - spin_unlock(&lun->lock); - mempool_free(gcb, rrpc->gcb_pool); pr_debug("nvm: block '%lu' is full, allow GC (sched)\n", rblk->parent->id); @@ -596,21 +576,20 @@ out: return addr; } -/* Simple round-robin Logical to physical address translation. - * - * Retrieve the mapping using the active append point. Then update the ap for - * the next write to the disk. +/* Map logical address to a physical page. The mapping implements a round robin + * approach and allocates a page from the next lun available. * - * Returns rrpc_addr with the physical address and block. Remember to return to - * rrpc->addr_cache when request is finished. + * Returns rrpc_addr with the physical address and block. Returns NULL if no + * blocks in the next rlun are available. */ static struct rrpc_addr *rrpc_map_page(struct rrpc *rrpc, sector_t laddr, int is_gc) { struct rrpc_lun *rlun; - struct rrpc_block *rblk; + struct rrpc_block *rblk, **cur_rblk; struct nvm_lun *lun; u64 paddr; + int gc_force = 0; rlun = rrpc_get_lun_rr(rrpc, is_gc); lun = rlun->parent; @@ -618,41 +597,65 @@ static struct rrpc_addr *rrpc_map_page(struct rrpc *rrpc, sector_t laddr, if (!is_gc && lun->nr_free_blocks < rrpc->nr_luns * 4) return NULL; - spin_lock(&rlun->lock); + /* + * page allocation steps: + * 1. Try to allocate new page from current rblk + * 2a. If succeed, proceed to map it in and return + * 2b. If fail, first try to allocate a new block from media manger, + * and then retry step 1. Retry until the normal block pool is + * exhausted. + * 3. If exhausted, and garbage collector is requesting the block, + * go to the reserved block and retry step 1. + * In the case that this fails as well, or it is not GC + * requesting, report not able to retrieve a block and let the + * caller handle further processing. + */ + spin_lock(&rlun->lock); + cur_rblk = &rlun->cur; rblk = rlun->cur; retry: paddr = rrpc_alloc_addr(rrpc, rblk); - if (paddr == ADDR_EMPTY) { - rblk = rrpc_get_blk(rrpc, rlun, 0); - if (rblk) { - rrpc_set_lun_cur(rlun, rblk); - goto retry; - } + if (paddr != ADDR_EMPTY) + goto done; - if (is_gc) { - /* retry from emergency gc block */ - paddr = rrpc_alloc_addr(rrpc, rlun->gc_cur); - if (paddr == ADDR_EMPTY) { - rblk = rrpc_get_blk(rrpc, rlun, 1); - if (!rblk) { - pr_err("rrpc: no more blocks"); - goto err; - } - - rlun->gc_cur = rblk; - paddr = rrpc_alloc_addr(rrpc, rlun->gc_cur); - } - rblk = rlun->gc_cur; - } + if (!list_empty(&rlun->wblk_list)) { +new_blk: + rblk = list_first_entry(&rlun->wblk_list, struct rrpc_block, + prio); + rrpc_set_lun_cur(rlun, rblk, cur_rblk); + list_del(&rblk->prio); + goto retry; + } + spin_unlock(&rlun->lock); + + rblk = rrpc_get_blk(rrpc, rlun, gc_force); + if (rblk) { + spin_lock(&rlun->lock); + list_add_tail(&rblk->prio, &rlun->wblk_list); + /* + * another thread might already have added a new block, + * Therefore, make sure that one is used, instead of the + * one just added. + */ + goto new_blk; } + if (unlikely(is_gc) && !gc_force) { + /* retry from emergency gc block */ + cur_rblk = &rlun->gc_cur; + rblk = rlun->gc_cur; + gc_force = 1; + spin_lock(&rlun->lock); + goto retry; + } + + pr_err("rrpc: failed to allocate new block\n"); + return NULL; +done: spin_unlock(&rlun->lock); return rrpc_update_map(rrpc, laddr, rblk, paddr); -err: - spin_unlock(&rlun->lock); - return NULL; } static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk) @@ -850,14 +853,14 @@ static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio, return NVM_IO_ERR; } - if (bio_rw(bio) == WRITE) + if (bio_op(bio) == REQ_OP_WRITE) return rrpc_write_ppalist_rq(rrpc, bio, rqd, flags, npages); return rrpc_read_ppalist_rq(rrpc, bio, rqd, flags, npages); } - if (bio_rw(bio) == WRITE) + if (bio_op(bio) == REQ_OP_WRITE) return rrpc_write_rq(rrpc, bio, rqd, flags); return rrpc_read_rq(rrpc, bio, rqd, flags); @@ -908,7 +911,7 @@ static blk_qc_t rrpc_make_rq(struct request_queue *q, struct bio *bio) struct nvm_rq *rqd; int err; - if (bio->bi_rw & REQ_DISCARD) { + if (bio_op(bio) == REQ_OP_DISCARD) { rrpc_discard(rrpc, bio); return BLK_QC_T_NONE; } @@ -1196,8 +1199,7 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) rlun->rrpc = rrpc; INIT_LIST_HEAD(&rlun->prio_list); - INIT_LIST_HEAD(&rlun->open_list); - INIT_LIST_HEAD(&rlun->closed_list); + INIT_LIST_HEAD(&rlun->wblk_list); INIT_WORK(&rlun->ws_gc, rrpc_lun_gc); spin_lock_init(&rlun->lock); @@ -1338,14 +1340,13 @@ static int rrpc_luns_configure(struct rrpc *rrpc) rblk = rrpc_get_blk(rrpc, rlun, 0); if (!rblk) goto err; - - rrpc_set_lun_cur(rlun, rblk); + rrpc_set_lun_cur(rlun, rblk, &rlun->cur); /* Emergency gc block */ rblk = rrpc_get_blk(rrpc, rlun, 1); if (!rblk) goto err; - rlun->gc_cur = rblk; + rrpc_set_lun_cur(rlun, rblk, &rlun->gc_cur); } return 0; diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h index 87e84b5fc1cc..5e87d52cb983 100644 --- a/drivers/lightnvm/rrpc.h +++ b/drivers/lightnvm/rrpc.h @@ -56,7 +56,6 @@ struct rrpc_block { struct nvm_block *parent; struct rrpc_lun *rlun; struct list_head prio; - struct list_head list; #define MAX_INVALID_PAGES_STORAGE 8 /* Bitmap for invalid page intries */ @@ -77,13 +76,7 @@ struct rrpc_lun { struct rrpc_block *blocks; /* Reference to block allocation */ struct list_head prio_list; /* Blocks that may be GC'ed */ - struct list_head open_list; /* In-use open blocks. These are blocks - * that can be both written to and read - * from - */ - struct list_head closed_list; /* In-use closed blocks. These are - * blocks that can _only_ be read from - */ + struct list_head wblk_list; /* Queued blocks to be written to */ struct work_struct ws_gc; @@ -188,7 +181,7 @@ static inline int request_intersects(struct rrpc_inflight_rq *r, } static int __rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr, - unsigned pages, struct rrpc_inflight_rq *r) + unsigned int pages, struct rrpc_inflight_rq *r) { sector_t laddr_end = laddr + pages - 1; struct rrpc_inflight_rq *rtmp; @@ -213,7 +206,7 @@ static int __rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr, } static inline int rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr, - unsigned pages, + unsigned int pages, struct rrpc_inflight_rq *r) { BUG_ON((laddr + pages) > rrpc->nr_sects); diff --git a/drivers/lightnvm/sysblk.c b/drivers/lightnvm/sysblk.c index 994697ac786e..a75bd28aaca3 100644 --- a/drivers/lightnvm/sysblk.c +++ b/drivers/lightnvm/sysblk.c @@ -39,7 +39,8 @@ static inline int scan_ppa_idx(int row, int blkid) return (row * MAX_BLKS_PR_SYSBLK) + blkid; } -void nvm_sysblk_to_cpu(struct nvm_sb_info *info, struct nvm_system_block *sb) +static void nvm_sysblk_to_cpu(struct nvm_sb_info *info, + struct nvm_system_block *sb) { info->seqnr = be32_to_cpu(sb->seqnr); info->erase_cnt = be32_to_cpu(sb->erase_cnt); @@ -48,7 +49,8 @@ void nvm_sysblk_to_cpu(struct nvm_sb_info *info, struct nvm_system_block *sb) info->fs_ppa.ppa = be64_to_cpu(sb->fs_ppa); } -void nvm_cpu_to_sysblk(struct nvm_system_block *sb, struct nvm_sb_info *info) +static void nvm_cpu_to_sysblk(struct nvm_system_block *sb, + struct nvm_sb_info *info) { sb->magic = cpu_to_be32(NVM_SYSBLK_MAGIC); sb->seqnr = cpu_to_be32(info->seqnr); @@ -86,7 +88,7 @@ static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas) return nr_rows; } -void nvm_setup_sysblk_scan(struct nvm_dev *dev, struct sysblk_scan *s, +static void nvm_setup_sysblk_scan(struct nvm_dev *dev, struct sysblk_scan *s, struct ppa_addr *sysblk_ppas) { memset(s, 0, sizeof(struct sysblk_scan)); diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 52ba8dd82821..3cbda1af87a0 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -3,7 +3,8 @@ # dm-mod-y += dm.o dm-table.o dm-target.o dm-linear.o dm-stripe.o \ - dm-ioctl.o dm-io.o dm-kcopyd.o dm-sysfs.o dm-stats.o + dm-ioctl.o dm-io.o dm-kcopyd.o dm-sysfs.o dm-stats.o \ + dm-rq.o dm-multipath-y += dm-path-selector.o dm-mpath.o dm-snapshot-y += dm-snap.o dm-exception-store.o dm-snap-transient.o \ dm-snap-persistent.o diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index eab505ee0027..76f7534d1dd1 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -294,10 +294,10 @@ static void bch_btree_node_read(struct btree *b) closure_init_stack(&cl); bio = bch_bbio_alloc(b->c); - bio->bi_rw = REQ_META|READ_SYNC; bio->bi_iter.bi_size = KEY_SIZE(&b->key) << 9; bio->bi_end_io = btree_node_read_endio; bio->bi_private = &cl; + bio_set_op_attrs(bio, REQ_OP_READ, REQ_META|READ_SYNC); bch_bio_map(bio, b->keys.set[0].data); @@ -396,8 +396,8 @@ static void do_btree_node_write(struct btree *b) b->bio->bi_end_io = btree_node_write_endio; b->bio->bi_private = cl; - b->bio->bi_rw = REQ_META|WRITE_SYNC|REQ_FUA; b->bio->bi_iter.bi_size = roundup(set_bytes(i), block_bytes(b->c)); + bio_set_op_attrs(b->bio, REQ_OP_WRITE, REQ_META|WRITE_SYNC|REQ_FUA); bch_bio_map(b->bio, i); /* diff --git a/drivers/md/bcache/closure.c b/drivers/md/bcache/closure.c index 9eaf1d6e8302..864e673aec39 100644 --- a/drivers/md/bcache/closure.c +++ b/drivers/md/bcache/closure.c @@ -112,7 +112,7 @@ bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl) EXPORT_SYMBOL(closure_wait); /** - * closure_sync - sleep until a closure a closure has nothing left to wait on + * closure_sync - sleep until a closure has nothing left to wait on * * Sleeps until the refcount hits 1 - the thread that's running the closure owns * the last refcount. diff --git a/drivers/md/bcache/closure.h b/drivers/md/bcache/closure.h index 782cc2c8a185..9b2fe2d3e3a9 100644 --- a/drivers/md/bcache/closure.h +++ b/drivers/md/bcache/closure.h @@ -31,7 +31,8 @@ * passing it, as you might expect, the function to run when nothing is pending * and the workqueue to run that function out of. * - * continue_at() also, critically, is a macro that returns the calling function. + * continue_at() also, critically, requires a 'return' immediately following the + * location where this macro is referenced, to return to the calling function. * There's good reason for this. * * To use safely closures asynchronously, they must always have a refcount while diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c index 8b1f1d5c1819..c28df164701e 100644 --- a/drivers/md/bcache/debug.c +++ b/drivers/md/bcache/debug.c @@ -52,9 +52,10 @@ void bch_btree_verify(struct btree *b) bio->bi_bdev = PTR_CACHE(b->c, &b->key, 0)->bdev; bio->bi_iter.bi_sector = PTR_OFFSET(&b->key, 0); bio->bi_iter.bi_size = KEY_SIZE(&v->key) << 9; + bio_set_op_attrs(bio, REQ_OP_READ, REQ_META|READ_SYNC); bch_bio_map(bio, sorted); - submit_bio_wait(REQ_META|READ_SYNC, bio); + submit_bio_wait(bio); bch_bbio_free(bio, b->c); memcpy(ondisk, sorted, KEY_SIZE(&v->key) << 9); @@ -113,11 +114,12 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio) check = bio_clone(bio, GFP_NOIO); if (!check) return; + bio_set_op_attrs(check, REQ_OP_READ, READ_SYNC); if (bio_alloc_pages(check, GFP_NOIO)) goto out_put; - submit_bio_wait(READ_SYNC, check); + submit_bio_wait(check); bio_for_each_segment(bv, bio, iter) { void *p1 = kmap_atomic(bv.bv_page); diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c index 86a0bb87124e..e97b0acf7b8d 100644 --- a/drivers/md/bcache/io.c +++ b/drivers/md/bcache/io.c @@ -25,7 +25,6 @@ struct bio *bch_bbio_alloc(struct cache_set *c) struct bio *bio = &b->bio; bio_init(bio); - bio->bi_flags |= BIO_POOL_NONE << BIO_POOL_OFFSET; bio->bi_max_vecs = bucket_pages(c); bio->bi_io_vec = bio->bi_inline_vecs; @@ -111,7 +110,7 @@ void bch_bbio_count_io_errors(struct cache_set *c, struct bio *bio, struct bbio *b = container_of(bio, struct bbio, bio); struct cache *ca = PTR_CACHE(c, &b->key, 0); - unsigned threshold = bio->bi_rw & REQ_WRITE + unsigned threshold = op_is_write(bio_op(bio)) ? c->congested_write_threshold_us : c->congested_read_threshold_us; diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c index 29eba7219b01..6925023e12d4 100644 --- a/drivers/md/bcache/journal.c +++ b/drivers/md/bcache/journal.c @@ -54,11 +54,11 @@ reread: left = ca->sb.bucket_size - offset; bio_reset(bio); bio->bi_iter.bi_sector = bucket + offset; bio->bi_bdev = ca->bdev; - bio->bi_rw = READ; bio->bi_iter.bi_size = len << 9; bio->bi_end_io = journal_read_endio; bio->bi_private = &cl; + bio_set_op_attrs(bio, REQ_OP_READ, 0); bch_bio_map(bio, data); closure_bio_submit(bio, &cl); @@ -418,7 +418,7 @@ static void journal_discard_work(struct work_struct *work) struct journal_device *ja = container_of(work, struct journal_device, discard_work); - submit_bio(0, &ja->discard_bio); + submit_bio(&ja->discard_bio); } static void do_journal_discard(struct cache *ca) @@ -449,10 +449,10 @@ static void do_journal_discard(struct cache *ca) atomic_set(&ja->discard_in_flight, DISCARD_IN_FLIGHT); bio_init(bio); + bio_set_op_attrs(bio, REQ_OP_DISCARD, 0); bio->bi_iter.bi_sector = bucket_to_sector(ca->set, ca->sb.d[ja->discard_idx]); bio->bi_bdev = ca->bdev; - bio->bi_rw = REQ_WRITE|REQ_DISCARD; bio->bi_max_vecs = 1; bio->bi_io_vec = bio->bi_inline_vecs; bio->bi_iter.bi_size = bucket_bytes(ca); @@ -626,11 +626,12 @@ static void journal_write_unlocked(struct closure *cl) bio_reset(bio); bio->bi_iter.bi_sector = PTR_OFFSET(k, i); bio->bi_bdev = ca->bdev; - bio->bi_rw = REQ_WRITE|REQ_SYNC|REQ_META|REQ_FLUSH|REQ_FUA; bio->bi_iter.bi_size = sectors << 9; bio->bi_end_io = journal_write_endio; bio->bi_private = w; + bio_set_op_attrs(bio, REQ_OP_WRITE, + REQ_SYNC|REQ_META|REQ_PREFLUSH|REQ_FUA); bch_bio_map(bio, w->data); trace_bcache_journal_write(bio); diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c index b929fc944e9c..1881319f2298 100644 --- a/drivers/md/bcache/movinggc.c +++ b/drivers/md/bcache/movinggc.c @@ -163,7 +163,7 @@ static void read_moving(struct cache_set *c) moving_init(io); bio = &io->bio.bio; - bio->bi_rw = READ; + bio_set_op_attrs(bio, REQ_OP_READ, 0); bio->bi_end_io = read_moving_endio; if (bio_alloc_pages(bio, GFP_KERNEL)) diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 25fa8445bb24..69f16f43f8ab 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -205,10 +205,10 @@ static void bch_data_insert_start(struct closure *cl) return bch_data_invalidate(cl); /* - * Journal writes are marked REQ_FLUSH; if the original write was a + * Journal writes are marked REQ_PREFLUSH; if the original write was a * flush, it'll wait on the journal write. */ - bio->bi_rw &= ~(REQ_FLUSH|REQ_FUA); + bio->bi_rw &= ~(REQ_PREFLUSH|REQ_FUA); do { unsigned i; @@ -253,7 +253,7 @@ static void bch_data_insert_start(struct closure *cl) trace_bcache_cache_insert(k); bch_keylist_push(&op->insert_keys); - n->bi_rw |= REQ_WRITE; + bio_set_op_attrs(n, REQ_OP_WRITE, 0); bch_submit_bbio(n, op->c, k, 0); } while (n != bio); @@ -378,12 +378,12 @@ static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) || c->gc_stats.in_use > CUTOFF_CACHE_ADD || - (bio->bi_rw & REQ_DISCARD)) + (bio_op(bio) == REQ_OP_DISCARD)) goto skip; if (mode == CACHE_MODE_NONE || (mode == CACHE_MODE_WRITEAROUND && - (bio->bi_rw & REQ_WRITE))) + op_is_write(bio_op(bio)))) goto skip; if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) || @@ -404,7 +404,7 @@ static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) if (!congested && mode == CACHE_MODE_WRITEBACK && - (bio->bi_rw & REQ_WRITE) && + op_is_write(bio_op(bio)) && (bio->bi_rw & REQ_SYNC)) goto rescale; @@ -657,7 +657,7 @@ static inline struct search *search_alloc(struct bio *bio, s->cache_miss = NULL; s->d = d; s->recoverable = 1; - s->write = (bio->bi_rw & REQ_WRITE) != 0; + s->write = op_is_write(bio_op(bio)); s->read_dirty_data = 0; s->start_time = jiffies; @@ -668,7 +668,7 @@ static inline struct search *search_alloc(struct bio *bio, s->iop.write_prio = 0; s->iop.error = 0; s->iop.flags = 0; - s->iop.flush_journal = (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0; + s->iop.flush_journal = (bio->bi_rw & (REQ_PREFLUSH|REQ_FUA)) != 0; s->iop.wq = bcache_wq; return s; @@ -899,7 +899,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s) * But check_overlapping drops dirty keys for which io hasn't started, * so we still want to call it. */ - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) s->iop.bypass = true; if (should_writeback(dc, s->orig_bio, @@ -913,22 +913,22 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s) s->iop.bio = s->orig_bio; bio_get(s->iop.bio); - if (!(bio->bi_rw & REQ_DISCARD) || + if ((bio_op(bio) != REQ_OP_DISCARD) || blk_queue_discard(bdev_get_queue(dc->bdev))) closure_bio_submit(bio, cl); } else if (s->iop.writeback) { bch_writeback_add(dc); s->iop.bio = bio; - if (bio->bi_rw & REQ_FLUSH) { + if (bio->bi_rw & REQ_PREFLUSH) { /* Also need to send a flush to the backing device */ struct bio *flush = bio_alloc_bioset(GFP_NOIO, 0, dc->disk.bio_split); - flush->bi_rw = WRITE_FLUSH; flush->bi_bdev = bio->bi_bdev; flush->bi_end_io = request_endio; flush->bi_private = cl; + bio_set_op_attrs(flush, REQ_OP_WRITE, WRITE_FLUSH); closure_bio_submit(flush, cl); } @@ -992,7 +992,7 @@ static blk_qc_t cached_dev_make_request(struct request_queue *q, cached_dev_read(dc, s); } } else { - if ((bio->bi_rw & REQ_DISCARD) && + if ((bio_op(bio) == REQ_OP_DISCARD) && !blk_queue_discard(bdev_get_queue(dc->bdev))) bio_endio(bio); else @@ -1103,7 +1103,7 @@ static blk_qc_t flash_dev_make_request(struct request_queue *q, &KEY(d->id, bio->bi_iter.bi_sector, 0), &KEY(d->id, bio_end_sector(bio), 0)); - s->iop.bypass = (bio->bi_rw & REQ_DISCARD) != 0; + s->iop.bypass = (bio_op(bio) == REQ_OP_DISCARD) != 0; s->iop.writeback = true; s->iop.bio = bio; diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index f5dbb4e884d8..88ef6d14cce3 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -134,7 +134,6 @@ static const char *read_super(struct cache_sb *sb, struct block_device *bdev, case BCACHE_SB_VERSION_CDEV: case BCACHE_SB_VERSION_CDEV_WITH_UUID: sb->nbuckets = le64_to_cpu(s->nbuckets); - sb->block_size = le16_to_cpu(s->block_size); sb->bucket_size = le16_to_cpu(s->bucket_size); sb->nr_in_set = le16_to_cpu(s->nr_in_set); @@ -212,8 +211,8 @@ static void __write_super(struct cache_sb *sb, struct bio *bio) unsigned i; bio->bi_iter.bi_sector = SB_SECTOR; - bio->bi_rw = REQ_SYNC|REQ_META; bio->bi_iter.bi_size = SB_SIZE; + bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC|REQ_META); bch_bio_map(bio, NULL); out->offset = cpu_to_le64(sb->offset); @@ -238,7 +237,7 @@ static void __write_super(struct cache_sb *sb, struct bio *bio) pr_debug("ver %llu, flags %llu, seq %llu", sb->version, sb->flags, sb->seq); - submit_bio(REQ_WRITE, bio); + submit_bio(bio); } static void bch_write_bdev_super_unlock(struct closure *cl) @@ -333,7 +332,7 @@ static void uuid_io_unlock(struct closure *cl) up(&c->uuid_write_mutex); } -static void uuid_io(struct cache_set *c, unsigned long rw, +static void uuid_io(struct cache_set *c, int op, unsigned long op_flags, struct bkey *k, struct closure *parent) { struct closure *cl = &c->uuid_write; @@ -348,21 +347,22 @@ static void uuid_io(struct cache_set *c, unsigned long rw, for (i = 0; i < KEY_PTRS(k); i++) { struct bio *bio = bch_bbio_alloc(c); - bio->bi_rw = REQ_SYNC|REQ_META|rw; + bio->bi_rw = REQ_SYNC|REQ_META|op_flags; bio->bi_iter.bi_size = KEY_SIZE(k) << 9; bio->bi_end_io = uuid_endio; bio->bi_private = cl; + bio_set_op_attrs(bio, op, REQ_SYNC|REQ_META|op_flags); bch_bio_map(bio, c->uuids); bch_submit_bbio(bio, c, k, i); - if (!(rw & WRITE)) + if (op != REQ_OP_WRITE) break; } bch_extent_to_text(buf, sizeof(buf), k); - pr_debug("%s UUIDs at %s", rw & REQ_WRITE ? "wrote" : "read", buf); + pr_debug("%s UUIDs at %s", op == REQ_OP_WRITE ? "wrote" : "read", buf); for (u = c->uuids; u < c->uuids + c->nr_uuids; u++) if (!bch_is_zero(u->uuid, 16)) @@ -381,7 +381,7 @@ static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl) return "bad uuid pointer"; bkey_copy(&c->uuid_bucket, k); - uuid_io(c, READ_SYNC, k, cl); + uuid_io(c, REQ_OP_READ, READ_SYNC, k, cl); if (j->version < BCACHE_JSET_VERSION_UUIDv1) { struct uuid_entry_v0 *u0 = (void *) c->uuids; @@ -426,7 +426,7 @@ static int __uuid_write(struct cache_set *c) return 1; SET_KEY_SIZE(&k.key, c->sb.bucket_size); - uuid_io(c, REQ_WRITE, &k.key, &cl); + uuid_io(c, REQ_OP_WRITE, 0, &k.key, &cl); closure_sync(&cl); bkey_copy(&c->uuid_bucket, &k.key); @@ -498,7 +498,8 @@ static void prio_endio(struct bio *bio) closure_put(&ca->prio); } -static void prio_io(struct cache *ca, uint64_t bucket, unsigned long rw) +static void prio_io(struct cache *ca, uint64_t bucket, int op, + unsigned long op_flags) { struct closure *cl = &ca->prio; struct bio *bio = bch_bbio_alloc(ca->set); @@ -507,11 +508,11 @@ static void prio_io(struct cache *ca, uint64_t bucket, unsigned long rw) bio->bi_iter.bi_sector = bucket * ca->sb.bucket_size; bio->bi_bdev = ca->bdev; - bio->bi_rw = REQ_SYNC|REQ_META|rw; bio->bi_iter.bi_size = bucket_bytes(ca); bio->bi_end_io = prio_endio; bio->bi_private = ca; + bio_set_op_attrs(bio, op, REQ_SYNC|REQ_META|op_flags); bch_bio_map(bio, ca->disk_buckets); closure_bio_submit(bio, &ca->prio); @@ -557,7 +558,7 @@ void bch_prio_write(struct cache *ca) BUG_ON(bucket == -1); mutex_unlock(&ca->set->bucket_lock); - prio_io(ca, bucket, REQ_WRITE); + prio_io(ca, bucket, REQ_OP_WRITE, 0); mutex_lock(&ca->set->bucket_lock); ca->prio_buckets[i] = bucket; @@ -599,7 +600,7 @@ static void prio_read(struct cache *ca, uint64_t bucket) ca->prio_last_buckets[bucket_nr] = bucket; bucket_nr++; - prio_io(ca, bucket, READ_SYNC); + prio_io(ca, bucket, REQ_OP_READ, READ_SYNC); if (p->csum != bch_crc64(&p->magic, bucket_bytes(ca) - 8)) pr_warn("bad csum reading priorities"); @@ -1518,7 +1519,8 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb) !(c->fill_iter = mempool_create_kmalloc_pool(1, iter_size)) || !(c->bio_split = bioset_create(4, offsetof(struct bbio, bio))) || !(c->uuids = alloc_bucket_pages(GFP_KERNEL, c)) || - !(c->moving_gc_wq = create_workqueue("bcache_gc")) || + !(c->moving_gc_wq = alloc_workqueue("bcache_gc", + WQ_MEM_RECLAIM, 0)) || bch_journal_alloc(c) || bch_btree_cache_alloc(c) || bch_open_buckets_alloc(c) || @@ -1803,7 +1805,7 @@ void bch_cache_release(struct kobject *kobj) module_put(THIS_MODULE); } -static int cache_alloc(struct cache_sb *sb, struct cache *ca) +static int cache_alloc(struct cache *ca) { size_t free; struct bucket *b; @@ -1858,7 +1860,7 @@ static int register_cache(struct cache_sb *sb, struct page *sb_page, if (blk_queue_discard(bdev_get_queue(ca->bdev))) ca->discard = CACHE_DISCARD(&ca->sb); - ret = cache_alloc(sb, ca); + ret = cache_alloc(ca); if (ret != 0) goto err; @@ -2097,7 +2099,7 @@ static int __init bcache_init(void) return bcache_major; } - if (!(bcache_wq = create_workqueue("bcache")) || + if (!(bcache_wq = alloc_workqueue("bcache", WQ_MEM_RECLAIM, 0)) || !(bcache_kobj = kobject_create_and_add("bcache", fs_kobj)) || sysfs_create_files(bcache_kobj, files) || bch_request_init() || diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index 60123677b382..d9fd2a62e5f6 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c @@ -182,7 +182,7 @@ static void write_dirty(struct closure *cl) struct keybuf_key *w = io->bio.bi_private; dirty_init(w); - io->bio.bi_rw = WRITE; + bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0); io->bio.bi_iter.bi_sector = KEY_START(&w->key); io->bio.bi_bdev = io->dc->bdev; io->bio.bi_end_io = dirty_endio; @@ -251,10 +251,10 @@ static void read_dirty(struct cached_dev *dc) io->dc = dc; dirty_init(w); + bio_set_op_attrs(&io->bio, REQ_OP_READ, 0); io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0); io->bio.bi_bdev = PTR_CACHE(dc->disk.c, &w->key, 0)->bdev; - io->bio.bi_rw = READ; io->bio.bi_end_io = read_dirty_endio; if (bio_alloc_pages(&io->bio, GFP_KERNEL)) diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c index d8129ec93ebd..6fff794e0c72 100644 --- a/drivers/md/bitmap.c +++ b/drivers/md/bitmap.c @@ -162,7 +162,7 @@ static int read_sb_page(struct mddev *mddev, loff_t offset, if (sync_page_io(rdev, target, roundup(size, bdev_logical_block_size(rdev->bdev)), - page, READ, true)) { + page, REQ_OP_READ, 0, true)) { page->index = index; return 0; } @@ -297,7 +297,7 @@ static void write_page(struct bitmap *bitmap, struct page *page, int wait) atomic_inc(&bitmap->pending_writes); set_buffer_locked(bh); set_buffer_mapped(bh); - submit_bh(WRITE | REQ_SYNC, bh); + submit_bh(REQ_OP_WRITE, REQ_SYNC, bh); bh = bh->b_this_page; } @@ -392,7 +392,7 @@ static int read_page(struct file *file, unsigned long index, atomic_inc(&bitmap->pending_writes); set_buffer_locked(bh); set_buffer_mapped(bh); - submit_bh(READ, bh); + submit_bh(REQ_OP_READ, 0, bh); } block++; bh = bh->b_this_page; diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index cd77216beff1..6571c81465e1 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -574,7 +574,8 @@ static void use_dmio(struct dm_buffer *b, int rw, sector_t block, { int r; struct dm_io_request io_req = { - .bi_rw = rw, + .bi_op = rw, + .bi_op_flags = 0, .notify.fn = dmio_complete, .notify.context = b, .client = b->c->dm_io, @@ -634,6 +635,7 @@ static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block, * the dm_buffer's inline bio is local to bufio. */ b->bio.bi_private = end_io; + bio_set_op_attrs(&b->bio, rw, 0); /* * We assume that if len >= PAGE_SIZE ptr is page-aligned. @@ -660,7 +662,7 @@ static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block, ptr += PAGE_SIZE; } while (len > 0); - submit_bio(rw, &b->bio); + submit_bio(&b->bio); } static void submit_io(struct dm_buffer *b, int rw, sector_t block, @@ -1326,7 +1328,8 @@ EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers); int dm_bufio_issue_flush(struct dm_bufio_client *c) { struct dm_io_request io_req = { - .bi_rw = WRITE_FLUSH, + .bi_op = REQ_OP_WRITE, + .bi_op_flags = WRITE_FLUSH, .mem.type = DM_IO_KMEM, .mem.ptr.addr = NULL, .client = c->dm_io, diff --git a/drivers/md/dm-builtin.c b/drivers/md/dm-builtin.c index 6c9049c51b2b..f092771878c2 100644 --- a/drivers/md/dm-builtin.c +++ b/drivers/md/dm-builtin.c @@ -1,4 +1,4 @@ -#include "dm.h" +#include "dm-core.h" /* * The kobject release method must not be placed in the module itself, diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index ee0510f9a85e..718744db62df 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -788,7 +788,8 @@ static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio) spin_lock_irqsave(&cache->lock, flags); if (cache->need_tick_bio && - !(bio->bi_rw & (REQ_FUA | REQ_FLUSH | REQ_DISCARD))) { + !(bio->bi_rw & (REQ_FUA | REQ_PREFLUSH)) && + bio_op(bio) != REQ_OP_DISCARD) { pb->tick = true; cache->need_tick_bio = false; } @@ -829,7 +830,7 @@ static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio) static int bio_triggers_commit(struct cache *cache, struct bio *bio) { - return bio->bi_rw & (REQ_FLUSH | REQ_FUA); + return bio->bi_rw & (REQ_PREFLUSH | REQ_FUA); } /* @@ -851,7 +852,7 @@ static void inc_ds(struct cache *cache, struct bio *bio, static bool accountable_bio(struct cache *cache, struct bio *bio) { return ((bio->bi_bdev == cache->origin_dev->bdev) && - !(bio->bi_rw & REQ_DISCARD)); + bio_op(bio) != REQ_OP_DISCARD); } static void accounted_begin(struct cache *cache, struct bio *bio) @@ -1067,7 +1068,8 @@ static void dec_io_migrations(struct cache *cache) static bool discard_or_flush(struct bio *bio) { - return bio->bi_rw & (REQ_FLUSH | REQ_FUA | REQ_DISCARD); + return bio_op(bio) == REQ_OP_DISCARD || + bio->bi_rw & (REQ_PREFLUSH | REQ_FUA); } static void __cell_defer(struct cache *cache, struct dm_bio_prison_cell *cell) @@ -1612,8 +1614,8 @@ static void process_flush_bio(struct cache *cache, struct bio *bio) remap_to_cache(cache, bio, 0); /* - * REQ_FLUSH is not directed at any particular block so we don't - * need to inc_ds(). REQ_FUA's are split into a write + REQ_FLUSH + * REQ_PREFLUSH is not directed at any particular block so we don't + * need to inc_ds(). REQ_FUA's are split into a write + REQ_PREFLUSH * by dm-core. */ issue(cache, bio); @@ -1978,9 +1980,9 @@ static void process_deferred_bios(struct cache *cache) bio = bio_list_pop(&bios); - if (bio->bi_rw & REQ_FLUSH) + if (bio->bi_rw & REQ_PREFLUSH) process_flush_bio(cache, bio); - else if (bio->bi_rw & REQ_DISCARD) + else if (bio_op(bio) == REQ_OP_DISCARD) process_discard_bio(cache, &structs, bio); else process_bio(cache, &structs, bio); diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h new file mode 100644 index 000000000000..40ceba1fe8be --- /dev/null +++ b/drivers/md/dm-core.h @@ -0,0 +1,149 @@ +/* + * Internal header file _only_ for device mapper core + * + * Copyright (C) 2016 Red Hat, Inc. All rights reserved. + * + * This file is released under the LGPL. + */ + +#ifndef DM_CORE_INTERNAL_H +#define DM_CORE_INTERNAL_H + +#include +#include +#include + +#include + +#include "dm.h" + +#define DM_RESERVED_MAX_IOS 1024 + +struct dm_kobject_holder { + struct kobject kobj; + struct completion completion; +}; + +/* + * DM core internal structure that used directly by dm.c and dm-rq.c + * DM targets must _not_ deference a mapped_device to directly access its members! + */ +struct mapped_device { + struct srcu_struct io_barrier; + struct mutex suspend_lock; + + /* + * The current mapping (struct dm_table *). + * Use dm_get_live_table{_fast} or take suspend_lock for + * dereference. + */ + void __rcu *map; + + struct list_head table_devices; + struct mutex table_devices_lock; + + unsigned long flags; + + struct request_queue *queue; + int numa_node_id; + + unsigned type; + /* Protect queue and type against concurrent access. */ + struct mutex type_lock; + + atomic_t holders; + atomic_t open_count; + + struct dm_target *immutable_target; + struct target_type *immutable_target_type; + + struct gendisk *disk; + char name[16]; + + void *interface_ptr; + + /* + * A list of ios that arrived while we were suspended. + */ + atomic_t pending[2]; + wait_queue_head_t wait; + struct work_struct work; + spinlock_t deferred_lock; + struct bio_list deferred; + + /* + * Event handling. + */ + wait_queue_head_t eventq; + atomic_t event_nr; + atomic_t uevent_seq; + struct list_head uevent_list; + spinlock_t uevent_lock; /* Protect access to uevent_list */ + + /* the number of internal suspends */ + unsigned internal_suspend_count; + + /* + * Processing queue (flush) + */ + struct workqueue_struct *wq; + + /* + * io objects are allocated from here. + */ + mempool_t *io_pool; + mempool_t *rq_pool; + + struct bio_set *bs; + + /* + * freeze/thaw support require holding onto a super block + */ + struct super_block *frozen_sb; + + /* forced geometry settings */ + struct hd_geometry geometry; + + struct block_device *bdev; + + /* kobject and completion */ + struct dm_kobject_holder kobj_holder; + + /* zero-length flush that will be cloned and submitted to targets */ + struct bio flush_bio; + + struct dm_stats stats; + + struct kthread_worker kworker; + struct task_struct *kworker_task; + + /* for request-based merge heuristic in dm_request_fn() */ + unsigned seq_rq_merge_deadline_usecs; + int last_rq_rw; + sector_t last_rq_pos; + ktime_t last_rq_start_time; + + /* for blk-mq request-based DM support */ + struct blk_mq_tag_set *tag_set; + bool use_blk_mq:1; + bool init_tio_pdu:1; +}; + +void dm_init_md_queue(struct mapped_device *md); +void dm_init_normal_md_queue(struct mapped_device *md); +int md_in_flight(struct mapped_device *md); +void disable_write_same(struct mapped_device *md); + +static inline struct completion *dm_get_completion_from_kobject(struct kobject *kobj) +{ + return &container_of(kobj, struct dm_kobject_holder, kobj)->completion; +} + +unsigned __dm_get_module_param(unsigned *module_param, unsigned def, unsigned max); + +static inline bool dm_message_test_buffer_overflow(char *result, unsigned maxlen) +{ + return !maxlen || strlen(result) + 1 >= maxlen; +} + +#endif diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 4f3cb3554944..8f2e3e2ffd26 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -683,7 +683,7 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc, u8 *data) { struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; - u64 sector = cpu_to_le64((u64)dmreq->iv_sector); + __le64 sector = cpu_to_le64(dmreq->iv_sector); u8 buf[TCW_WHITENING_SIZE]; SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm); int i, r; @@ -722,7 +722,7 @@ static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv, struct dm_crypt_request *dmreq) { struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; - u64 sector = cpu_to_le64((u64)dmreq->iv_sector); + __le64 sector = cpu_to_le64(dmreq->iv_sector); u8 *src; int r = 0; @@ -1136,7 +1136,7 @@ static void clone_init(struct dm_crypt_io *io, struct bio *clone) clone->bi_private = io; clone->bi_end_io = crypt_endio; clone->bi_bdev = cc->dev->bdev; - clone->bi_rw = io->base_bio->bi_rw; + bio_set_op_attrs(clone, bio_op(io->base_bio), io->base_bio->bi_rw); } static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) @@ -1911,11 +1911,12 @@ static int crypt_map(struct dm_target *ti, struct bio *bio) struct crypt_config *cc = ti->private; /* - * If bio is REQ_FLUSH or REQ_DISCARD, just bypass crypt queues. - * - for REQ_FLUSH device-mapper core ensures that no IO is in-flight - * - for REQ_DISCARD caller must use flush if IO ordering matters + * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues. + * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight + * - for REQ_OP_DISCARD caller must use flush if IO ordering matters */ - if (unlikely(bio->bi_rw & (REQ_FLUSH | REQ_DISCARD))) { + if (unlikely(bio->bi_rw & REQ_PREFLUSH || + bio_op(bio) == REQ_OP_DISCARD)) { bio->bi_bdev = cc->dev->bdev; if (bio_sectors(bio)) bio->bi_iter.bi_sector = cc->start + diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c index 665bf3285618..2faf49d8f4d7 100644 --- a/drivers/md/dm-era-target.c +++ b/drivers/md/dm-era-target.c @@ -1540,9 +1540,9 @@ static int era_map(struct dm_target *ti, struct bio *bio) remap_to_origin(era, bio); /* - * REQ_FLUSH bios carry no data, so we're not interested in them. + * REQ_PREFLUSH bios carry no data, so we're not interested in them. */ - if (!(bio->bi_rw & REQ_FLUSH) && + if (!(bio->bi_rw & REQ_PREFLUSH) && (bio_data_dir(bio) == WRITE) && !metadata_current_marked(era->md, block)) { defer_bio(era, bio); diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c index b7341de87015..29b99fb6a16a 100644 --- a/drivers/md/dm-flakey.c +++ b/drivers/md/dm-flakey.c @@ -266,7 +266,7 @@ static void corrupt_bio_data(struct bio *bio, struct flakey_c *fc) data[fc->corrupt_bio_byte - 1] = fc->corrupt_bio_value; DMDEBUG("Corrupting data bio=%p by writing %u to byte %u " - "(rw=%c bi_rw=%lu bi_sector=%llu cur_bytes=%u)\n", + "(rw=%c bi_rw=%u bi_sector=%llu cur_bytes=%u)\n", bio, fc->corrupt_bio_value, fc->corrupt_bio_byte, (bio_data_dir(bio) == WRITE) ? 'w' : 'r', bio->bi_rw, (unsigned long long)bio->bi_iter.bi_sector, bio_bytes); diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index 06d426eb5a30..daa03e41654a 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -5,7 +5,7 @@ * This file is released under the GPL. */ -#include "dm.h" +#include "dm-core.h" #include @@ -278,8 +278,9 @@ static void km_dp_init(struct dpages *dp, void *data) /*----------------------------------------------------------------- * IO routines that accept a list of pages. *---------------------------------------------------------------*/ -static void do_region(int rw, unsigned region, struct dm_io_region *where, - struct dpages *dp, struct io *io) +static void do_region(int op, int op_flags, unsigned region, + struct dm_io_region *where, struct dpages *dp, + struct io *io) { struct bio *bio; struct page *page; @@ -295,24 +296,25 @@ static void do_region(int rw, unsigned region, struct dm_io_region *where, /* * Reject unsupported discard and write same requests. */ - if (rw & REQ_DISCARD) + if (op == REQ_OP_DISCARD) special_cmd_max_sectors = q->limits.max_discard_sectors; - else if (rw & REQ_WRITE_SAME) + else if (op == REQ_OP_WRITE_SAME) special_cmd_max_sectors = q->limits.max_write_same_sectors; - if ((rw & (REQ_DISCARD | REQ_WRITE_SAME)) && special_cmd_max_sectors == 0) { + if ((op == REQ_OP_DISCARD || op == REQ_OP_WRITE_SAME) && + special_cmd_max_sectors == 0) { dec_count(io, region, -EOPNOTSUPP); return; } /* - * where->count may be zero if rw holds a flush and we need to + * where->count may be zero if op holds a flush and we need to * send a zero-sized flush. */ do { /* * Allocate a suitably sized-bio. */ - if ((rw & REQ_DISCARD) || (rw & REQ_WRITE_SAME)) + if ((op == REQ_OP_DISCARD) || (op == REQ_OP_WRITE_SAME)) num_bvecs = 1; else num_bvecs = min_t(int, BIO_MAX_PAGES, @@ -322,13 +324,14 @@ static void do_region(int rw, unsigned region, struct dm_io_region *where, bio->bi_iter.bi_sector = where->sector + (where->count - remaining); bio->bi_bdev = where->bdev; bio->bi_end_io = endio; + bio_set_op_attrs(bio, op, op_flags); store_io_and_region_in_bio(bio, io, region); - if (rw & REQ_DISCARD) { + if (op == REQ_OP_DISCARD) { num_sectors = min_t(sector_t, special_cmd_max_sectors, remaining); bio->bi_iter.bi_size = num_sectors << SECTOR_SHIFT; remaining -= num_sectors; - } else if (rw & REQ_WRITE_SAME) { + } else if (op == REQ_OP_WRITE_SAME) { /* * WRITE SAME only uses a single page. */ @@ -355,11 +358,11 @@ static void do_region(int rw, unsigned region, struct dm_io_region *where, } atomic_inc(&io->count); - submit_bio(rw, bio); + submit_bio(bio); } while (remaining); } -static void dispatch_io(int rw, unsigned int num_regions, +static void dispatch_io(int op, int op_flags, unsigned int num_regions, struct dm_io_region *where, struct dpages *dp, struct io *io, int sync) { @@ -369,7 +372,7 @@ static void dispatch_io(int rw, unsigned int num_regions, BUG_ON(num_regions > DM_IO_MAX_REGIONS); if (sync) - rw |= REQ_SYNC; + op_flags |= REQ_SYNC; /* * For multiple regions we need to be careful to rewind @@ -377,8 +380,8 @@ static void dispatch_io(int rw, unsigned int num_regions, */ for (i = 0; i < num_regions; i++) { *dp = old_pages; - if (where[i].count || (rw & REQ_FLUSH)) - do_region(rw, i, where + i, dp, io); + if (where[i].count || (op_flags & REQ_PREFLUSH)) + do_region(op, op_flags, i, where + i, dp, io); } /* @@ -402,13 +405,13 @@ static void sync_io_complete(unsigned long error, void *context) } static int sync_io(struct dm_io_client *client, unsigned int num_regions, - struct dm_io_region *where, int rw, struct dpages *dp, - unsigned long *error_bits) + struct dm_io_region *where, int op, int op_flags, + struct dpages *dp, unsigned long *error_bits) { struct io *io; struct sync_io sio; - if (num_regions > 1 && (rw & RW_MASK) != WRITE) { + if (num_regions > 1 && !op_is_write(op)) { WARN_ON(1); return -EIO; } @@ -425,7 +428,7 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; - dispatch_io(rw, num_regions, where, dp, io, 1); + dispatch_io(op, op_flags, num_regions, where, dp, io, 1); wait_for_completion_io(&sio.wait); @@ -436,12 +439,12 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, } static int async_io(struct dm_io_client *client, unsigned int num_regions, - struct dm_io_region *where, int rw, struct dpages *dp, - io_notify_fn fn, void *context) + struct dm_io_region *where, int op, int op_flags, + struct dpages *dp, io_notify_fn fn, void *context) { struct io *io; - if (num_regions > 1 && (rw & RW_MASK) != WRITE) { + if (num_regions > 1 && !op_is_write(op)) { WARN_ON(1); fn(1, context); return -EIO; @@ -457,7 +460,7 @@ static int async_io(struct dm_io_client *client, unsigned int num_regions, io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; - dispatch_io(rw, num_regions, where, dp, io, 0); + dispatch_io(op, op_flags, num_regions, where, dp, io, 0); return 0; } @@ -480,7 +483,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, case DM_IO_VMA: flush_kernel_vmap_range(io_req->mem.ptr.vma, size); - if ((io_req->bi_rw & RW_MASK) == READ) { + if (io_req->bi_op == REQ_OP_READ) { dp->vma_invalidate_address = io_req->mem.ptr.vma; dp->vma_invalidate_size = size; } @@ -518,10 +521,12 @@ int dm_io(struct dm_io_request *io_req, unsigned num_regions, if (!io_req->notify.fn) return sync_io(io_req->client, num_regions, where, - io_req->bi_rw, &dp, sync_error_bits); + io_req->bi_op, io_req->bi_op_flags, &dp, + sync_error_bits); - return async_io(io_req->client, num_regions, where, io_req->bi_rw, - &dp, io_req->notify.fn, io_req->notify.context); + return async_io(io_req->client, num_regions, where, io_req->bi_op, + io_req->bi_op_flags, &dp, io_req->notify.fn, + io_req->notify.context); } EXPORT_SYMBOL(dm_io); diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 2c7ca258c4e4..966eb4b61aed 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -5,7 +5,7 @@ * This file is released under the GPL. */ -#include "dm.h" +#include "dm-core.h" #include #include @@ -1267,6 +1267,15 @@ static int populate_table(struct dm_table *table, return dm_table_complete(table); } +static bool is_valid_type(unsigned cur, unsigned new) +{ + if (cur == new || + (cur == DM_TYPE_BIO_BASED && new == DM_TYPE_DAX_BIO_BASED)) + return true; + + return false; +} + static int table_load(struct dm_ioctl *param, size_t param_size) { int r; @@ -1309,7 +1318,7 @@ static int table_load(struct dm_ioctl *param, size_t param_size) DMWARN("unable to set up device queue for new table."); goto err_unlock_md_type; } - } else if (dm_get_md_type(md) != dm_table_get_type(t)) { + } else if (!is_valid_type(dm_get_md_type(md), dm_table_get_type(t))) { DMWARN("can't change device type after initial table load."); r = -EINVAL; goto err_unlock_md_type; @@ -1670,8 +1679,7 @@ static int check_version(unsigned int cmd, struct dm_ioctl __user *user) return r; } -#define DM_PARAMS_KMALLOC 0x0001 /* Params alloced with kmalloc */ -#define DM_PARAMS_VMALLOC 0x0002 /* Params alloced with vmalloc */ +#define DM_PARAMS_MALLOC 0x0001 /* Params allocated with kvmalloc() */ #define DM_WIPE_BUFFER 0x0010 /* Wipe input buffer before returning from ioctl */ static void free_params(struct dm_ioctl *param, size_t param_size, int param_flags) @@ -1679,10 +1687,8 @@ static void free_params(struct dm_ioctl *param, size_t param_size, int param_fla if (param_flags & DM_WIPE_BUFFER) memset(param, 0, param_size); - if (param_flags & DM_PARAMS_KMALLOC) - kfree(param); - if (param_flags & DM_PARAMS_VMALLOC) - vfree(param); + if (param_flags & DM_PARAMS_MALLOC) + kvfree(param); } static int copy_params(struct dm_ioctl __user *user, struct dm_ioctl *param_kernel, @@ -1714,19 +1720,14 @@ static int copy_params(struct dm_ioctl __user *user, struct dm_ioctl *param_kern * Use kmalloc() rather than vmalloc() when we can. */ dmi = NULL; - if (param_kernel->data_size <= KMALLOC_MAX_SIZE) { + if (param_kernel->data_size <= KMALLOC_MAX_SIZE) dmi = kmalloc(param_kernel->data_size, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); - if (dmi) - *param_flags |= DM_PARAMS_KMALLOC; - } if (!dmi) { unsigned noio_flag; noio_flag = memalloc_noio_save(); dmi = __vmalloc(param_kernel->data_size, GFP_NOIO | __GFP_HIGH | __GFP_HIGHMEM, PAGE_KERNEL); memalloc_noio_restore(noio_flag); - if (dmi) - *param_flags |= DM_PARAMS_VMALLOC; } if (!dmi) { @@ -1735,6 +1736,8 @@ static int copy_params(struct dm_ioctl __user *user, struct dm_ioctl *param_kern return -ENOMEM; } + *param_flags |= DM_PARAMS_MALLOC; + if (copy_from_user(dmi, user, param_kernel->data_size)) goto bad; diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 1452ed9aacb4..9e9d04cb7d51 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -26,7 +26,7 @@ #include #include -#include "dm.h" +#include "dm-core.h" #define SUB_JOB_SIZE 128 #define SPLIT_COUNT 8 @@ -465,7 +465,7 @@ static void complete_io(unsigned long error, void *context) io_job_finish(kc->throttle); if (error) { - if (job->rw & WRITE) + if (op_is_write(job->rw)) job->write_err |= error; else job->read_err = 1; @@ -477,7 +477,7 @@ static void complete_io(unsigned long error, void *context) } } - if (job->rw & WRITE) + if (op_is_write(job->rw)) push(&kc->complete_jobs, job); else { @@ -496,7 +496,8 @@ static int run_io_job(struct kcopyd_job *job) { int r; struct dm_io_request io_req = { - .bi_rw = job->rw, + .bi_op = job->rw, + .bi_op_flags = 0, .mem.type = DM_IO_PAGE_LIST, .mem.ptr.pl = job->pages, .mem.offset = 0, @@ -550,7 +551,7 @@ static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc, if (r < 0) { /* error this rogue job */ - if (job->rw & WRITE) + if (op_is_write(job->rw)) job->write_err = (unsigned long) -1L; else job->read_err = 1; @@ -734,7 +735,7 @@ int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, /* * Use WRITE SAME to optimize zeroing if all dests support it. */ - job->rw = WRITE | REQ_WRITE_SAME; + job->rw = REQ_OP_WRITE_SAME; for (i = 0; i < job->num_dests; i++) if (!bdev_write_same(job->dests[i].bdev)) { job->rw = WRITE; diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c index 05c35aacb3aa..6d35dd4e9efb 100644 --- a/drivers/md/dm-linear.c +++ b/drivers/md/dm-linear.c @@ -141,9 +141,27 @@ static int linear_iterate_devices(struct dm_target *ti, return fn(ti, lc->dev, lc->start, ti->len, data); } +static long linear_direct_access(struct dm_target *ti, sector_t sector, + void __pmem **kaddr, pfn_t *pfn, long size) +{ + struct linear_c *lc = ti->private; + struct block_device *bdev = lc->dev->bdev; + struct blk_dax_ctl dax = { + .sector = linear_map_sector(ti, sector), + .size = size, + }; + long ret; + + ret = bdev_direct_access(bdev, &dax); + *kaddr = dax.addr; + *pfn = dax.pfn; + + return ret; +} + static struct target_type linear_target = { .name = "linear", - .version = {1, 2, 1}, + .version = {1, 3, 0}, .module = THIS_MODULE, .ctr = linear_ctr, .dtr = linear_dtr, @@ -151,6 +169,7 @@ static struct target_type linear_target = { .status = linear_status, .prepare_ioctl = linear_prepare_ioctl, .iterate_devices = linear_iterate_devices, + .direct_access = linear_direct_access, }; int __init dm_linear_init(void) diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c index 608302e222af..b5dbf7a0515e 100644 --- a/drivers/md/dm-log-writes.c +++ b/drivers/md/dm-log-writes.c @@ -205,6 +205,7 @@ static int write_metadata(struct log_writes_c *lc, void *entry, bio->bi_bdev = lc->logdev->bdev; bio->bi_end_io = log_end_io; bio->bi_private = lc; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); page = alloc_page(GFP_KERNEL); if (!page) { @@ -226,7 +227,7 @@ static int write_metadata(struct log_writes_c *lc, void *entry, DMERR("Couldn't add page to the log block"); goto error_bio; } - submit_bio(WRITE, bio); + submit_bio(bio); return 0; error_bio: bio_put(bio); @@ -269,6 +270,7 @@ static int log_one_block(struct log_writes_c *lc, bio->bi_bdev = lc->logdev->bdev; bio->bi_end_io = log_end_io; bio->bi_private = lc; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); for (i = 0; i < block->vec_cnt; i++) { /* @@ -279,7 +281,7 @@ static int log_one_block(struct log_writes_c *lc, block->vecs[i].bv_len, 0); if (ret != block->vecs[i].bv_len) { atomic_inc(&lc->io_blocks); - submit_bio(WRITE, bio); + submit_bio(bio); bio = bio_alloc(GFP_KERNEL, block->vec_cnt - i); if (!bio) { DMERR("Couldn't alloc log bio"); @@ -290,6 +292,7 @@ static int log_one_block(struct log_writes_c *lc, bio->bi_bdev = lc->logdev->bdev; bio->bi_end_io = log_end_io; bio->bi_private = lc; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); ret = bio_add_page(bio, block->vecs[i].bv_page, block->vecs[i].bv_len, 0); @@ -301,7 +304,7 @@ static int log_one_block(struct log_writes_c *lc, } sector += block->vecs[i].bv_len >> SECTOR_SHIFT; } - submit_bio(WRITE, bio); + submit_bio(bio); out: kfree(block->data); kfree(block); @@ -552,9 +555,9 @@ static int log_writes_map(struct dm_target *ti, struct bio *bio) struct bio_vec bv; size_t alloc_size; int i = 0; - bool flush_bio = (bio->bi_rw & REQ_FLUSH); + bool flush_bio = (bio->bi_rw & REQ_PREFLUSH); bool fua_bio = (bio->bi_rw & REQ_FUA); - bool discard_bio = (bio->bi_rw & REQ_DISCARD); + bool discard_bio = (bio_op(bio) == REQ_OP_DISCARD); pb->block = NULL; diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index 627d19186d5a..4ca2d1df5b44 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -293,7 +293,7 @@ static void header_from_disk(struct log_header_core *core, struct log_header_dis static int rw_header(struct log_c *lc, int rw) { - lc->io_req.bi_rw = rw; + lc->io_req.bi_op = rw; return dm_io(&lc->io_req, 1, &lc->header_location, NULL); } @@ -306,7 +306,8 @@ static int flush_header(struct log_c *lc) .count = 0, }; - lc->io_req.bi_rw = WRITE_FLUSH; + lc->io_req.bi_op = REQ_OP_WRITE; + lc->io_req.bi_op_flags = WRITE_FLUSH; return dm_io(&lc->io_req, 1, &null_location, NULL); } diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index 52baf8a5b0f4..7eac080fcb18 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -7,7 +7,8 @@ #include -#include "dm.h" +#include "dm-rq.h" +#include "dm-bio-record.h" #include "dm-path-selector.h" #include "dm-uevent.h" @@ -89,6 +90,8 @@ struct multipath { atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */ atomic_t pg_init_count; /* Number of times pg_init called */ + unsigned queue_mode; + /* * We must use a mempool of dm_mpath_io structs so that we * can resubmit bios on error. @@ -97,10 +100,13 @@ struct multipath { struct mutex work_mutex; struct work_struct trigger_event; + + struct work_struct process_queued_bios; + struct bio_list queued_bios; }; /* - * Context information attached to each bio we process. + * Context information attached to each io we process. */ struct dm_mpath_io { struct pgpath *pgpath; @@ -114,6 +120,7 @@ static struct kmem_cache *_mpio_cache; static struct workqueue_struct *kmultipathd, *kmpath_handlerd; static void trigger_event(struct work_struct *work); static void activate_path(struct work_struct *work); +static void process_queued_bios(struct work_struct *work); /*----------------------------------------------- * Multipath state flags. @@ -185,7 +192,7 @@ static void free_priority_group(struct priority_group *pg, kfree(pg); } -static struct multipath *alloc_multipath(struct dm_target *ti, bool use_blk_mq) +static struct multipath *alloc_multipath(struct dm_target *ti) { struct multipath *m; @@ -203,15 +210,7 @@ static struct multipath *alloc_multipath(struct dm_target *ti, bool use_blk_mq) mutex_init(&m->work_mutex); m->mpio_pool = NULL; - if (!use_blk_mq) { - unsigned min_ios = dm_get_reserved_rq_based_ios(); - - m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache); - if (!m->mpio_pool) { - kfree(m); - return NULL; - } - } + m->queue_mode = DM_TYPE_NONE; m->ti = ti; ti->private = m; @@ -220,6 +219,39 @@ static struct multipath *alloc_multipath(struct dm_target *ti, bool use_blk_mq) return m; } +static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) +{ + if (m->queue_mode == DM_TYPE_NONE) { + /* + * Default to request-based. + */ + if (dm_use_blk_mq(dm_table_get_md(ti->table))) + m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; + else + m->queue_mode = DM_TYPE_REQUEST_BASED; + } + + if (m->queue_mode == DM_TYPE_REQUEST_BASED) { + unsigned min_ios = dm_get_reserved_rq_based_ios(); + + m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache); + if (!m->mpio_pool) + return -ENOMEM; + } + else if (m->queue_mode == DM_TYPE_BIO_BASED) { + INIT_WORK(&m->process_queued_bios, process_queued_bios); + /* + * bio-based doesn't support any direct scsi_dh management; + * it just discovers if a scsi_dh is attached. + */ + set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); + } + + dm_table_set_type(ti->table, m->queue_mode); + + return 0; +} + static void free_multipath(struct multipath *m) { struct priority_group *pg, *tmp; @@ -272,6 +304,41 @@ static void clear_request_fn_mpio(struct multipath *m, union map_info *info) } } +static size_t multipath_per_bio_data_size(void) +{ + return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details); +} + +static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio) +{ + return dm_per_bio_data(bio, multipath_per_bio_data_size()); +} + +static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio) +{ + /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */ + struct dm_mpath_io *mpio = get_mpio_from_bio(bio); + void *bio_details = mpio + 1; + + return bio_details; +} + +static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p, + struct dm_bio_details **bio_details_p) +{ + struct dm_mpath_io *mpio = get_mpio_from_bio(bio); + struct dm_bio_details *bio_details = get_bio_details_from_bio(bio); + + memset(mpio, 0, sizeof(*mpio)); + memset(bio_details, 0, sizeof(*bio_details)); + dm_bio_record(bio_details, bio); + + if (mpio_p) + *mpio_p = mpio; + if (bio_details_p) + *bio_details_p = bio_details; +} + /*----------------------------------------------- * Path selection *-----------------------------------------------*/ @@ -431,16 +498,26 @@ failed: * and multipath_resume() calls and we have no need to check * for the DMF_NOFLUSH_SUSPENDING flag. */ -static int must_push_back(struct multipath *m) +static bool __must_push_back(struct multipath *m) +{ + return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) != + test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) && + dm_noflush_suspending(m->ti)); +} + +static bool must_push_back_rq(struct multipath *m) { return (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) || - ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) != - test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) && - dm_noflush_suspending(m->ti))); + __must_push_back(m)); +} + +static bool must_push_back_bio(struct multipath *m) +{ + return __must_push_back(m); } /* - * Map cloned requests + * Map cloned requests (request-based multipath) */ static int __multipath_map(struct dm_target *ti, struct request *clone, union map_info *map_context, @@ -459,7 +536,7 @@ static int __multipath_map(struct dm_target *ti, struct request *clone, pgpath = choose_pgpath(m, nr_bytes); if (!pgpath) { - if (!must_push_back(m)) + if (!must_push_back_rq(m)) r = -EIO; /* Failed */ return r; } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) || @@ -529,6 +606,108 @@ static void multipath_release_clone(struct request *clone) blk_mq_free_request(clone); } +/* + * Map cloned bios (bio-based multipath) + */ +static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio) +{ + size_t nr_bytes = bio->bi_iter.bi_size; + struct pgpath *pgpath; + unsigned long flags; + bool queue_io; + + /* Do we need to select a new pgpath? */ + pgpath = lockless_dereference(m->current_pgpath); + queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags); + if (!pgpath || !queue_io) + pgpath = choose_pgpath(m, nr_bytes); + + if ((pgpath && queue_io) || + (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) { + /* Queue for the daemon to resubmit */ + spin_lock_irqsave(&m->lock, flags); + bio_list_add(&m->queued_bios, bio); + spin_unlock_irqrestore(&m->lock, flags); + /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */ + if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) + pg_init_all_paths(m); + else if (!queue_io) + queue_work(kmultipathd, &m->process_queued_bios); + return DM_MAPIO_SUBMITTED; + } + + if (!pgpath) { + if (!must_push_back_bio(m)) + return -EIO; + return DM_MAPIO_REQUEUE; + } + + mpio->pgpath = pgpath; + mpio->nr_bytes = nr_bytes; + + bio->bi_error = 0; + bio->bi_bdev = pgpath->path.dev->bdev; + bio->bi_rw |= REQ_FAILFAST_TRANSPORT; + + if (pgpath->pg->ps.type->start_io) + pgpath->pg->ps.type->start_io(&pgpath->pg->ps, + &pgpath->path, + nr_bytes); + return DM_MAPIO_REMAPPED; +} + +static int multipath_map_bio(struct dm_target *ti, struct bio *bio) +{ + struct multipath *m = ti->private; + struct dm_mpath_io *mpio = NULL; + + multipath_init_per_bio_data(bio, &mpio, NULL); + + return __multipath_map_bio(m, bio, mpio); +} + +static void process_queued_bios_list(struct multipath *m) +{ + if (m->queue_mode == DM_TYPE_BIO_BASED) + queue_work(kmultipathd, &m->process_queued_bios); +} + +static void process_queued_bios(struct work_struct *work) +{ + int r; + unsigned long flags; + struct bio *bio; + struct bio_list bios; + struct blk_plug plug; + struct multipath *m = + container_of(work, struct multipath, process_queued_bios); + + bio_list_init(&bios); + + spin_lock_irqsave(&m->lock, flags); + + if (bio_list_empty(&m->queued_bios)) { + spin_unlock_irqrestore(&m->lock, flags); + return; + } + + bio_list_merge(&bios, &m->queued_bios); + bio_list_init(&m->queued_bios); + + spin_unlock_irqrestore(&m->lock, flags); + + blk_start_plug(&plug); + while ((bio = bio_list_pop(&bios))) { + r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio)); + if (r < 0 || r == DM_MAPIO_REQUEUE) { + bio->bi_error = r; + bio_endio(bio); + } else if (r == DM_MAPIO_REMAPPED) + generic_make_request(bio); + } + blk_finish_plug(&plug); +} + /* * If we run out of usable paths, should we queue I/O or error it? */ @@ -557,8 +736,10 @@ static int queue_if_no_path(struct multipath *m, bool queue_if_no_path, spin_unlock_irqrestore(&m->lock, flags); - if (!queue_if_no_path) + if (!queue_if_no_path) { dm_table_run_md_queue_async(m->ti->table); + process_queued_bios_list(m); + } return 0; } @@ -798,6 +979,12 @@ static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m) if (!hw_argc) return 0; + if (m->queue_mode == DM_TYPE_BIO_BASED) { + dm_consume_args(as, hw_argc); + DMERR("bio-based multipath doesn't allow hardware handler args"); + return 0; + } + m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL); if (hw_argc > 1) { @@ -833,7 +1020,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m) const char *arg_name; static struct dm_arg _args[] = { - {0, 6, "invalid number of feature args"}, + {0, 8, "invalid number of feature args"}, {1, 50, "pg_init_retries must be between 1 and 50"}, {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"}, }; @@ -873,6 +1060,24 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m) continue; } + if (!strcasecmp(arg_name, "queue_mode") && + (argc >= 1)) { + const char *queue_mode_name = dm_shift_arg(as); + + if (!strcasecmp(queue_mode_name, "bio")) + m->queue_mode = DM_TYPE_BIO_BASED; + else if (!strcasecmp(queue_mode_name, "rq")) + m->queue_mode = DM_TYPE_REQUEST_BASED; + else if (!strcasecmp(queue_mode_name, "mq")) + m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; + else { + ti->error = "Unknown 'queue_mode' requested"; + r = -EINVAL; + } + argc--; + continue; + } + ti->error = "Unrecognised multipath feature request"; r = -EINVAL; } while (argc && !r); @@ -880,8 +1085,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m) return r; } -static int multipath_ctr(struct dm_target *ti, unsigned int argc, - char **argv) +static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv) { /* target arguments */ static struct dm_arg _args[] = { @@ -894,12 +1098,11 @@ static int multipath_ctr(struct dm_target *ti, unsigned int argc, struct dm_arg_set as; unsigned pg_count = 0; unsigned next_pg_num; - bool use_blk_mq = dm_use_blk_mq(dm_table_get_md(ti->table)); as.argc = argc; as.argv = argv; - m = alloc_multipath(ti, use_blk_mq); + m = alloc_multipath(ti); if (!m) { ti->error = "can't allocate multipath"; return -EINVAL; @@ -909,6 +1112,10 @@ static int multipath_ctr(struct dm_target *ti, unsigned int argc, if (r) goto bad; + r = alloc_multipath_stage2(ti, m); + if (r) + goto bad; + r = parse_hw_handler(&as, m); if (r) goto bad; @@ -958,7 +1165,9 @@ static int multipath_ctr(struct dm_target *ti, unsigned int argc, ti->num_flush_bios = 1; ti->num_discard_bios = 1; ti->num_write_same_bios = 1; - if (use_blk_mq) + if (m->queue_mode == DM_TYPE_BIO_BASED) + ti->per_io_data_size = multipath_per_bio_data_size(); + else if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED) ti->per_io_data_size = sizeof(struct dm_mpath_io); return 0; @@ -1083,8 +1292,10 @@ static int reinstate_path(struct pgpath *pgpath) out: spin_unlock_irqrestore(&m->lock, flags); - if (run_queue) + if (run_queue) { dm_table_run_md_queue_async(m->ti->table); + process_queued_bios_list(m); + } return r; } @@ -1281,6 +1492,8 @@ static void pg_init_done(void *data, int errors) } clear_bit(MPATHF_QUEUE_IO, &m->flags); + process_queued_bios_list(m); + /* * Wake up any thread waiting to suspend. */ @@ -1328,7 +1541,7 @@ static int do_end_io(struct multipath *m, struct request *clone, * during end I/O handling, since those clone requests don't have * bio clones. If we queue them inside the multipath target, * we need to make bio clones, that requires memory allocation. - * (See drivers/md/dm.c:end_clone_bio() about why the clone requests + * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests * don't have bio clones.) * Instead of queueing the clone request here, we queue the original * request into dm core, which will remake a clone request and @@ -1347,7 +1560,7 @@ static int do_end_io(struct multipath *m, struct request *clone, if (!atomic_read(&m->nr_valid_paths)) { if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { - if (!must_push_back(m)) + if (!must_push_back_rq(m)) r = -EIO; } else { if (error == -EBADE) @@ -1381,6 +1594,64 @@ static int multipath_end_io(struct dm_target *ti, struct request *clone, return r; } +static int do_end_io_bio(struct multipath *m, struct bio *clone, + int error, struct dm_mpath_io *mpio) +{ + unsigned long flags; + + if (!error) + return 0; /* I/O complete */ + + if (noretry_error(error)) + return error; + + if (mpio->pgpath) + fail_path(mpio->pgpath); + + if (!atomic_read(&m->nr_valid_paths)) { + if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { + if (!must_push_back_bio(m)) + return -EIO; + return DM_ENDIO_REQUEUE; + } else { + if (error == -EBADE) + return error; + } + } + + /* Queue for the daemon to resubmit */ + dm_bio_restore(get_bio_details_from_bio(clone), clone); + + spin_lock_irqsave(&m->lock, flags); + bio_list_add(&m->queued_bios, clone); + spin_unlock_irqrestore(&m->lock, flags); + if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) + queue_work(kmultipathd, &m->process_queued_bios); + + return DM_ENDIO_INCOMPLETE; +} + +static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone, int error) +{ + struct multipath *m = ti->private; + struct dm_mpath_io *mpio = get_mpio_from_bio(clone); + struct pgpath *pgpath; + struct path_selector *ps; + int r; + + BUG_ON(!mpio); + + r = do_end_io_bio(m, clone, error, mpio); + pgpath = mpio->pgpath; + if (pgpath) { + ps = &pgpath->pg->ps; + if (ps->type->end_io) + ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); + } + + return r; +} + /* * Suspend can't complete until all the I/O is processed so if * the last path fails we must error any remaining I/O. @@ -1454,7 +1725,9 @@ static void multipath_status(struct dm_target *ti, status_type_t type, DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) + (m->pg_init_retries > 0) * 2 + (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 + - test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)); + test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) + + (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2); + if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) DMEMIT("queue_if_no_path "); if (m->pg_init_retries) @@ -1463,6 +1736,16 @@ static void multipath_status(struct dm_target *ti, status_type_t type, DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs); if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) DMEMIT("retain_attached_hw_handler "); + if (m->queue_mode != DM_TYPE_REQUEST_BASED) { + switch(m->queue_mode) { + case DM_TYPE_BIO_BASED: + DMEMIT("queue_mode bio "); + break; + case DM_TYPE_MQ_REQUEST_BASED: + DMEMIT("queue_mode mq "); + break; + } + } } if (!m->hw_handler_name || type == STATUSTYPE_INFO) @@ -1642,6 +1925,7 @@ static int multipath_prepare_ioctl(struct dm_target *ti, if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) pg_init_all_paths(m); dm_table_run_md_queue_async(m->ti->table); + process_queued_bios_list(m); } /* @@ -1748,7 +2032,7 @@ static int multipath_busy(struct dm_target *ti) *---------------------------------------------------------------*/ static struct target_type multipath_target = { .name = "multipath", - .version = {1, 11, 0}, + .version = {1, 12, 0}, .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, .module = THIS_MODULE, .ctr = multipath_ctr, @@ -1757,6 +2041,8 @@ static struct target_type multipath_target = { .clone_and_map_rq = multipath_clone_and_map, .release_clone_rq = multipath_release_clone, .rq_end_io = multipath_end_io, + .map = multipath_map_bio, + .end_io = multipath_end_io_bio, .presuspend = multipath_presuspend, .postsuspend = multipath_postsuspend, .resume = multipath_resume, @@ -1771,14 +2057,14 @@ static int __init dm_multipath_init(void) { int r; - /* allocate a slab for the dm_ios */ + /* allocate a slab for the dm_mpath_ios */ _mpio_cache = KMEM_CACHE(dm_mpath_io, 0); if (!_mpio_cache) return -ENOMEM; r = dm_register_target(&multipath_target); if (r < 0) { - DMERR("register failed %d", r); + DMERR("request-based register failed %d", r); r = -EINVAL; goto bad_register_target; } @@ -1804,10 +2090,6 @@ static int __init dm_multipath_init(void) goto bad_alloc_kmpath_handlerd; } - DMINFO("version %u.%u.%u loaded", - multipath_target.version[0], multipath_target.version[1], - multipath_target.version[2]); - return 0; bad_alloc_kmpath_handlerd: diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 52532745a50f..84983549b5e1 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -1,6 +1,6 @@ /* * Copyright (C) 2010-2011 Neil Brown - * Copyright (C) 2010-2015 Red Hat, Inc. All rights reserved. + * Copyright (C) 2010-2016 Red Hat, Inc. All rights reserved. * * This file is released under the GPL. */ @@ -17,7 +17,12 @@ #include #define DM_MSG_PREFIX "raid" -#define MAX_RAID_DEVICES 253 /* raid4/5/6 limit */ +#define MAX_RAID_DEVICES 253 /* md-raid kernel limit */ + +/* + * Minimum sectors of free reshape space per raid device + */ +#define MIN_FREE_RESHAPE_SPACE to_sector(4*4096) static bool devices_handle_discard_safely = false; @@ -25,12 +30,12 @@ static bool devices_handle_discard_safely = false; * The following flags are used by dm-raid.c to set up the array state. * They must be cleared before md_run is called. */ -#define FirstUse 10 /* rdev flag */ +#define FirstUse 10 /* rdev flag */ struct raid_dev { /* * Two DM devices, one to hold metadata and one to hold the - * actual data/parity. The reason for this is to not confuse + * actual data/parity. The reason for this is to not confuse * ti->len and give more flexibility in altering size and * characteristics. * @@ -45,26 +50,176 @@ struct raid_dev { struct md_rdev rdev; }; +/* + * Bits for establishing rs->ctr_flags + * + * 1 = no flag value + * 2 = flag with value + */ +#define __CTR_FLAG_SYNC 0 /* 1 */ /* Not with raid0! */ +#define __CTR_FLAG_NOSYNC 1 /* 1 */ /* Not with raid0! */ +#define __CTR_FLAG_REBUILD 2 /* 2 */ /* Not with raid0! */ +#define __CTR_FLAG_DAEMON_SLEEP 3 /* 2 */ /* Not with raid0! */ +#define __CTR_FLAG_MIN_RECOVERY_RATE 4 /* 2 */ /* Not with raid0! */ +#define __CTR_FLAG_MAX_RECOVERY_RATE 5 /* 2 */ /* Not with raid0! */ +#define __CTR_FLAG_MAX_WRITE_BEHIND 6 /* 2 */ /* Only with raid1! */ +#define __CTR_FLAG_WRITE_MOSTLY 7 /* 2 */ /* Only with raid1! */ +#define __CTR_FLAG_STRIPE_CACHE 8 /* 2 */ /* Only with raid4/5/6! */ +#define __CTR_FLAG_REGION_SIZE 9 /* 2 */ /* Not with raid0! */ +#define __CTR_FLAG_RAID10_COPIES 10 /* 2 */ /* Only with raid10 */ +#define __CTR_FLAG_RAID10_FORMAT 11 /* 2 */ /* Only with raid10 */ +/* New for v1.9.0 */ +#define __CTR_FLAG_DELTA_DISKS 12 /* 2 */ /* Only with reshapable raid1/4/5/6/10! */ +#define __CTR_FLAG_DATA_OFFSET 13 /* 2 */ /* Only with reshapable raid4/5/6/10! */ +#define __CTR_FLAG_RAID10_USE_NEAR_SETS 14 /* 2 */ /* Only with raid10! */ + /* * Flags for rs->ctr_flags field. */ -#define CTR_FLAG_SYNC 0x1 -#define CTR_FLAG_NOSYNC 0x2 -#define CTR_FLAG_REBUILD 0x4 -#define CTR_FLAG_DAEMON_SLEEP 0x8 -#define CTR_FLAG_MIN_RECOVERY_RATE 0x10 -#define CTR_FLAG_MAX_RECOVERY_RATE 0x20 -#define CTR_FLAG_MAX_WRITE_BEHIND 0x40 -#define CTR_FLAG_STRIPE_CACHE 0x80 -#define CTR_FLAG_REGION_SIZE 0x100 -#define CTR_FLAG_RAID10_COPIES 0x200 -#define CTR_FLAG_RAID10_FORMAT 0x400 +#define CTR_FLAG_SYNC (1 << __CTR_FLAG_SYNC) +#define CTR_FLAG_NOSYNC (1 << __CTR_FLAG_NOSYNC) +#define CTR_FLAG_REBUILD (1 << __CTR_FLAG_REBUILD) +#define CTR_FLAG_DAEMON_SLEEP (1 << __CTR_FLAG_DAEMON_SLEEP) +#define CTR_FLAG_MIN_RECOVERY_RATE (1 << __CTR_FLAG_MIN_RECOVERY_RATE) +#define CTR_FLAG_MAX_RECOVERY_RATE (1 << __CTR_FLAG_MAX_RECOVERY_RATE) +#define CTR_FLAG_MAX_WRITE_BEHIND (1 << __CTR_FLAG_MAX_WRITE_BEHIND) +#define CTR_FLAG_WRITE_MOSTLY (1 << __CTR_FLAG_WRITE_MOSTLY) +#define CTR_FLAG_STRIPE_CACHE (1 << __CTR_FLAG_STRIPE_CACHE) +#define CTR_FLAG_REGION_SIZE (1 << __CTR_FLAG_REGION_SIZE) +#define CTR_FLAG_RAID10_COPIES (1 << __CTR_FLAG_RAID10_COPIES) +#define CTR_FLAG_RAID10_FORMAT (1 << __CTR_FLAG_RAID10_FORMAT) +#define CTR_FLAG_DELTA_DISKS (1 << __CTR_FLAG_DELTA_DISKS) +#define CTR_FLAG_DATA_OFFSET (1 << __CTR_FLAG_DATA_OFFSET) +#define CTR_FLAG_RAID10_USE_NEAR_SETS (1 << __CTR_FLAG_RAID10_USE_NEAR_SETS) + +/* + * Definitions of various constructor flags to + * be used in checks of valid / invalid flags + * per raid level. + */ +/* Define all any sync flags */ +#define CTR_FLAGS_ANY_SYNC (CTR_FLAG_SYNC | CTR_FLAG_NOSYNC) + +/* Define flags for options without argument (e.g. 'nosync') */ +#define CTR_FLAG_OPTIONS_NO_ARGS (CTR_FLAGS_ANY_SYNC | \ + CTR_FLAG_RAID10_USE_NEAR_SETS) + +/* Define flags for options with one argument (e.g. 'delta_disks +2') */ +#define CTR_FLAG_OPTIONS_ONE_ARG (CTR_FLAG_REBUILD | \ + CTR_FLAG_WRITE_MOSTLY | \ + CTR_FLAG_DAEMON_SLEEP | \ + CTR_FLAG_MIN_RECOVERY_RATE | \ + CTR_FLAG_MAX_RECOVERY_RATE | \ + CTR_FLAG_MAX_WRITE_BEHIND | \ + CTR_FLAG_STRIPE_CACHE | \ + CTR_FLAG_REGION_SIZE | \ + CTR_FLAG_RAID10_COPIES | \ + CTR_FLAG_RAID10_FORMAT | \ + CTR_FLAG_DELTA_DISKS | \ + CTR_FLAG_DATA_OFFSET) + +/* Valid options definitions per raid level... */ + +/* "raid0" does only accept data offset */ +#define RAID0_VALID_FLAGS (CTR_FLAG_DATA_OFFSET) + +/* "raid1" does not accept stripe cache, data offset, delta_disks or any raid10 options */ +#define RAID1_VALID_FLAGS (CTR_FLAGS_ANY_SYNC | \ + CTR_FLAG_REBUILD | \ + CTR_FLAG_WRITE_MOSTLY | \ + CTR_FLAG_DAEMON_SLEEP | \ + CTR_FLAG_MIN_RECOVERY_RATE | \ + CTR_FLAG_MAX_RECOVERY_RATE | \ + CTR_FLAG_MAX_WRITE_BEHIND | \ + CTR_FLAG_REGION_SIZE | \ + CTR_FLAG_DELTA_DISKS | \ + CTR_FLAG_DATA_OFFSET) + +/* "raid10" does not accept any raid1 or stripe cache options */ +#define RAID10_VALID_FLAGS (CTR_FLAGS_ANY_SYNC | \ + CTR_FLAG_REBUILD | \ + CTR_FLAG_DAEMON_SLEEP | \ + CTR_FLAG_MIN_RECOVERY_RATE | \ + CTR_FLAG_MAX_RECOVERY_RATE | \ + CTR_FLAG_REGION_SIZE | \ + CTR_FLAG_RAID10_COPIES | \ + CTR_FLAG_RAID10_FORMAT | \ + CTR_FLAG_DELTA_DISKS | \ + CTR_FLAG_DATA_OFFSET | \ + CTR_FLAG_RAID10_USE_NEAR_SETS) + +/* + * "raid4/5/6" do not accept any raid1 or raid10 specific options + * + * "raid6" does not accept "nosync", because it is not guaranteed + * that both parity and q-syndrome are being written properly with + * any writes + */ +#define RAID45_VALID_FLAGS (CTR_FLAGS_ANY_SYNC | \ + CTR_FLAG_REBUILD | \ + CTR_FLAG_DAEMON_SLEEP | \ + CTR_FLAG_MIN_RECOVERY_RATE | \ + CTR_FLAG_MAX_RECOVERY_RATE | \ + CTR_FLAG_MAX_WRITE_BEHIND | \ + CTR_FLAG_STRIPE_CACHE | \ + CTR_FLAG_REGION_SIZE | \ + CTR_FLAG_DELTA_DISKS | \ + CTR_FLAG_DATA_OFFSET) + +#define RAID6_VALID_FLAGS (CTR_FLAG_SYNC | \ + CTR_FLAG_REBUILD | \ + CTR_FLAG_DAEMON_SLEEP | \ + CTR_FLAG_MIN_RECOVERY_RATE | \ + CTR_FLAG_MAX_RECOVERY_RATE | \ + CTR_FLAG_MAX_WRITE_BEHIND | \ + CTR_FLAG_STRIPE_CACHE | \ + CTR_FLAG_REGION_SIZE | \ + CTR_FLAG_DELTA_DISKS | \ + CTR_FLAG_DATA_OFFSET) +/* ...valid options definitions per raid level */ + +/* + * Flags for rs->runtime_flags field + * (RT_FLAG prefix meaning "runtime flag") + * + * These are all internal and used to define runtime state, + * e.g. to prevent another resume from preresume processing + * the raid set all over again. + */ +#define RT_FLAG_RS_PRERESUMED 0 +#define RT_FLAG_RS_RESUMED 1 +#define RT_FLAG_RS_BITMAP_LOADED 2 +#define RT_FLAG_UPDATE_SBS 3 +#define RT_FLAG_RESHAPE_RS 4 +#define RT_FLAG_KEEP_RS_FROZEN 5 + +/* Array elements of 64 bit needed for rebuild/failed disk bits */ +#define DISKS_ARRAY_ELEMS ((MAX_RAID_DEVICES + (sizeof(uint64_t) * 8 - 1)) / sizeof(uint64_t) / 8) + +/* + * raid set level, layout and chunk sectors backup/restore + */ +struct rs_layout { + int new_level; + int new_layout; + int new_chunk_sectors; +}; struct raid_set { struct dm_target *ti; uint32_t bitmap_loaded; - uint32_t ctr_flags; + uint32_t stripe_cache_entries; + unsigned long ctr_flags; + unsigned long runtime_flags; + + uint64_t rebuild_disks[DISKS_ARRAY_ELEMS]; + + int raid_disks; + int delta_disks; + int data_offset; + int raid10_copies; + int requested_bitmap_chunk_sectors; struct mddev md; struct raid_type *raid_type; @@ -73,82 +228,446 @@ struct raid_set { struct raid_dev dev[0]; }; +static void rs_config_backup(struct raid_set *rs, struct rs_layout *l) +{ + struct mddev *mddev = &rs->md; + + l->new_level = mddev->new_level; + l->new_layout = mddev->new_layout; + l->new_chunk_sectors = mddev->new_chunk_sectors; +} + +static void rs_config_restore(struct raid_set *rs, struct rs_layout *l) +{ + struct mddev *mddev = &rs->md; + + mddev->new_level = l->new_level; + mddev->new_layout = l->new_layout; + mddev->new_chunk_sectors = l->new_chunk_sectors; +} + +/* raid10 algorithms (i.e. formats) */ +#define ALGORITHM_RAID10_DEFAULT 0 +#define ALGORITHM_RAID10_NEAR 1 +#define ALGORITHM_RAID10_OFFSET 2 +#define ALGORITHM_RAID10_FAR 3 + /* Supported raid types and properties. */ static struct raid_type { const char *name; /* RAID algorithm. */ const char *descr; /* Descriptor text for logging. */ - const unsigned parity_devs; /* # of parity devices. */ - const unsigned minimal_devs; /* minimal # of devices in set. */ - const unsigned level; /* RAID level. */ - const unsigned algorithm; /* RAID algorithm. */ + const unsigned int parity_devs; /* # of parity devices. */ + const unsigned int minimal_devs;/* minimal # of devices in set. */ + const unsigned int level; /* RAID level. */ + const unsigned int algorithm; /* RAID algorithm. */ } raid_types[] = { - {"raid0", "RAID0 (striping)", 0, 2, 0, 0 /* NONE */}, - {"raid1", "RAID1 (mirroring)", 0, 2, 1, 0 /* NONE */}, - {"raid10", "RAID10 (striped mirrors)", 0, 2, 10, UINT_MAX /* Varies */}, - {"raid4", "RAID4 (dedicated parity disk)", 1, 2, 5, ALGORITHM_PARITY_0}, - {"raid5_la", "RAID5 (left asymmetric)", 1, 2, 5, ALGORITHM_LEFT_ASYMMETRIC}, - {"raid5_ra", "RAID5 (right asymmetric)", 1, 2, 5, ALGORITHM_RIGHT_ASYMMETRIC}, - {"raid5_ls", "RAID5 (left symmetric)", 1, 2, 5, ALGORITHM_LEFT_SYMMETRIC}, - {"raid5_rs", "RAID5 (right symmetric)", 1, 2, 5, ALGORITHM_RIGHT_SYMMETRIC}, - {"raid6_zr", "RAID6 (zero restart)", 2, 4, 6, ALGORITHM_ROTATING_ZERO_RESTART}, - {"raid6_nr", "RAID6 (N restart)", 2, 4, 6, ALGORITHM_ROTATING_N_RESTART}, - {"raid6_nc", "RAID6 (N continue)", 2, 4, 6, ALGORITHM_ROTATING_N_CONTINUE} + {"raid0", "raid0 (striping)", 0, 2, 0, 0 /* NONE */}, + {"raid1", "raid1 (mirroring)", 0, 2, 1, 0 /* NONE */}, + {"raid10_far", "raid10 far (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_FAR}, + {"raid10_offset", "raid10 offset (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_OFFSET}, + {"raid10_near", "raid10 near (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_NEAR}, + {"raid10", "raid10 (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_DEFAULT}, + {"raid4", "raid4 (dedicated last parity disk)", 1, 2, 4, ALGORITHM_PARITY_N}, /* raid4 layout = raid5_n */ + {"raid5_n", "raid5 (dedicated last parity disk)", 1, 2, 5, ALGORITHM_PARITY_N}, + {"raid5_ls", "raid5 (left symmetric)", 1, 2, 5, ALGORITHM_LEFT_SYMMETRIC}, + {"raid5_rs", "raid5 (right symmetric)", 1, 2, 5, ALGORITHM_RIGHT_SYMMETRIC}, + {"raid5_la", "raid5 (left asymmetric)", 1, 2, 5, ALGORITHM_LEFT_ASYMMETRIC}, + {"raid5_ra", "raid5 (right asymmetric)", 1, 2, 5, ALGORITHM_RIGHT_ASYMMETRIC}, + {"raid6_zr", "raid6 (zero restart)", 2, 4, 6, ALGORITHM_ROTATING_ZERO_RESTART}, + {"raid6_nr", "raid6 (N restart)", 2, 4, 6, ALGORITHM_ROTATING_N_RESTART}, + {"raid6_nc", "raid6 (N continue)", 2, 4, 6, ALGORITHM_ROTATING_N_CONTINUE}, + {"raid6_n_6", "raid6 (dedicated parity/Q n/6)", 2, 4, 6, ALGORITHM_PARITY_N_6}, + {"raid6_ls_6", "raid6 (left symmetric dedicated Q 6)", 2, 4, 6, ALGORITHM_LEFT_SYMMETRIC_6}, + {"raid6_rs_6", "raid6 (right symmetric dedicated Q 6)", 2, 4, 6, ALGORITHM_RIGHT_SYMMETRIC_6}, + {"raid6_la_6", "raid6 (left asymmetric dedicated Q 6)", 2, 4, 6, ALGORITHM_LEFT_ASYMMETRIC_6}, + {"raid6_ra_6", "raid6 (right asymmetric dedicated Q 6)", 2, 4, 6, ALGORITHM_RIGHT_ASYMMETRIC_6} +}; + +/* True, if @v is in inclusive range [@min, @max] */ +static bool __within_range(long v, long min, long max) +{ + return v >= min && v <= max; +} + +/* All table line arguments are defined here */ +static struct arg_name_flag { + const unsigned long flag; + const char *name; +} __arg_name_flags[] = { + { CTR_FLAG_SYNC, "sync"}, + { CTR_FLAG_NOSYNC, "nosync"}, + { CTR_FLAG_REBUILD, "rebuild"}, + { CTR_FLAG_DAEMON_SLEEP, "daemon_sleep"}, + { CTR_FLAG_MIN_RECOVERY_RATE, "min_recovery_rate"}, + { CTR_FLAG_MAX_RECOVERY_RATE, "max_recovery_rate"}, + { CTR_FLAG_MAX_WRITE_BEHIND, "max_write_behind"}, + { CTR_FLAG_WRITE_MOSTLY, "write_mostly"}, + { CTR_FLAG_STRIPE_CACHE, "stripe_cache"}, + { CTR_FLAG_REGION_SIZE, "region_size"}, + { CTR_FLAG_RAID10_COPIES, "raid10_copies"}, + { CTR_FLAG_RAID10_FORMAT, "raid10_format"}, + { CTR_FLAG_DATA_OFFSET, "data_offset"}, + { CTR_FLAG_DELTA_DISKS, "delta_disks"}, + { CTR_FLAG_RAID10_USE_NEAR_SETS, "raid10_use_near_sets"}, }; -static char *raid10_md_layout_to_format(int layout) +/* Return argument name string for given @flag */ +static const char *dm_raid_arg_name_by_flag(const uint32_t flag) +{ + if (hweight32(flag) == 1) { + struct arg_name_flag *anf = __arg_name_flags + ARRAY_SIZE(__arg_name_flags); + + while (anf-- > __arg_name_flags) + if (flag & anf->flag) + return anf->name; + + } else + DMERR("%s called with more than one flag!", __func__); + + return NULL; +} + +/* + * Bool helpers to test for various raid levels of a raid set. + * It's level as reported by the superblock rather than + * the requested raid_type passed to the constructor. + */ +/* Return true, if raid set in @rs is raid0 */ +static bool rs_is_raid0(struct raid_set *rs) +{ + return !rs->md.level; +} + +/* Return true, if raid set in @rs is raid1 */ +static bool rs_is_raid1(struct raid_set *rs) +{ + return rs->md.level == 1; +} + +/* Return true, if raid set in @rs is raid10 */ +static bool rs_is_raid10(struct raid_set *rs) +{ + return rs->md.level == 10; +} + +/* Return true, if raid set in @rs is level 6 */ +static bool rs_is_raid6(struct raid_set *rs) +{ + return rs->md.level == 6; +} + +/* Return true, if raid set in @rs is level 4, 5 or 6 */ +static bool rs_is_raid456(struct raid_set *rs) +{ + return __within_range(rs->md.level, 4, 6); +} + +/* Return true, if raid set in @rs is reshapable */ +static bool __is_raid10_far(int layout); +static bool rs_is_reshapable(struct raid_set *rs) +{ + return rs_is_raid456(rs) || + (rs_is_raid10(rs) && !__is_raid10_far(rs->md.new_layout)); +} + +/* Return true, if raid set in @rs is recovering */ +static bool rs_is_recovering(struct raid_set *rs) +{ + return rs->md.recovery_cp < rs->dev[0].rdev.sectors; +} + +/* Return true, if raid set in @rs is reshaping */ +static bool rs_is_reshaping(struct raid_set *rs) +{ + return rs->md.reshape_position != MaxSector; +} + +/* + * bool helpers to test for various raid levels of a raid type @rt + */ + +/* Return true, if raid type in @rt is raid0 */ +static bool rt_is_raid0(struct raid_type *rt) +{ + return !rt->level; +} + +/* Return true, if raid type in @rt is raid1 */ +static bool rt_is_raid1(struct raid_type *rt) +{ + return rt->level == 1; +} + +/* Return true, if raid type in @rt is raid10 */ +static bool rt_is_raid10(struct raid_type *rt) +{ + return rt->level == 10; +} + +/* Return true, if raid type in @rt is raid4/5 */ +static bool rt_is_raid45(struct raid_type *rt) +{ + return __within_range(rt->level, 4, 5); +} + +/* Return true, if raid type in @rt is raid6 */ +static bool rt_is_raid6(struct raid_type *rt) +{ + return rt->level == 6; +} + +/* Return true, if raid type in @rt is raid4/5/6 */ +static bool rt_is_raid456(struct raid_type *rt) +{ + return __within_range(rt->level, 4, 6); +} +/* END: raid level bools */ + +/* Return valid ctr flags for the raid level of @rs */ +static unsigned long __valid_flags(struct raid_set *rs) +{ + if (rt_is_raid0(rs->raid_type)) + return RAID0_VALID_FLAGS; + else if (rt_is_raid1(rs->raid_type)) + return RAID1_VALID_FLAGS; + else if (rt_is_raid10(rs->raid_type)) + return RAID10_VALID_FLAGS; + else if (rt_is_raid45(rs->raid_type)) + return RAID45_VALID_FLAGS; + else if (rt_is_raid6(rs->raid_type)) + return RAID6_VALID_FLAGS; + + return 0; +} + +/* + * Check for valid flags set on @rs + * + * Has to be called after parsing of the ctr flags! + */ +static int rs_check_for_valid_flags(struct raid_set *rs) +{ + if (rs->ctr_flags & ~__valid_flags(rs)) { + rs->ti->error = "Invalid flags combination"; + return -EINVAL; + } + + return 0; +} + +/* MD raid10 bit definitions and helpers */ +#define RAID10_OFFSET (1 << 16) /* stripes with data copies area adjacent on devices */ +#define RAID10_BROCKEN_USE_FAR_SETS (1 << 17) /* Broken in raid10.c: use sets instead of whole stripe rotation */ +#define RAID10_USE_FAR_SETS (1 << 18) /* Use sets instead of whole stripe rotation */ +#define RAID10_FAR_COPIES_SHIFT 8 /* raid10 # far copies shift (2nd byte of layout) */ + +/* Return md raid10 near copies for @layout */ +static unsigned int __raid10_near_copies(int layout) +{ + return layout & 0xFF; +} + +/* Return md raid10 far copies for @layout */ +static unsigned int __raid10_far_copies(int layout) +{ + return __raid10_near_copies(layout >> RAID10_FAR_COPIES_SHIFT); +} + +/* Return true if md raid10 offset for @layout */ +static bool __is_raid10_offset(int layout) +{ + return !!(layout & RAID10_OFFSET); +} + +/* Return true if md raid10 near for @layout */ +static bool __is_raid10_near(int layout) +{ + return !__is_raid10_offset(layout) && __raid10_near_copies(layout) > 1; +} + +/* Return true if md raid10 far for @layout */ +static bool __is_raid10_far(int layout) +{ + return !__is_raid10_offset(layout) && __raid10_far_copies(layout) > 1; +} + +/* Return md raid10 layout string for @layout */ +static const char *raid10_md_layout_to_format(int layout) { /* - * Bit 16 and 17 stand for "offset" and "use_far_sets" + * Bit 16 stands for "offset" + * (i.e. adjacent stripes hold copies) + * * Refer to MD's raid10.c for details */ - if ((layout & 0x10000) && (layout & 0x20000)) + if (__is_raid10_offset(layout)) return "offset"; - if ((layout & 0xFF) > 1) + if (__raid10_near_copies(layout) > 1) return "near"; + WARN_ON(__raid10_far_copies(layout) < 2); + return "far"; } -static unsigned raid10_md_layout_to_copies(int layout) +/* Return md raid10 algorithm for @name */ +static int raid10_name_to_format(const char *name) { - if ((layout & 0xFF) > 1) - return layout & 0xFF; - return (layout >> 8) & 0xFF; + if (!strcasecmp(name, "near")) + return ALGORITHM_RAID10_NEAR; + else if (!strcasecmp(name, "offset")) + return ALGORITHM_RAID10_OFFSET; + else if (!strcasecmp(name, "far")) + return ALGORITHM_RAID10_FAR; + + return -EINVAL; +} + +/* Return md raid10 copies for @layout */ +static unsigned int raid10_md_layout_to_copies(int layout) +{ + return max(__raid10_near_copies(layout), __raid10_far_copies(layout)); } -static int raid10_format_to_md_layout(char *format, unsigned copies) +/* Return md raid10 format id for @format string */ +static int raid10_format_to_md_layout(struct raid_set *rs, + unsigned int algorithm, + unsigned int copies) { - unsigned n = 1, f = 1; + unsigned int n = 1, f = 1, r = 0; - if (!strcasecmp("near", format)) + /* + * MD resilienece flaw: + * + * enabling use_far_sets for far/offset formats causes copies + * to be colocated on the same devs together with their origins! + * + * -> disable it for now in the definition above + */ + if (algorithm == ALGORITHM_RAID10_DEFAULT || + algorithm == ALGORITHM_RAID10_NEAR) n = copies; - else + + else if (algorithm == ALGORITHM_RAID10_OFFSET) { f = copies; + r = RAID10_OFFSET; + if (!test_bit(__CTR_FLAG_RAID10_USE_NEAR_SETS, &rs->ctr_flags)) + r |= RAID10_USE_FAR_SETS; - if (!strcasecmp("offset", format)) - return 0x30000 | (f << 8) | n; + } else if (algorithm == ALGORITHM_RAID10_FAR) { + f = copies; + r = !RAID10_OFFSET; + if (!test_bit(__CTR_FLAG_RAID10_USE_NEAR_SETS, &rs->ctr_flags)) + r |= RAID10_USE_FAR_SETS; - if (!strcasecmp("far", format)) - return 0x20000 | (f << 8) | n; + } else + return -EINVAL; - return (f << 8) | n; + return r | (f << RAID10_FAR_COPIES_SHIFT) | n; } +/* END: MD raid10 bit definitions and helpers */ -static struct raid_type *get_raid_type(char *name) +/* Check for any of the raid10 algorithms */ +static bool __got_raid10(struct raid_type *rtp, const int layout) { - int i; + if (rtp->level == 10) { + switch (rtp->algorithm) { + case ALGORITHM_RAID10_DEFAULT: + case ALGORITHM_RAID10_NEAR: + return __is_raid10_near(layout); + case ALGORITHM_RAID10_OFFSET: + return __is_raid10_offset(layout); + case ALGORITHM_RAID10_FAR: + return __is_raid10_far(layout); + default: + break; + } + } + + return false; +} + +/* Return raid_type for @name */ +static struct raid_type *get_raid_type(const char *name) +{ + struct raid_type *rtp = raid_types + ARRAY_SIZE(raid_types); + + while (rtp-- > raid_types) + if (!strcasecmp(rtp->name, name)) + return rtp; + + return NULL; +} + +/* Return raid_type for @name based derived from @level and @layout */ +static struct raid_type *get_raid_type_by_ll(const int level, const int layout) +{ + struct raid_type *rtp = raid_types + ARRAY_SIZE(raid_types); - for (i = 0; i < ARRAY_SIZE(raid_types); i++) - if (!strcmp(raid_types[i].name, name)) - return &raid_types[i]; + while (rtp-- > raid_types) { + /* RAID10 special checks based on @layout flags/properties */ + if (rtp->level == level && + (__got_raid10(rtp, layout) || rtp->algorithm == layout)) + return rtp; + } return NULL; } -static struct raid_set *context_alloc(struct dm_target *ti, struct raid_type *raid_type, unsigned raid_devs) +/* + * Conditionally change bdev capacity of @rs + * in case of a disk add/remove reshape + */ +static void rs_set_capacity(struct raid_set *rs) +{ + struct mddev *mddev = &rs->md; + struct md_rdev *rdev; + struct gendisk *gendisk = dm_disk(dm_table_get_md(rs->ti->table)); + + /* + * raid10 sets rdev->sector to the device size, which + * is unintended in case of out-of-place reshaping + */ + rdev_for_each(rdev, mddev) + rdev->sectors = mddev->dev_sectors; + + set_capacity(gendisk, mddev->array_sectors); + revalidate_disk(gendisk); +} + +/* + * Set the mddev properties in @rs to the current + * ones retrieved from the freshest superblock + */ +static void rs_set_cur(struct raid_set *rs) +{ + struct mddev *mddev = &rs->md; + + mddev->new_level = mddev->level; + mddev->new_layout = mddev->layout; + mddev->new_chunk_sectors = mddev->chunk_sectors; +} + +/* + * Set the mddev properties in @rs to the new + * ones requested by the ctr + */ +static void rs_set_new(struct raid_set *rs) +{ + struct mddev *mddev = &rs->md; + + mddev->level = mddev->new_level; + mddev->layout = mddev->new_layout; + mddev->chunk_sectors = mddev->new_chunk_sectors; + mddev->raid_disks = rs->raid_disks; + mddev->delta_disks = 0; +} + +static struct raid_set *raid_set_alloc(struct dm_target *ti, struct raid_type *raid_type, + unsigned int raid_devs) { - unsigned i; + unsigned int i; struct raid_set *rs; if (raid_devs <= raid_type->parity_devs) { @@ -164,15 +683,19 @@ static struct raid_set *context_alloc(struct dm_target *ti, struct raid_type *ra mddev_init(&rs->md); + rs->raid_disks = raid_devs; + rs->delta_disks = 0; + rs->ti = ti; rs->raid_type = raid_type; + rs->stripe_cache_entries = 256; rs->md.raid_disks = raid_devs; rs->md.level = raid_type->level; rs->md.new_level = rs->md.level; rs->md.layout = raid_type->algorithm; rs->md.new_layout = rs->md.layout; rs->md.delta_disks = 0; - rs->md.recovery_cp = 0; + rs->md.recovery_cp = MaxSector; for (i = 0; i < raid_devs; i++) md_rdev_init(&rs->dev[i].rdev); @@ -189,11 +712,11 @@ static struct raid_set *context_alloc(struct dm_target *ti, struct raid_type *ra return rs; } -static void context_free(struct raid_set *rs) +static void raid_set_free(struct raid_set *rs) { int i; - for (i = 0; i < rs->md.raid_disks; i++) { + for (i = 0; i < rs->raid_disks; i++) { if (rs->dev[i].meta_dev) dm_put_device(rs->ti, rs->dev[i].meta_dev); md_rdev_clear(&rs->dev[i].rdev); @@ -218,16 +741,22 @@ static void context_free(struct raid_set *rs) * - * * This code parses those words. If there is a failure, - * the caller must use context_free to unwind the operations. + * the caller must use raid_set_free() to unwind the operations. */ -static int dev_parms(struct raid_set *rs, char **argv) +static int parse_dev_params(struct raid_set *rs, struct dm_arg_set *as) { int i; int rebuild = 0; int metadata_available = 0; - int ret = 0; + int r = 0; + const char *arg; - for (i = 0; i < rs->md.raid_disks; i++, argv += 2) { + /* Put off the number of raid devices argument to get to dev pairs */ + arg = dm_shift_arg(as); + if (!arg) + return -EINVAL; + + for (i = 0; i < rs->raid_disks; i++) { rs->dev[i].rdev.raid_disk = i; rs->dev[i].meta_dev = NULL; @@ -240,39 +769,49 @@ static int dev_parms(struct raid_set *rs, char **argv) rs->dev[i].rdev.data_offset = 0; rs->dev[i].rdev.mddev = &rs->md; - if (strcmp(argv[0], "-")) { - ret = dm_get_device(rs->ti, argv[0], - dm_table_get_mode(rs->ti->table), - &rs->dev[i].meta_dev); - rs->ti->error = "RAID metadata device lookup failure"; - if (ret) - return ret; + arg = dm_shift_arg(as); + if (!arg) + return -EINVAL; + + if (strcmp(arg, "-")) { + r = dm_get_device(rs->ti, arg, dm_table_get_mode(rs->ti->table), + &rs->dev[i].meta_dev); + if (r) { + rs->ti->error = "RAID metadata device lookup failure"; + return r; + } rs->dev[i].rdev.sb_page = alloc_page(GFP_KERNEL); - if (!rs->dev[i].rdev.sb_page) + if (!rs->dev[i].rdev.sb_page) { + rs->ti->error = "Failed to allocate superblock page"; return -ENOMEM; + } } - if (!strcmp(argv[1], "-")) { + arg = dm_shift_arg(as); + if (!arg) + return -EINVAL; + + if (!strcmp(arg, "-")) { if (!test_bit(In_sync, &rs->dev[i].rdev.flags) && (!rs->dev[i].rdev.recovery_offset)) { rs->ti->error = "Drive designated for rebuild not specified"; return -EINVAL; } - rs->ti->error = "No data device supplied with metadata device"; - if (rs->dev[i].meta_dev) + if (rs->dev[i].meta_dev) { + rs->ti->error = "No data device supplied with metadata device"; return -EINVAL; + } continue; } - ret = dm_get_device(rs->ti, argv[1], - dm_table_get_mode(rs->ti->table), - &rs->dev[i].data_dev); - if (ret) { + r = dm_get_device(rs->ti, arg, dm_table_get_mode(rs->ti->table), + &rs->dev[i].data_dev); + if (r) { rs->ti->error = "RAID device lookup failure"; - return ret; + return r; } if (rs->dev[i].meta_dev) { @@ -280,7 +819,7 @@ static int dev_parms(struct raid_set *rs, char **argv) rs->dev[i].rdev.meta_bdev = rs->dev[i].meta_dev->bdev; } rs->dev[i].rdev.bdev = rs->dev[i].data_dev->bdev; - list_add(&rs->dev[i].rdev.same_set, &rs->md.disks); + list_add_tail(&rs->dev[i].rdev.same_set, &rs->md.disks); if (!test_bit(In_sync, &rs->dev[i].rdev.flags)) rebuild++; } @@ -301,8 +840,7 @@ static int dev_parms(struct raid_set *rs, char **argv) * * User could specify 'nosync' option if desperate. */ - DMERR("Unable to rebuild drive while array is not in-sync"); - rs->ti->error = "RAID device lookup failure"; + rs->ti->error = "Unable to rebuild drive while array is not in-sync"; return -EINVAL; } @@ -325,7 +863,7 @@ static int validate_region_size(struct raid_set *rs, unsigned long region_size) if (!region_size) { /* - * Choose a reasonable default. All figures in sectors. + * Choose a reasonable default. All figures in sectors. */ if (min_region_size > (1 << 13)) { /* If not a power of 2, make it the next power of 2 */ @@ -366,7 +904,7 @@ static int validate_region_size(struct raid_set *rs, unsigned long region_size) /* * Convert sectors to bytes. */ - rs->md.bitmap_info.chunksize = (region_size << 9); + rs->md.bitmap_info.chunksize = to_bytes(region_size); return 0; } @@ -382,9 +920,9 @@ static int validate_region_size(struct raid_set *rs, unsigned long region_size) */ static int validate_raid_redundancy(struct raid_set *rs) { - unsigned i, rebuild_cnt = 0; - unsigned rebuilds_per_group = 0, copies, d; - unsigned group_size, last_group_start; + unsigned int i, rebuild_cnt = 0; + unsigned int rebuilds_per_group = 0, copies; + unsigned int group_size, last_group_start; for (i = 0; i < rs->md.raid_disks; i++) if (!test_bit(In_sync, &rs->dev[i].rdev.flags) || @@ -403,7 +941,7 @@ static int validate_raid_redundancy(struct raid_set *rs) goto too_many; break; case 10: - copies = raid10_md_layout_to_copies(rs->md.layout); + copies = raid10_md_layout_to_copies(rs->md.new_layout); if (rebuild_cnt < copies) break; @@ -417,17 +955,16 @@ static int validate_raid_redundancy(struct raid_set *rs) * simple case where the number of devices is a multiple of the * number of copies, we must also handle cases where the number * of devices is not a multiple of the number of copies. - * E.g. dev1 dev2 dev3 dev4 dev5 - * A A B B C - * C D D E E + * E.g. dev1 dev2 dev3 dev4 dev5 + * A A B B C + * C D D E E */ - if (!strcmp("near", raid10_md_layout_to_format(rs->md.layout))) { - for (i = 0; i < rs->md.raid_disks * copies; i++) { + if (__is_raid10_near(rs->md.new_layout)) { + for (i = 0; i < rs->md.raid_disks; i++) { if (!(i % copies)) rebuilds_per_group = 0; - d = i % rs->md.raid_disks; - if ((!rs->dev[d].rdev.sb_page || - !test_bit(In_sync, &rs->dev[d].rdev.flags)) && + if ((!rs->dev[i].rdev.sb_page || + !test_bit(In_sync, &rs->dev[i].rdev.flags)) && (++rebuilds_per_group >= copies)) goto too_many; } @@ -442,7 +979,7 @@ static int validate_raid_redundancy(struct raid_set *rs) * use the 'use_far_sets' variant.) * * This check is somewhat complicated by the need to account - * for arrays that are not a multiple of (far) copies. This + * for arrays that are not a multiple of (far) copies. This * results in the need to treat the last (potentially larger) * set differently. */ @@ -475,42 +1012,48 @@ too_many: * * Argument definitions * The number of sectors per disk that - * will form the "stripe" + * will form the "stripe" * [[no]sync] Force or prevent recovery of the - * entire array + * entire array * [rebuild ] Rebuild the drive indicated by the index * [daemon_sleep ] Time between bitmap daemon work to - * clear bits + * clear bits * [min_recovery_rate ] Throttle RAID initialization * [max_recovery_rate ] Throttle RAID initialization * [write_mostly ] Indicate a write mostly drive via index * [max_write_behind ] See '-write-behind=' (man mdadm) * [stripe_cache ] Stripe cache size for higher RAIDs - * [region_size ] Defines granularity of bitmap + * [region_size ] Defines granularity of bitmap * * RAID10-only options: - * [raid10_copies <# copies>] Number of copies. (Default: 2) + * [raid10_copies <# copies>] Number of copies. (Default: 2) * [raid10_format ] Layout algorithm. (Default: near) */ -static int parse_raid_params(struct raid_set *rs, char **argv, - unsigned num_raid_params) -{ - char *raid10_format = "near"; - unsigned raid10_copies = 2; - unsigned i; - unsigned long value, region_size = 0; - sector_t sectors_per_dev = rs->ti->len; +static int parse_raid_params(struct raid_set *rs, struct dm_arg_set *as, + unsigned int num_raid_params) +{ + int value, raid10_format = ALGORITHM_RAID10_DEFAULT; + unsigned int raid10_copies = 2; + unsigned int i, write_mostly = 0; + unsigned int region_size = 0; sector_t max_io_len; - char *key; + const char *arg, *key; + struct raid_dev *rd; + struct raid_type *rt = rs->raid_type; + + arg = dm_shift_arg(as); + num_raid_params--; /* Account for chunk_size argument */ + + if (kstrtoint(arg, 10, &value) < 0) { + rs->ti->error = "Bad numerical argument given for chunk_size"; + return -EINVAL; + } /* * First, parse the in-order required arguments * "chunk_size" is the only argument of this type. */ - if ((kstrtoul(argv[0], 10, &value) < 0)) { - rs->ti->error = "Bad chunk size"; - return -EINVAL; - } else if (rs->raid_type->level == 1) { + if (rt_is_raid1(rt)) { if (value) DMERR("Ignoring chunk size parameter for RAID 1"); value = 0; @@ -523,8 +1066,6 @@ static int parse_raid_params(struct raid_set *rs, char **argv, } rs->md.new_chunk_sectors = rs->md.chunk_sectors = value; - argv++; - num_raid_params--; /* * We set each individual device as In_sync with a completed @@ -532,18 +1073,18 @@ static int parse_raid_params(struct raid_set *rs, char **argv, * replacement then one of the following cases applies: * * 1) User specifies 'rebuild'. - * - Device is reset when param is read. + * - Device is reset when param is read. * 2) A new device is supplied. - * - No matching superblock found, resets device. + * - No matching superblock found, resets device. * 3) Device failure was transient and returns on reload. - * - Failure noticed, resets device for bitmap replay. + * - Failure noticed, resets device for bitmap replay. * 4) Device hadn't completed recovery after previous failure. - * - Superblock is read and overrides recovery_offset. + * - Superblock is read and overrides recovery_offset. * * What is found in the superblocks of the devices is always * authoritative, unless 'rebuild' or '[no]sync' was specified. */ - for (i = 0; i < rs->md.raid_disks; i++) { + for (i = 0; i < rs->raid_disks; i++) { set_bit(In_sync, &rs->dev[i].rdev.flags); rs->dev[i].rdev.recovery_offset = MaxSector; } @@ -552,72 +1093,112 @@ static int parse_raid_params(struct raid_set *rs, char **argv, * Second, parse the unordered optional arguments */ for (i = 0; i < num_raid_params; i++) { - if (!strcasecmp(argv[i], "nosync")) { - rs->md.recovery_cp = MaxSector; - rs->ctr_flags |= CTR_FLAG_NOSYNC; + key = dm_shift_arg(as); + if (!key) { + rs->ti->error = "Not enough raid parameters given"; + return -EINVAL; + } + + if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_NOSYNC))) { + if (test_and_set_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)) { + rs->ti->error = "Only one 'nosync' argument allowed"; + return -EINVAL; + } + continue; + } + if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_SYNC))) { + if (test_and_set_bit(__CTR_FLAG_SYNC, &rs->ctr_flags)) { + rs->ti->error = "Only one 'sync' argument allowed"; + return -EINVAL; + } continue; } - if (!strcasecmp(argv[i], "sync")) { - rs->md.recovery_cp = 0; - rs->ctr_flags |= CTR_FLAG_SYNC; + if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_USE_NEAR_SETS))) { + if (test_and_set_bit(__CTR_FLAG_RAID10_USE_NEAR_SETS, &rs->ctr_flags)) { + rs->ti->error = "Only one 'raid10_use_new_sets' argument allowed"; + return -EINVAL; + } continue; } - /* The rest of the optional arguments come in key/value pairs */ - if ((i + 1) >= num_raid_params) { + arg = dm_shift_arg(as); + i++; /* Account for the argument pairs */ + if (!arg) { rs->ti->error = "Wrong number of raid parameters given"; return -EINVAL; } - key = argv[i++]; + /* + * Parameters that take a string value are checked here. + */ - /* Parameters that take a string value are checked here. */ - if (!strcasecmp(key, "raid10_format")) { - if (rs->raid_type->level != 10) { + if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_FORMAT))) { + if (test_and_set_bit(__CTR_FLAG_RAID10_FORMAT, &rs->ctr_flags)) { + rs->ti->error = "Only one 'raid10_format' argument pair allowed"; + return -EINVAL; + } + if (!rt_is_raid10(rt)) { rs->ti->error = "'raid10_format' is an invalid parameter for this RAID type"; return -EINVAL; } - if (strcmp("near", argv[i]) && - strcmp("far", argv[i]) && - strcmp("offset", argv[i])) { + raid10_format = raid10_name_to_format(arg); + if (raid10_format < 0) { rs->ti->error = "Invalid 'raid10_format' value given"; - return -EINVAL; + return raid10_format; } - raid10_format = argv[i]; - rs->ctr_flags |= CTR_FLAG_RAID10_FORMAT; continue; } - if (kstrtoul(argv[i], 10, &value) < 0) { + if (kstrtoint(arg, 10, &value) < 0) { rs->ti->error = "Bad numerical argument given in raid params"; return -EINVAL; } - /* Parameters that take a numeric value are checked here */ - if (!strcasecmp(key, "rebuild")) { - if (value >= rs->md.raid_disks) { + if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_REBUILD))) { + /* + * "rebuild" is being passed in by userspace to provide + * indexes of replaced devices and to set up additional + * devices on raid level takeover. + */ + if (!__within_range(value, 0, rs->raid_disks - 1)) { rs->ti->error = "Invalid rebuild index given"; return -EINVAL; } - clear_bit(In_sync, &rs->dev[value].rdev.flags); - rs->dev[value].rdev.recovery_offset = 0; - rs->ctr_flags |= CTR_FLAG_REBUILD; - } else if (!strcasecmp(key, "write_mostly")) { - if (rs->raid_type->level != 1) { + + if (test_and_set_bit(value, (void *) rs->rebuild_disks)) { + rs->ti->error = "rebuild for this index already given"; + return -EINVAL; + } + + rd = rs->dev + value; + clear_bit(In_sync, &rd->rdev.flags); + clear_bit(Faulty, &rd->rdev.flags); + rd->rdev.recovery_offset = 0; + set_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags); + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_WRITE_MOSTLY))) { + if (!rt_is_raid1(rt)) { rs->ti->error = "write_mostly option is only valid for RAID1"; return -EINVAL; } - if (value >= rs->md.raid_disks) { - rs->ti->error = "Invalid write_mostly drive index given"; + + if (!__within_range(value, 0, rs->md.raid_disks - 1)) { + rs->ti->error = "Invalid write_mostly index given"; return -EINVAL; } + + write_mostly++; set_bit(WriteMostly, &rs->dev[value].rdev.flags); - } else if (!strcasecmp(key, "max_write_behind")) { - if (rs->raid_type->level != 1) { + set_bit(__CTR_FLAG_WRITE_MOSTLY, &rs->ctr_flags); + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_MAX_WRITE_BEHIND))) { + if (!rt_is_raid1(rt)) { rs->ti->error = "max_write_behind option is only valid for RAID1"; return -EINVAL; } - rs->ctr_flags |= CTR_FLAG_MAX_WRITE_BEHIND; + + if (test_and_set_bit(__CTR_FLAG_MAX_WRITE_BEHIND, &rs->ctr_flags)) { + rs->ti->error = "Only one max_write_behind argument pair allowed"; + return -EINVAL; + } /* * In device-mapper, we specify things in sectors, but @@ -628,65 +1209,122 @@ static int parse_raid_params(struct raid_set *rs, char **argv, rs->ti->error = "Max write-behind limit out of range"; return -EINVAL; } + rs->md.bitmap_info.max_write_behind = value; - } else if (!strcasecmp(key, "daemon_sleep")) { - rs->ctr_flags |= CTR_FLAG_DAEMON_SLEEP; + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_DAEMON_SLEEP))) { + if (test_and_set_bit(__CTR_FLAG_DAEMON_SLEEP, &rs->ctr_flags)) { + rs->ti->error = "Only one daemon_sleep argument pair allowed"; + return -EINVAL; + } if (!value || (value > MAX_SCHEDULE_TIMEOUT)) { rs->ti->error = "daemon sleep period out of range"; return -EINVAL; } rs->md.bitmap_info.daemon_sleep = value; - } else if (!strcasecmp(key, "stripe_cache")) { - rs->ctr_flags |= CTR_FLAG_STRIPE_CACHE; + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_DATA_OFFSET))) { + /* Userspace passes new data_offset after having extended the the data image LV */ + if (test_and_set_bit(__CTR_FLAG_DATA_OFFSET, &rs->ctr_flags)) { + rs->ti->error = "Only one data_offset argument pair allowed"; + return -EINVAL; + } + /* Ensure sensible data offset */ + if (value < 0 || + (value && (value < MIN_FREE_RESHAPE_SPACE || value % to_sector(PAGE_SIZE)))) { + rs->ti->error = "Bogus data_offset value"; + return -EINVAL; + } + rs->data_offset = value; + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_DELTA_DISKS))) { + /* Define the +/-# of disks to add to/remove from the given raid set */ + if (test_and_set_bit(__CTR_FLAG_DELTA_DISKS, &rs->ctr_flags)) { + rs->ti->error = "Only one delta_disks argument pair allowed"; + return -EINVAL; + } + /* Ensure MAX_RAID_DEVICES and raid type minimal_devs! */ + if (!__within_range(abs(value), 1, MAX_RAID_DEVICES - rt->minimal_devs)) { + rs->ti->error = "Too many delta_disk requested"; + return -EINVAL; + } - /* - * In device-mapper, we specify things in sectors, but - * MD records this value in kB - */ - value /= 2; + rs->delta_disks = value; + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_STRIPE_CACHE))) { + if (test_and_set_bit(__CTR_FLAG_STRIPE_CACHE, &rs->ctr_flags)) { + rs->ti->error = "Only one stripe_cache argument pair allowed"; + return -EINVAL; + } - if ((rs->raid_type->level != 5) && - (rs->raid_type->level != 6)) { + if (!rt_is_raid456(rt)) { rs->ti->error = "Inappropriate argument: stripe_cache"; return -EINVAL; } - if (raid5_set_cache_size(&rs->md, (int)value)) { - rs->ti->error = "Bad stripe_cache size"; + + rs->stripe_cache_entries = value; + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_MIN_RECOVERY_RATE))) { + if (test_and_set_bit(__CTR_FLAG_MIN_RECOVERY_RATE, &rs->ctr_flags)) { + rs->ti->error = "Only one min_recovery_rate argument pair allowed"; return -EINVAL; } - } else if (!strcasecmp(key, "min_recovery_rate")) { - rs->ctr_flags |= CTR_FLAG_MIN_RECOVERY_RATE; if (value > INT_MAX) { rs->ti->error = "min_recovery_rate out of range"; return -EINVAL; } rs->md.sync_speed_min = (int)value; - } else if (!strcasecmp(key, "max_recovery_rate")) { - rs->ctr_flags |= CTR_FLAG_MAX_RECOVERY_RATE; + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_MAX_RECOVERY_RATE))) { + if (test_and_set_bit(__CTR_FLAG_MIN_RECOVERY_RATE, &rs->ctr_flags)) { + rs->ti->error = "Only one max_recovery_rate argument pair allowed"; + return -EINVAL; + } if (value > INT_MAX) { rs->ti->error = "max_recovery_rate out of range"; return -EINVAL; } rs->md.sync_speed_max = (int)value; - } else if (!strcasecmp(key, "region_size")) { - rs->ctr_flags |= CTR_FLAG_REGION_SIZE; + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_REGION_SIZE))) { + if (test_and_set_bit(__CTR_FLAG_REGION_SIZE, &rs->ctr_flags)) { + rs->ti->error = "Only one region_size argument pair allowed"; + return -EINVAL; + } + region_size = value; - } else if (!strcasecmp(key, "raid10_copies") && - (rs->raid_type->level == 10)) { - if ((value < 2) || (value > 0xFF)) { + rs->requested_bitmap_chunk_sectors = value; + } else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_COPIES))) { + if (test_and_set_bit(__CTR_FLAG_RAID10_COPIES, &rs->ctr_flags)) { + rs->ti->error = "Only one raid10_copies argument pair allowed"; + return -EINVAL; + } + + if (!__within_range(value, 2, rs->md.raid_disks)) { rs->ti->error = "Bad value for 'raid10_copies'"; return -EINVAL; } - rs->ctr_flags |= CTR_FLAG_RAID10_COPIES; + raid10_copies = value; } else { DMERR("Unable to parse RAID parameter: %s", key); - rs->ti->error = "Unable to parse RAID parameters"; + rs->ti->error = "Unable to parse RAID parameter"; return -EINVAL; } } - if (validate_region_size(rs, region_size)) + if (test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags) && + test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)) { + rs->ti->error = "sync and nosync are mutually exclusive"; + return -EINVAL; + } + + if (test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags) && + (test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags) || + test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags))) { + rs->ti->error = "sync/nosync and rebuild are mutually exclusive"; + return -EINVAL; + } + + if (write_mostly >= rs->md.raid_disks) { + rs->ti->error = "Can't set all raid1 devices to write_mostly"; + return -EINVAL; + } + + if (validate_region_size(rs, region_size)) return -EINVAL; if (rs->md.chunk_sectors) @@ -697,47 +1335,193 @@ static int parse_raid_params(struct raid_set *rs, char **argv, if (dm_set_target_max_io_len(rs->ti, max_io_len)) return -EINVAL; - if (rs->raid_type->level == 10) { + if (rt_is_raid10(rt)) { if (raid10_copies > rs->md.raid_disks) { rs->ti->error = "Not enough devices to satisfy specification"; return -EINVAL; } - /* - * If the format is not "near", we only support - * two copies at the moment. - */ - if (strcmp("near", raid10_format) && (raid10_copies > 2)) { - rs->ti->error = "Too many copies for given RAID10 format."; + rs->md.new_layout = raid10_format_to_md_layout(rs, raid10_format, raid10_copies); + if (rs->md.new_layout < 0) { + rs->ti->error = "Error getting raid10 format"; + return rs->md.new_layout; + } + + rt = get_raid_type_by_ll(10, rs->md.new_layout); + if (!rt) { + rs->ti->error = "Failed to recognize new raid10 layout"; return -EINVAL; } - /* (Len * #mirrors) / #devices */ - sectors_per_dev = rs->ti->len * raid10_copies; - sector_div(sectors_per_dev, rs->md.raid_disks); - - rs->md.layout = raid10_format_to_md_layout(raid10_format, - raid10_copies); - rs->md.new_layout = rs->md.layout; - } else if ((!rs->raid_type->level || rs->raid_type->level > 1) && - sector_div(sectors_per_dev, - (rs->md.raid_disks - rs->raid_type->parity_devs))) { - rs->ti->error = "Target length not divisible by number of data devices"; - return -EINVAL; + if ((rt->algorithm == ALGORITHM_RAID10_DEFAULT || + rt->algorithm == ALGORITHM_RAID10_NEAR) && + test_bit(__CTR_FLAG_RAID10_USE_NEAR_SETS, &rs->ctr_flags)) { + rs->ti->error = "RAID10 format 'near' and 'raid10_use_near_sets' are incompatible"; + return -EINVAL; + } } - rs->md.dev_sectors = sectors_per_dev; + + rs->raid10_copies = raid10_copies; /* Assume there are no metadata devices until the drives are parsed */ rs->md.persistent = 0; rs->md.external = 1; + /* Check, if any invalid ctr arguments have been passed in for the raid level */ + return rs_check_for_valid_flags(rs); +} + +/* Set raid4/5/6 cache size */ +static int rs_set_raid456_stripe_cache(struct raid_set *rs) +{ + int r; + struct r5conf *conf; + struct mddev *mddev = &rs->md; + uint32_t min_stripes = max(mddev->chunk_sectors, mddev->new_chunk_sectors) / 2; + uint32_t nr_stripes = rs->stripe_cache_entries; + + if (!rt_is_raid456(rs->raid_type)) { + rs->ti->error = "Inappropriate raid level; cannot change stripe_cache size"; + return -EINVAL; + } + + if (nr_stripes < min_stripes) { + DMINFO("Adjusting requested %u stripe cache entries to %u to suit stripe size", + nr_stripes, min_stripes); + nr_stripes = min_stripes; + } + + conf = mddev->private; + if (!conf) { + rs->ti->error = "Cannot change stripe_cache size on inactive RAID set"; + return -EINVAL; + } + + /* Try setting number of stripes in raid456 stripe cache */ + if (conf->min_nr_stripes != nr_stripes) { + r = raid5_set_cache_size(mddev, nr_stripes); + if (r) { + rs->ti->error = "Failed to set raid4/5/6 stripe cache size"; + return r; + } + + DMINFO("%u stripe cache entries", nr_stripes); + } + return 0; } +/* Return # of data stripes as kept in mddev as of @rs (i.e. as of superblock) */ +static unsigned int mddev_data_stripes(struct raid_set *rs) +{ + return rs->md.raid_disks - rs->raid_type->parity_devs; +} + +/* Return # of data stripes of @rs (i.e. as of ctr) */ +static unsigned int rs_data_stripes(struct raid_set *rs) +{ + return rs->raid_disks - rs->raid_type->parity_devs; +} + +/* Calculate the sectors per device and per array used for @rs */ +static int rs_set_dev_and_array_sectors(struct raid_set *rs, bool use_mddev) +{ + int delta_disks; + unsigned int data_stripes; + struct mddev *mddev = &rs->md; + struct md_rdev *rdev; + sector_t array_sectors = rs->ti->len, dev_sectors = rs->ti->len; + + if (use_mddev) { + delta_disks = mddev->delta_disks; + data_stripes = mddev_data_stripes(rs); + } else { + delta_disks = rs->delta_disks; + data_stripes = rs_data_stripes(rs); + } + + /* Special raid1 case w/o delta_disks support (yet) */ + if (rt_is_raid1(rs->raid_type)) + ; + else if (rt_is_raid10(rs->raid_type)) { + if (rs->raid10_copies < 2 || + delta_disks < 0) { + rs->ti->error = "Bogus raid10 data copies or delta disks"; + return -EINVAL; + } + + dev_sectors *= rs->raid10_copies; + if (sector_div(dev_sectors, data_stripes)) + goto bad; + + array_sectors = (data_stripes + delta_disks) * dev_sectors; + if (sector_div(array_sectors, rs->raid10_copies)) + goto bad; + + } else if (sector_div(dev_sectors, data_stripes)) + goto bad; + + else + /* Striped layouts */ + array_sectors = (data_stripes + delta_disks) * dev_sectors; + + rdev_for_each(rdev, mddev) + rdev->sectors = dev_sectors; + + mddev->array_sectors = array_sectors; + mddev->dev_sectors = dev_sectors; + + return 0; +bad: + rs->ti->error = "Target length not divisible by number of data devices"; + return -EINVAL; +} + +/* Setup recovery on @rs */ +static void __rs_setup_recovery(struct raid_set *rs, sector_t dev_sectors) +{ + /* raid0 does not recover */ + if (rs_is_raid0(rs)) + rs->md.recovery_cp = MaxSector; + /* + * A raid6 set has to be recovered either + * completely or for the grown part to + * ensure proper parity and Q-Syndrome + */ + else if (rs_is_raid6(rs)) + rs->md.recovery_cp = dev_sectors; + /* + * Other raid set types may skip recovery + * depending on the 'nosync' flag. + */ + else + rs->md.recovery_cp = test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags) + ? MaxSector : dev_sectors; +} + +/* Setup recovery on @rs based on raid type, device size and 'nosync' flag */ +static void rs_setup_recovery(struct raid_set *rs, sector_t dev_sectors) +{ + if (!dev_sectors) + /* New raid set or 'sync' flag provided */ + __rs_setup_recovery(rs, 0); + else if (dev_sectors == MaxSector) + /* Prevent recovery */ + __rs_setup_recovery(rs, MaxSector); + else if (rs->dev[0].rdev.sectors < dev_sectors) + /* Grown raid set */ + __rs_setup_recovery(rs, rs->dev[0].rdev.sectors); + else + __rs_setup_recovery(rs, MaxSector); +} + static void do_table_event(struct work_struct *ws) { struct raid_set *rs = container_of(ws, struct raid_set, md.event_work); + smp_rmb(); /* Make sure we access most actual mddev properties */ + if (!rs_is_reshaping(rs)) + rs_set_capacity(rs); dm_table_event(rs->ti->table); } @@ -748,6 +1532,211 @@ static int raid_is_congested(struct dm_target_callbacks *cb, int bits) return mddev_congested(&rs->md, bits); } +/* + * Make sure a valid takover (level switch) is being requested on @rs + * + * Conversions of raid sets from one MD personality to another + * have to conform to restrictions which are enforced here. + */ +static int rs_check_takeover(struct raid_set *rs) +{ + struct mddev *mddev = &rs->md; + unsigned int near_copies; + + if (rs->md.degraded) { + rs->ti->error = "Can't takeover degraded raid set"; + return -EPERM; + } + + if (rs_is_reshaping(rs)) { + rs->ti->error = "Can't takeover reshaping raid set"; + return -EPERM; + } + + switch (mddev->level) { + case 0: + /* raid0 -> raid1/5 with one disk */ + if ((mddev->new_level == 1 || mddev->new_level == 5) && + mddev->raid_disks == 1) + return 0; + + /* raid0 -> raid10 */ + if (mddev->new_level == 10 && + !(rs->raid_disks % mddev->raid_disks)) + return 0; + + /* raid0 with multiple disks -> raid4/5/6 */ + if (__within_range(mddev->new_level, 4, 6) && + mddev->new_layout == ALGORITHM_PARITY_N && + mddev->raid_disks > 1) + return 0; + + break; + + case 10: + /* Can't takeover raid10_offset! */ + if (__is_raid10_offset(mddev->layout)) + break; + + near_copies = __raid10_near_copies(mddev->layout); + + /* raid10* -> raid0 */ + if (mddev->new_level == 0) { + /* Can takeover raid10_near with raid disks divisable by data copies! */ + if (near_copies > 1 && + !(mddev->raid_disks % near_copies)) { + mddev->raid_disks /= near_copies; + mddev->delta_disks = mddev->raid_disks; + return 0; + } + + /* Can takeover raid10_far */ + if (near_copies == 1 && + __raid10_far_copies(mddev->layout) > 1) + return 0; + + break; + } + + /* raid10_{near,far} -> raid1 */ + if (mddev->new_level == 1 && + max(near_copies, __raid10_far_copies(mddev->layout)) == mddev->raid_disks) + return 0; + + /* raid10_{near,far} with 2 disks -> raid4/5 */ + if (__within_range(mddev->new_level, 4, 5) && + mddev->raid_disks == 2) + return 0; + break; + + case 1: + /* raid1 with 2 disks -> raid4/5 */ + if (__within_range(mddev->new_level, 4, 5) && + mddev->raid_disks == 2) { + mddev->degraded = 1; + return 0; + } + + /* raid1 -> raid0 */ + if (mddev->new_level == 0 && + mddev->raid_disks == 1) + return 0; + + /* raid1 -> raid10 */ + if (mddev->new_level == 10) + return 0; + break; + + case 4: + /* raid4 -> raid0 */ + if (mddev->new_level == 0) + return 0; + + /* raid4 -> raid1/5 with 2 disks */ + if ((mddev->new_level == 1 || mddev->new_level == 5) && + mddev->raid_disks == 2) + return 0; + + /* raid4 -> raid5/6 with parity N */ + if (__within_range(mddev->new_level, 5, 6) && + mddev->layout == ALGORITHM_PARITY_N) + return 0; + break; + + case 5: + /* raid5 with parity N -> raid0 */ + if (mddev->new_level == 0 && + mddev->layout == ALGORITHM_PARITY_N) + return 0; + + /* raid5 with parity N -> raid4 */ + if (mddev->new_level == 4 && + mddev->layout == ALGORITHM_PARITY_N) + return 0; + + /* raid5 with 2 disks -> raid1/4/10 */ + if ((mddev->new_level == 1 || mddev->new_level == 4 || mddev->new_level == 10) && + mddev->raid_disks == 2) + return 0; + + /* raid5_* -> raid6_*_6 with Q-Syndrome N (e.g. raid5_ra -> raid6_ra_6 */ + if (mddev->new_level == 6 && + ((mddev->layout == ALGORITHM_PARITY_N && mddev->new_layout == ALGORITHM_PARITY_N) || + __within_range(mddev->new_layout, ALGORITHM_LEFT_ASYMMETRIC_6, ALGORITHM_RIGHT_SYMMETRIC_6))) + return 0; + break; + + case 6: + /* raid6 with parity N -> raid0 */ + if (mddev->new_level == 0 && + mddev->layout == ALGORITHM_PARITY_N) + return 0; + + /* raid6 with parity N -> raid4 */ + if (mddev->new_level == 4 && + mddev->layout == ALGORITHM_PARITY_N) + return 0; + + /* raid6_*_n with Q-Syndrome N -> raid5_* */ + if (mddev->new_level == 5 && + ((mddev->layout == ALGORITHM_PARITY_N && mddev->new_layout == ALGORITHM_PARITY_N) || + __within_range(mddev->new_layout, ALGORITHM_LEFT_ASYMMETRIC, ALGORITHM_RIGHT_SYMMETRIC))) + return 0; + + default: + break; + } + + rs->ti->error = "takeover not possible"; + return -EINVAL; +} + +/* True if @rs requested to be taken over */ +static bool rs_takeover_requested(struct raid_set *rs) +{ + return rs->md.new_level != rs->md.level; +} + +/* True if @rs is requested to reshape by ctr */ +static bool rs_reshape_requested(struct raid_set *rs) +{ + bool change; + struct mddev *mddev = &rs->md; + + if (rs_takeover_requested(rs)) + return false; + + if (!mddev->level) + return false; + + change = mddev->new_layout != mddev->layout || + mddev->new_chunk_sectors != mddev->chunk_sectors || + rs->delta_disks; + + /* Historical case to support raid1 reshape without delta disks */ + if (mddev->level == 1) { + if (rs->delta_disks) + return !!rs->delta_disks; + + return !change && + mddev->raid_disks != rs->raid_disks; + } + + if (mddev->level == 10) + return change && + !__is_raid10_far(mddev->new_layout) && + rs->delta_disks >= 0; + + return change; +} + +/* Features */ +#define FEATURE_FLAG_SUPPORTS_V190 0x1 /* Supports extended superblock */ + +/* State flags for sb->flags */ +#define SB_FLAG_RESHAPE_ACTIVE 0x1 +#define SB_FLAG_RESHAPE_BACKWARDS 0x2 + /* * This structure is never routinely used by userspace, unlike md superblocks. * Devices with this superblock should only ever be accessed via device-mapper. @@ -755,13 +1744,14 @@ static int raid_is_congested(struct dm_target_callbacks *cb, int bits) #define DM_RAID_MAGIC 0x64526D44 struct dm_raid_superblock { __le32 magic; /* "DmRd" */ - __le32 features; /* Used to indicate possible future changes */ + __le32 compat_features; /* Used to indicate compatible features (like 1.9.0 ondisk metadata extension) */ - __le32 num_devices; /* Number of devices in this array. (Max 64) */ - __le32 array_position; /* The position of this drive in the array */ + __le32 num_devices; /* Number of devices in this raid set. (Max 64) */ + __le32 array_position; /* The position of this drive in the raid set */ __le64 events; /* Incremented by md when superblock updated */ - __le64 failed_devices; /* Bit field of devices to indicate failures */ + __le64 failed_devices; /* Pre 1.9.0 part of bit field of devices to */ + /* indicate failures (see extension below) */ /* * This offset tracks the progress of the repair or replacement of @@ -770,21 +1760,95 @@ struct dm_raid_superblock { __le64 disk_recovery_offset; /* - * This offset tracks the progress of the initial array + * This offset tracks the progress of the initial raid set * synchronisation/parity calculation. */ __le64 array_resync_offset; /* - * RAID characteristics + * raid characteristics */ __le32 level; __le32 layout; __le32 stripe_sectors; - /* Remainder of a logical block is zero-filled when writing (see super_sync()). */ + /******************************************************************** + * BELOW FOLLOW V1.9.0 EXTENSIONS TO THE PRISTINE SUPERBLOCK FORMAT!!! + * + * FEATURE_FLAG_SUPPORTS_V190 in the features member indicates that those exist + */ + + __le32 flags; /* Flags defining array states for reshaping */ + + /* + * This offset tracks the progress of a raid + * set reshape in order to be able to restart it + */ + __le64 reshape_position; + + /* + * These define the properties of the array in case of an interrupted reshape + */ + __le32 new_level; + __le32 new_layout; + __le32 new_stripe_sectors; + __le32 delta_disks; + + __le64 array_sectors; /* Array size in sectors */ + + /* + * Sector offsets to data on devices (reshaping). + * Needed to support out of place reshaping, thus + * not writing over any stripes whilst converting + * them from old to new layout + */ + __le64 data_offset; + __le64 new_data_offset; + + __le64 sectors; /* Used device size in sectors */ + + /* + * Additonal Bit field of devices indicating failures to support + * up to 256 devices with the 1.9.0 on-disk metadata format + */ + __le64 extended_failed_devices[DISKS_ARRAY_ELEMS - 1]; + + __le32 incompat_features; /* Used to indicate any incompatible features */ + + /* Always set rest up to logical block size to 0 when writing (see get_metadata_device() below). */ } __packed; +/* + * Check for reshape constraints on raid set @rs: + * + * - reshape function non-existent + * - degraded set + * - ongoing recovery + * - ongoing reshape + * + * Returns 0 if none or -EPERM if given constraint + * and error message reference in @errmsg + */ +static int rs_check_reshape(struct raid_set *rs) +{ + struct mddev *mddev = &rs->md; + + if (!mddev->pers || !mddev->pers->check_reshape) + rs->ti->error = "Reshape not supported"; + else if (mddev->degraded) + rs->ti->error = "Can't reshape degraded raid set"; + else if (rs_is_recovering(rs)) + rs->ti->error = "Convert request on recovering raid set prohibited"; + else if (rs_is_reshaping(rs)) + rs->ti->error = "raid set already reshaping!"; + else if (!(rs_is_raid1(rs) || rs_is_raid10(rs) || rs_is_raid456(rs))) + rs->ti->error = "Reshaping only supported for raid1/4/5/6/10"; + else + return 0; + + return -EPERM; +} + static int read_disk_sb(struct md_rdev *rdev, int size) { BUG_ON(!rdev->sb_page); @@ -792,7 +1856,7 @@ static int read_disk_sb(struct md_rdev *rdev, int size) if (rdev->sb_loaded) return 0; - if (!sync_page_io(rdev, 0, size, rdev->sb_page, READ, 1)) { + if (!sync_page_io(rdev, 0, size, rdev->sb_page, REQ_OP_READ, 0, true)) { DMERR("Failed to read superblock of device at position %d", rdev->raid_disk); md_error(rdev->mddev, rdev); @@ -804,31 +1868,67 @@ static int read_disk_sb(struct md_rdev *rdev, int size) return 0; } +static void sb_retrieve_failed_devices(struct dm_raid_superblock *sb, uint64_t *failed_devices) +{ + failed_devices[0] = le64_to_cpu(sb->failed_devices); + memset(failed_devices + 1, 0, sizeof(sb->extended_failed_devices)); + + if (le32_to_cpu(sb->compat_features) & FEATURE_FLAG_SUPPORTS_V190) { + int i = ARRAY_SIZE(sb->extended_failed_devices); + + while (i--) + failed_devices[i+1] = le64_to_cpu(sb->extended_failed_devices[i]); + } +} + +static void sb_update_failed_devices(struct dm_raid_superblock *sb, uint64_t *failed_devices) +{ + int i = ARRAY_SIZE(sb->extended_failed_devices); + + sb->failed_devices = cpu_to_le64(failed_devices[0]); + while (i--) + sb->extended_failed_devices[i] = cpu_to_le64(failed_devices[i+1]); +} + +/* + * Synchronize the superblock members with the raid set properties + * + * All superblock data is little endian. + */ static void super_sync(struct mddev *mddev, struct md_rdev *rdev) { - int i; - uint64_t failed_devices; + bool update_failed_devices = false; + unsigned int i; + uint64_t failed_devices[DISKS_ARRAY_ELEMS]; struct dm_raid_superblock *sb; struct raid_set *rs = container_of(mddev, struct raid_set, md); + /* No metadata device, no superblock */ + if (!rdev->meta_bdev) + return; + + BUG_ON(!rdev->sb_page); + sb = page_address(rdev->sb_page); - failed_devices = le64_to_cpu(sb->failed_devices); - for (i = 0; i < mddev->raid_disks; i++) - if (!rs->dev[i].data_dev || - test_bit(Faulty, &(rs->dev[i].rdev.flags))) - failed_devices |= (1ULL << i); + sb_retrieve_failed_devices(sb, failed_devices); - memset(sb + 1, 0, rdev->sb_size - sizeof(*sb)); + for (i = 0; i < rs->raid_disks; i++) + if (!rs->dev[i].data_dev || test_bit(Faulty, &rs->dev[i].rdev.flags)) { + update_failed_devices = true; + set_bit(i, (void *) failed_devices); + } + + if (update_failed_devices) + sb_update_failed_devices(sb, failed_devices); sb->magic = cpu_to_le32(DM_RAID_MAGIC); - sb->features = cpu_to_le32(0); /* No features yet */ + sb->compat_features = cpu_to_le32(FEATURE_FLAG_SUPPORTS_V190); sb->num_devices = cpu_to_le32(mddev->raid_disks); sb->array_position = cpu_to_le32(rdev->raid_disk); sb->events = cpu_to_le64(mddev->events); - sb->failed_devices = cpu_to_le64(failed_devices); sb->disk_recovery_offset = cpu_to_le64(rdev->recovery_offset); sb->array_resync_offset = cpu_to_le64(mddev->recovery_cp); @@ -836,6 +1936,33 @@ static void super_sync(struct mddev *mddev, struct md_rdev *rdev) sb->level = cpu_to_le32(mddev->level); sb->layout = cpu_to_le32(mddev->layout); sb->stripe_sectors = cpu_to_le32(mddev->chunk_sectors); + + sb->new_level = cpu_to_le32(mddev->new_level); + sb->new_layout = cpu_to_le32(mddev->new_layout); + sb->new_stripe_sectors = cpu_to_le32(mddev->new_chunk_sectors); + + sb->delta_disks = cpu_to_le32(mddev->delta_disks); + + smp_rmb(); /* Make sure we access most recent reshape position */ + sb->reshape_position = cpu_to_le64(mddev->reshape_position); + if (le64_to_cpu(sb->reshape_position) != MaxSector) { + /* Flag ongoing reshape */ + sb->flags |= cpu_to_le32(SB_FLAG_RESHAPE_ACTIVE); + + if (mddev->delta_disks < 0 || mddev->reshape_backwards) + sb->flags |= cpu_to_le32(SB_FLAG_RESHAPE_BACKWARDS); + } else { + /* Clear reshape flags */ + sb->flags &= ~(cpu_to_le32(SB_FLAG_RESHAPE_ACTIVE|SB_FLAG_RESHAPE_BACKWARDS)); + } + + sb->array_sectors = cpu_to_le64(mddev->array_sectors); + sb->data_offset = cpu_to_le64(rdev->data_offset); + sb->new_data_offset = cpu_to_le64(rdev->new_data_offset); + sb->sectors = cpu_to_le64(rdev->sectors); + + /* Zero out the rest of the payload after the size of the superblock */ + memset(sb + 1, 0, rdev->sb_size - sizeof(*sb)); } /* @@ -848,7 +1975,7 @@ static void super_sync(struct mddev *mddev, struct md_rdev *rdev) */ static int super_load(struct md_rdev *rdev, struct md_rdev *refdev) { - int ret; + int r; struct dm_raid_superblock *sb; struct dm_raid_superblock *refsb; uint64_t events_sb, events_refsb; @@ -860,9 +1987,9 @@ static int super_load(struct md_rdev *rdev, struct md_rdev *refdev) return -EINVAL; } - ret = read_disk_sb(rdev, rdev->sb_size); - if (ret) - return ret; + r = read_disk_sb(rdev, rdev->sb_size); + if (r) + return r; sb = page_address(rdev->sb_page); @@ -876,6 +2003,7 @@ static int super_load(struct md_rdev *rdev, struct md_rdev *refdev) super_sync(rdev->mddev, rdev); set_bit(FirstUse, &rdev->flags); + sb->compat_features = cpu_to_le32(FEATURE_FLAG_SUPPORTS_V190); /* Force writing of superblocks to disk */ set_bit(MD_CHANGE_DEVS, &rdev->mddev->flags); @@ -895,129 +2023,212 @@ static int super_load(struct md_rdev *rdev, struct md_rdev *refdev) return (events_sb > events_refsb) ? 1 : 0; } -static int super_init_validation(struct mddev *mddev, struct md_rdev *rdev) +static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) { int role; - struct raid_set *rs = container_of(mddev, struct raid_set, md); + unsigned int d; + struct mddev *mddev = &rs->md; uint64_t events_sb; - uint64_t failed_devices; + uint64_t failed_devices[DISKS_ARRAY_ELEMS]; struct dm_raid_superblock *sb; - uint32_t new_devs = 0; - uint32_t rebuilds = 0; + uint32_t new_devs = 0, rebuild_and_new = 0, rebuilds = 0; struct md_rdev *r; struct dm_raid_superblock *sb2; sb = page_address(rdev->sb_page); events_sb = le64_to_cpu(sb->events); - failed_devices = le64_to_cpu(sb->failed_devices); /* * Initialise to 1 if this is a new superblock. */ mddev->events = events_sb ? : 1; + mddev->reshape_position = MaxSector; + /* - * Reshaping is not currently allowed + * Reshaping is supported, e.g. reshape_position is valid + * in superblock and superblock content is authoritative. */ - if (le32_to_cpu(sb->level) != mddev->level) { - DMERR("Reshaping arrays not yet supported. (RAID level change)"); - return -EINVAL; - } - if (le32_to_cpu(sb->layout) != mddev->layout) { - DMERR("Reshaping arrays not yet supported. (RAID layout change)"); - DMERR(" 0x%X vs 0x%X", le32_to_cpu(sb->layout), mddev->layout); - DMERR(" Old layout: %s w/ %d copies", - raid10_md_layout_to_format(le32_to_cpu(sb->layout)), - raid10_md_layout_to_copies(le32_to_cpu(sb->layout))); - DMERR(" New layout: %s w/ %d copies", - raid10_md_layout_to_format(mddev->layout), - raid10_md_layout_to_copies(mddev->layout)); - return -EINVAL; - } - if (le32_to_cpu(sb->stripe_sectors) != mddev->chunk_sectors) { - DMERR("Reshaping arrays not yet supported. (stripe sectors change)"); - return -EINVAL; - } + if (le32_to_cpu(sb->compat_features) & FEATURE_FLAG_SUPPORTS_V190) { + /* Superblock is authoritative wrt given raid set layout! */ + mddev->raid_disks = le32_to_cpu(sb->num_devices); + mddev->level = le32_to_cpu(sb->level); + mddev->layout = le32_to_cpu(sb->layout); + mddev->chunk_sectors = le32_to_cpu(sb->stripe_sectors); + mddev->new_level = le32_to_cpu(sb->new_level); + mddev->new_layout = le32_to_cpu(sb->new_layout); + mddev->new_chunk_sectors = le32_to_cpu(sb->new_stripe_sectors); + mddev->delta_disks = le32_to_cpu(sb->delta_disks); + mddev->array_sectors = le64_to_cpu(sb->array_sectors); + + /* raid was reshaping and got interrupted */ + if (le32_to_cpu(sb->flags) & SB_FLAG_RESHAPE_ACTIVE) { + if (test_bit(__CTR_FLAG_DELTA_DISKS, &rs->ctr_flags)) { + DMERR("Reshape requested but raid set is still reshaping"); + return -EINVAL; + } - /* We can only change the number of devices in RAID1 right now */ - if ((rs->raid_type->level != 1) && - (le32_to_cpu(sb->num_devices) != mddev->raid_disks)) { - DMERR("Reshaping arrays not yet supported. (device count change)"); - return -EINVAL; + if (mddev->delta_disks < 0 || + (!mddev->delta_disks && (le32_to_cpu(sb->flags) & SB_FLAG_RESHAPE_BACKWARDS))) + mddev->reshape_backwards = 1; + else + mddev->reshape_backwards = 0; + + mddev->reshape_position = le64_to_cpu(sb->reshape_position); + rs->raid_type = get_raid_type_by_ll(mddev->level, mddev->layout); + } + + } else { + /* + * No takeover/reshaping, because we don't have the extended v1.9.0 metadata + */ + if (le32_to_cpu(sb->level) != mddev->level) { + DMERR("Reshaping/takeover raid sets not yet supported. (raid level/stripes/size change)"); + return -EINVAL; + } + if (le32_to_cpu(sb->layout) != mddev->layout) { + DMERR("Reshaping raid sets not yet supported. (raid layout change)"); + DMERR(" 0x%X vs 0x%X", le32_to_cpu(sb->layout), mddev->layout); + DMERR(" Old layout: %s w/ %d copies", + raid10_md_layout_to_format(le32_to_cpu(sb->layout)), + raid10_md_layout_to_copies(le32_to_cpu(sb->layout))); + DMERR(" New layout: %s w/ %d copies", + raid10_md_layout_to_format(mddev->layout), + raid10_md_layout_to_copies(mddev->layout)); + return -EINVAL; + } + if (le32_to_cpu(sb->stripe_sectors) != mddev->chunk_sectors) { + DMERR("Reshaping raid sets not yet supported. (stripe sectors change)"); + return -EINVAL; + } + + /* We can only change the number of devices in raid1 with old (i.e. pre 1.0.7) metadata */ + if (!rt_is_raid1(rs->raid_type) && + (le32_to_cpu(sb->num_devices) != mddev->raid_disks)) { + DMERR("Reshaping raid sets not yet supported. (device count change from %u to %u)", + sb->num_devices, mddev->raid_disks); + return -EINVAL; + } + + /* Table line is checked vs. authoritative superblock */ + rs_set_new(rs); } - if (!(rs->ctr_flags & (CTR_FLAG_SYNC | CTR_FLAG_NOSYNC))) + if (!test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)) mddev->recovery_cp = le64_to_cpu(sb->array_resync_offset); /* * During load, we set FirstUse if a new superblock was written. * There are two reasons we might not have a superblock: - * 1) The array is brand new - in which case, all of the - * devices must have their In_sync bit set. Also, + * 1) The raid set is brand new - in which case, all of the + * devices must have their In_sync bit set. Also, * recovery_cp must be 0, unless forced. - * 2) This is a new device being added to an old array + * 2) This is a new device being added to an old raid set * and the new device needs to be rebuilt - in which * case the In_sync bit will /not/ be set and * recovery_cp must be MaxSector. + * 3) This is/are a new device(s) being added to an old + * raid set during takeover to a higher raid level + * to provide capacity for redundancy or during reshape + * to add capacity to grow the raid set. */ + d = 0; rdev_for_each(r, mddev) { + if (test_bit(FirstUse, &r->flags)) + new_devs++; + if (!test_bit(In_sync, &r->flags)) { - DMINFO("Device %d specified for rebuild: " - "Clearing superblock", r->raid_disk); + DMINFO("Device %d specified for rebuild; clearing superblock", + r->raid_disk); rebuilds++; - } else if (test_bit(FirstUse, &r->flags)) - new_devs++; + + if (test_bit(FirstUse, &r->flags)) + rebuild_and_new++; + } + + d++; } - if (!rebuilds) { - if (new_devs == mddev->raid_disks) { - DMINFO("Superblocks created for new array"); + if (new_devs == rs->raid_disks || !rebuilds) { + /* Replace a broken device */ + if (new_devs == 1 && !rs->delta_disks) + ; + if (new_devs == rs->raid_disks) { + DMINFO("Superblocks created for new raid set"); set_bit(MD_ARRAY_FIRST_USE, &mddev->flags); - } else if (new_devs) { - DMERR("New device injected " - "into existing array without 'rebuild' " - "parameter specified"); + } else if (new_devs != rebuilds && + new_devs != rs->delta_disks) { + DMERR("New device injected into existing raid set without " + "'delta_disks' or 'rebuild' parameter specified"); return -EINVAL; } - } else if (new_devs) { - DMERR("'rebuild' devices cannot be " - "injected into an array with other first-time devices"); - return -EINVAL; - } else if (mddev->recovery_cp != MaxSector) { - DMERR("'rebuild' specified while array is not in-sync"); + } else if (new_devs && new_devs != rebuilds) { + DMERR("%u 'rebuild' devices cannot be injected into" + " a raid set with %u other first-time devices", + rebuilds, new_devs); return -EINVAL; + } else if (rebuilds) { + if (rebuild_and_new && rebuilds != rebuild_and_new) { + DMERR("new device%s provided without 'rebuild'", + new_devs > 1 ? "s" : ""); + return -EINVAL; + } else if (rs_is_recovering(rs)) { + DMERR("'rebuild' specified while raid set is not in-sync (recovery_cp=%llu)", + (unsigned long long) mddev->recovery_cp); + return -EINVAL; + } else if (rs_is_reshaping(rs)) { + DMERR("'rebuild' specified while raid set is being reshaped (reshape_position=%llu)", + (unsigned long long) mddev->reshape_position); + return -EINVAL; + } } /* * Now we set the Faulty bit for those devices that are * recorded in the superblock as failed. */ + sb_retrieve_failed_devices(sb, failed_devices); rdev_for_each(r, mddev) { if (!r->sb_page) continue; sb2 = page_address(r->sb_page); sb2->failed_devices = 0; + memset(sb2->extended_failed_devices, 0, sizeof(sb2->extended_failed_devices)); /* * Check for any device re-ordering. */ if (!test_bit(FirstUse, &r->flags) && (r->raid_disk >= 0)) { role = le32_to_cpu(sb2->array_position); + if (role < 0) + continue; + if (role != r->raid_disk) { - if (rs->raid_type->level != 1) { - rs->ti->error = "Cannot change device " - "positions in RAID array"; + if (__is_raid10_near(mddev->layout)) { + if (mddev->raid_disks % __raid10_near_copies(mddev->layout) || + rs->raid_disks % rs->raid10_copies) { + rs->ti->error = + "Cannot change raid10 near set to odd # of devices!"; + return -EINVAL; + } + + sb2->array_position = cpu_to_le32(r->raid_disk); + + } else if (!(rs_is_raid10(rs) && rt_is_raid0(rs->raid_type)) && + !(rs_is_raid0(rs) && rt_is_raid10(rs->raid_type)) && + !rt_is_raid1(rs->raid_type)) { + rs->ti->error = "Cannot change device positions in raid set"; return -EINVAL; } - DMINFO("RAID1 device #%d now at position #%d", - role, r->raid_disk); + + DMINFO("raid device #%d now at position #%d", role, r->raid_disk); } /* * Partial recovery is performed on * returning failed devices. */ - if (failed_devices & (1 << role)) + if (test_bit(role, (void *) failed_devices)) set_bit(Faulty, &r->flags); } } @@ -1028,41 +2239,60 @@ static int super_init_validation(struct mddev *mddev, struct md_rdev *rdev) static int super_validate(struct raid_set *rs, struct md_rdev *rdev) { struct mddev *mddev = &rs->md; - struct dm_raid_superblock *sb = page_address(rdev->sb_page); + struct dm_raid_superblock *sb; + + if (rs_is_raid0(rs) || !rdev->sb_page) + return 0; + + sb = page_address(rdev->sb_page); /* * If mddev->events is not set, we know we have not yet initialized * the array. */ - if (!mddev->events && super_init_validation(mddev, rdev)) + if (!mddev->events && super_init_validation(rs, rdev)) + return -EINVAL; + + if (le32_to_cpu(sb->compat_features) != FEATURE_FLAG_SUPPORTS_V190) { + rs->ti->error = "Unable to assemble array: Unknown flag(s) in compatible feature flags"; return -EINVAL; + } - if (le32_to_cpu(sb->features)) { - rs->ti->error = "Unable to assemble array: No feature flags supported yet"; + if (sb->incompat_features) { + rs->ti->error = "Unable to assemble array: No incompatible feature flags supported yet"; return -EINVAL; } /* Enable bitmap creation for RAID levels != 0 */ - mddev->bitmap_info.offset = (rs->raid_type->level) ? to_sector(4096) : 0; + mddev->bitmap_info.offset = rt_is_raid0(rs->raid_type) ? 0 : to_sector(4096); rdev->mddev->bitmap_info.default_offset = mddev->bitmap_info.offset; - if (!test_bit(FirstUse, &rdev->flags)) { + if (!test_and_clear_bit(FirstUse, &rdev->flags)) { + /* Retrieve device size stored in superblock to be prepared for shrink */ + rdev->sectors = le64_to_cpu(sb->sectors); rdev->recovery_offset = le64_to_cpu(sb->disk_recovery_offset); - if (rdev->recovery_offset != MaxSector) - clear_bit(In_sync, &rdev->flags); + if (rdev->recovery_offset == MaxSector) + set_bit(In_sync, &rdev->flags); + /* + * If no reshape in progress -> we're recovering single + * disk(s) and have to set the device(s) to out-of-sync + */ + else if (!rs_is_reshaping(rs)) + clear_bit(In_sync, &rdev->flags); /* Mandatory for recovery */ } /* * If a device comes back, set it as not In_sync and no longer faulty. */ - if (test_bit(Faulty, &rdev->flags)) { - clear_bit(Faulty, &rdev->flags); + if (test_and_clear_bit(Faulty, &rdev->flags)) { + rdev->recovery_offset = 0; clear_bit(In_sync, &rdev->flags); rdev->saved_raid_disk = rdev->raid_disk; - rdev->recovery_offset = 0; } - clear_bit(FirstUse, &rdev->flags); + /* Reshape support -> restore repective data offsets */ + rdev->data_offset = le64_to_cpu(sb->data_offset); + rdev->new_data_offset = le64_to_cpu(sb->new_data_offset); return 0; } @@ -1072,7 +2302,7 @@ static int super_validate(struct raid_set *rs, struct md_rdev *rdev) */ static int analyse_superblocks(struct dm_target *ti, struct raid_set *rs) { - int ret; + int r; struct raid_dev *dev; struct md_rdev *rdev, *tmp, *freshest; struct mddev *mddev = &rs->md; @@ -1082,24 +2312,22 @@ static int analyse_superblocks(struct dm_target *ti, struct raid_set *rs) /* * Skipping super_load due to CTR_FLAG_SYNC will cause * the array to undergo initialization again as - * though it were new. This is the intended effect + * though it were new. This is the intended effect * of the "sync" directive. * * When reshaping capability is added, we must ensure * that the "sync" directive is disallowed during the * reshape. */ - rdev->sectors = to_sector(i_size_read(rdev->bdev->bd_inode)); - - if (rs->ctr_flags & CTR_FLAG_SYNC) + if (test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags)) continue; if (!rdev->meta_bdev) continue; - ret = super_load(rdev, freshest); + r = super_load(rdev, freshest); - switch (ret) { + switch (r) { case 1: freshest = rdev; break; @@ -1116,57 +2344,368 @@ static int analyse_superblocks(struct dm_target *ti, struct raid_set *rs) if (rdev->sb_page) put_page(rdev->sb_page); - rdev->sb_page = NULL; + rdev->sb_page = NULL; + + rdev->sb_loaded = 0; + + /* + * We might be able to salvage the data device + * even though the meta device has failed. For + * now, we behave as though '- -' had been + * set for this device in the table. + */ + if (dev->data_dev) + dm_put_device(ti, dev->data_dev); + + dev->data_dev = NULL; + rdev->bdev = NULL; + + list_del(&rdev->same_set); + } + } + + if (!freshest) + return 0; + + if (validate_raid_redundancy(rs)) { + rs->ti->error = "Insufficient redundancy to activate array"; + return -EINVAL; + } + + /* + * Validation of the freshest device provides the source of + * validation for the remaining devices. + */ + rs->ti->error = "Unable to assemble array: Invalid superblocks"; + if (super_validate(rs, freshest)) + return -EINVAL; + + rdev_for_each(rdev, mddev) + if ((rdev != freshest) && super_validate(rs, rdev)) + return -EINVAL; + return 0; +} + +/* + * Adjust data_offset and new_data_offset on all disk members of @rs + * for out of place reshaping if requested by contructor + * + * We need free space at the beginning of each raid disk for forward + * and at the end for backward reshapes which userspace has to provide + * via remapping/reordering of space. + */ +static int rs_adjust_data_offsets(struct raid_set *rs) +{ + sector_t data_offset = 0, new_data_offset = 0; + struct md_rdev *rdev; + + /* Constructor did not request data offset change */ + if (!test_bit(__CTR_FLAG_DATA_OFFSET, &rs->ctr_flags)) { + if (!rs_is_reshapable(rs)) + goto out; + + return 0; + } + + /* HM FIXME: get InSync raid_dev? */ + rdev = &rs->dev[0].rdev; + + if (rs->delta_disks < 0) { + /* + * Removing disks (reshaping backwards): + * + * - before reshape: data is at offset 0 and free space + * is at end of each component LV + * + * - after reshape: data is at offset rs->data_offset != 0 on each component LV + */ + data_offset = 0; + new_data_offset = rs->data_offset; + + } else if (rs->delta_disks > 0) { + /* + * Adding disks (reshaping forwards): + * + * - before reshape: data is at offset rs->data_offset != 0 and + * free space is at begin of each component LV + * + * - after reshape: data is at offset 0 on each component LV + */ + data_offset = rs->data_offset; + new_data_offset = 0; + + } else { + /* + * User space passes in 0 for data offset after having removed reshape space + * + * - or - (data offset != 0) + * + * Changing RAID layout or chunk size -> toggle offsets + * + * - before reshape: data is at offset rs->data_offset 0 and + * free space is at end of each component LV + * -or- + * data is at offset rs->data_offset != 0 and + * free space is at begin of each component LV + * + * - after reshape: data is at offset 0 if it was at offset != 0 + * or at offset != 0 if it was at offset 0 + * on each component LV + * + */ + data_offset = rs->data_offset ? rdev->data_offset : 0; + new_data_offset = data_offset ? 0 : rs->data_offset; + set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags); + } + + /* + * Make sure we got a minimum amount of free sectors per device + */ + if (rs->data_offset && + to_sector(i_size_read(rdev->bdev->bd_inode)) - rdev->sectors < MIN_FREE_RESHAPE_SPACE) { + rs->ti->error = data_offset ? "No space for forward reshape" : + "No space for backward reshape"; + return -ENOSPC; + } +out: + /* Adjust data offsets on all rdevs */ + rdev_for_each(rdev, &rs->md) { + rdev->data_offset = data_offset; + rdev->new_data_offset = new_data_offset; + } + + return 0; +} + +/* Userpace reordered disks -> adjust raid_disk indexes in @rs */ +static void __reorder_raid_disk_indexes(struct raid_set *rs) +{ + int i = 0; + struct md_rdev *rdev; + + rdev_for_each(rdev, &rs->md) { + rdev->raid_disk = i++; + rdev->saved_raid_disk = rdev->new_raid_disk = -1; + } +} + +/* + * Setup @rs for takeover by a different raid level + */ +static int rs_setup_takeover(struct raid_set *rs) +{ + struct mddev *mddev = &rs->md; + struct md_rdev *rdev; + unsigned int d = mddev->raid_disks = rs->raid_disks; + sector_t new_data_offset = rs->dev[0].rdev.data_offset ? 0 : rs->data_offset; + + if (rt_is_raid10(rs->raid_type)) { + if (mddev->level == 0) { + /* Userpace reordered disks -> adjust raid_disk indexes */ + __reorder_raid_disk_indexes(rs); + + /* raid0 -> raid10_far layout */ + mddev->layout = raid10_format_to_md_layout(rs, ALGORITHM_RAID10_FAR, + rs->raid10_copies); + } else if (mddev->level == 1) + /* raid1 -> raid10_near layout */ + mddev->layout = raid10_format_to_md_layout(rs, ALGORITHM_RAID10_NEAR, + rs->raid_disks); + else + return -EINVAL; + + } + + clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags); + mddev->recovery_cp = MaxSector; + + while (d--) { + rdev = &rs->dev[d].rdev; + + if (test_bit(d, (void *) rs->rebuild_disks)) { + clear_bit(In_sync, &rdev->flags); + clear_bit(Faulty, &rdev->flags); + mddev->recovery_cp = rdev->recovery_offset = 0; + /* Bitmap has to be created when we do an "up" takeover */ + set_bit(MD_ARRAY_FIRST_USE, &mddev->flags); + } + + rdev->new_data_offset = new_data_offset; + } + + return 0; +} + +/* Prepare @rs for reshape */ +static int rs_prepare_reshape(struct raid_set *rs) +{ + bool reshape; + struct mddev *mddev = &rs->md; + + if (rs_is_raid10(rs)) { + if (rs->raid_disks != mddev->raid_disks && + __is_raid10_near(mddev->layout) && + rs->raid10_copies && + rs->raid10_copies != __raid10_near_copies(mddev->layout)) { + /* + * raid disk have to be multiple of data copies to allow this conversion, + * + * This is actually not a reshape it is a + * rebuild of any additional mirrors per group + */ + if (rs->raid_disks % rs->raid10_copies) { + rs->ti->error = "Can't reshape raid10 mirror groups"; + return -EINVAL; + } + + /* Userpace reordered disks to add/remove mirrors -> adjust raid_disk indexes */ + __reorder_raid_disk_indexes(rs); + mddev->layout = raid10_format_to_md_layout(rs, ALGORITHM_RAID10_NEAR, + rs->raid10_copies); + mddev->new_layout = mddev->layout; + reshape = false; + } else + reshape = true; + + } else if (rs_is_raid456(rs)) + reshape = true; + + else if (rs_is_raid1(rs)) { + if (rs->delta_disks) { + /* Process raid1 via delta_disks */ + mddev->degraded = rs->delta_disks < 0 ? -rs->delta_disks : rs->delta_disks; + reshape = true; + } else { + /* Process raid1 without delta_disks */ + mddev->raid_disks = rs->raid_disks; + set_bit(RT_FLAG_KEEP_RS_FROZEN, &rs->runtime_flags); + reshape = false; + } + } else { + rs->ti->error = "Called with bogus raid type"; + return -EINVAL; + } + + if (reshape) { + set_bit(RT_FLAG_RESHAPE_RS, &rs->runtime_flags); + set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags); + set_bit(RT_FLAG_KEEP_RS_FROZEN, &rs->runtime_flags); + } else if (mddev->raid_disks < rs->raid_disks) + /* Create new superblocks and bitmaps, if any new disks */ + set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags); + + return 0; +} + +/* + * + * - change raid layout + * - change chunk size + * - add disks + * - remove disks + */ +static int rs_setup_reshape(struct raid_set *rs) +{ + int r = 0; + unsigned int cur_raid_devs, d; + struct mddev *mddev = &rs->md; + struct md_rdev *rdev; + + mddev->delta_disks = rs->delta_disks; + cur_raid_devs = mddev->raid_disks; + + /* Ignore impossible layout change whilst adding/removing disks */ + if (mddev->delta_disks && + mddev->layout != mddev->new_layout) { + DMINFO("Ignoring invalid layout change with delta_disks=%d", rs->delta_disks); + mddev->new_layout = mddev->layout; + } + + /* + * Adjust array size: + * + * - in case of adding disks, array size has + * to grow after the disk adding reshape, + * which'll hapen in the event handler; + * reshape will happen forward, so space has to + * be available at the beginning of each disk + * + * - in case of removing disks, array size + * has to shrink before starting the reshape, + * which'll happen here; + * reshape will happen backward, so space has to + * be available at the end of each disk + * + * - data_offset and new_data_offset are + * adjusted for aforementioned out of place + * reshaping based on userspace passing in + * the "data_offset " key/value + * pair via the constructor + */ - rdev->sb_loaded = 0; + /* Add disk(s) */ + if (rs->delta_disks > 0) { + /* Prepare disks for check in raid4/5/6/10 {check|start}_reshape */ + for (d = cur_raid_devs; d < rs->raid_disks; d++) { + rdev = &rs->dev[d].rdev; + clear_bit(In_sync, &rdev->flags); /* - * We might be able to salvage the data device - * even though the meta device has failed. For - * now, we behave as though '- -' had been - * set for this device in the table. + * save_raid_disk needs to be -1, or recovery_offset will be set to 0 + * by md, which'll store that erroneously in the superblock on reshape */ - if (dev->data_dev) - dm_put_device(ti, dev->data_dev); - - dev->data_dev = NULL; - rdev->bdev = NULL; + rdev->saved_raid_disk = -1; + rdev->raid_disk = d; - list_del(&rdev->same_set); + rdev->sectors = mddev->dev_sectors; + rdev->recovery_offset = rs_is_raid1(rs) ? 0 : MaxSector; } - } - if (!freshest) - return 0; - - if (validate_raid_redundancy(rs)) { - rs->ti->error = "Insufficient redundancy to activate array"; - return -EINVAL; - } + mddev->reshape_backwards = 0; /* adding disks -> forward reshape */ - /* - * Validation of the freshest device provides the source of - * validation for the remaining devices. - */ - ti->error = "Unable to assemble array: Invalid superblocks"; - if (super_validate(rs, freshest)) - return -EINVAL; + /* Remove disk(s) */ + } else if (rs->delta_disks < 0) { + r = rs_set_dev_and_array_sectors(rs, true); + mddev->reshape_backwards = 1; /* removing disk(s) -> backward reshape */ - rdev_for_each(rdev, mddev) - if ((rdev != freshest) && super_validate(rs, rdev)) - return -EINVAL; + /* Change layout and/or chunk size */ + } else { + /* + * Reshape layout (e.g. raid5_ls -> raid5_n) and/or chunk size: + * + * keeping number of disks and do layout change -> + * + * toggle reshape_backward depending on data_offset: + * + * - free space upfront -> reshape forward + * + * - free space at the end -> reshape backward + * + * + * This utilizes free reshape space avoiding the need + * for userspace to move (parts of) LV segments in + * case of layout/chunksize change (for disk + * adding/removing reshape space has to be at + * the proper address (see above with delta_disks): + * + * add disk(s) -> begin + * remove disk(s)-> end + */ + mddev->reshape_backwards = rs->dev[0].rdev.data_offset ? 0 : 1; + } - return 0; + return r; } /* * Enable/disable discard support on RAID set depending on * RAID level and discard properties of underlying RAID members. */ -static void configure_discard_support(struct dm_target *ti, struct raid_set *rs) +static void configure_discard_support(struct raid_set *rs) { int i; bool raid456; + struct dm_target *ti = rs->ti; /* Assume discards not supported until after checks below. */ ti->discards_supported = false; @@ -1174,7 +2713,7 @@ static void configure_discard_support(struct dm_target *ti, struct raid_set *rs) /* RAID level 4,5,6 require discard_zeroes_data for data integrity! */ raid456 = (rs->md.level == 4 || rs->md.level == 5 || rs->md.level == 6); - for (i = 0; i < rs->md.raid_disks; i++) { + for (i = 0; i < rs->raid_disks; i++) { struct request_queue *q; if (!rs->dev[i].rdev.bdev) @@ -1207,118 +2746,252 @@ static void configure_discard_support(struct dm_target *ti, struct raid_set *rs) } /* - * Construct a RAID4/5/6 mapping: + * Construct a RAID0/1/10/4/5/6 mapping: * Args: - * <#raid_params> \ - * <#raid_devs> { .. } + * <#raid_params> {0,} \ + * <#raid_devs> [ ]{1,} * - * varies by . See 'parse_raid_params' for + * varies by . See 'parse_raid_params' for * details on possible . + * + * Userspace is free to initialize the metadata devices, hence the superblocks to + * enforce recreation based on the passed in table parameters. + * */ -static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv) +static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv) { - int ret; + int r; + bool resize; struct raid_type *rt; - unsigned long num_raid_params, num_raid_devs; + unsigned int num_raid_params, num_raid_devs; + sector_t calculated_dev_sectors; struct raid_set *rs = NULL; - - /* Must have at least <#raid_params> */ - if (argc < 2) { - ti->error = "Too few arguments"; + const char *arg; + struct rs_layout rs_layout; + struct dm_arg_set as = { argc, argv }, as_nrd; + struct dm_arg _args[] = { + { 0, as.argc, "Cannot understand number of raid parameters" }, + { 1, 254, "Cannot understand number of raid devices parameters" } + }; + + /* Must have */ + arg = dm_shift_arg(&as); + if (!arg) { + ti->error = "No arguments"; return -EINVAL; } - /* raid type */ - rt = get_raid_type(argv[0]); + rt = get_raid_type(arg); if (!rt) { ti->error = "Unrecognised raid_type"; return -EINVAL; } - argc--; - argv++; - /* number of RAID parameters */ - if (kstrtoul(argv[0], 10, &num_raid_params) < 0) { - ti->error = "Cannot understand number of RAID parameters"; + /* Must have <#raid_params> */ + if (dm_read_arg_group(_args, &as, &num_raid_params, &ti->error)) return -EINVAL; - } - argc--; - argv++; - - /* Skip over RAID params for now and find out # of devices */ - if (num_raid_params >= argc) { - ti->error = "Arguments do not agree with counts given"; - return -EINVAL; - } - if ((kstrtoul(argv[num_raid_params], 10, &num_raid_devs) < 0) || - (num_raid_devs > MAX_RAID_DEVICES)) { - ti->error = "Cannot understand number of raid devices"; + /* number of raid device tupples */ + as_nrd = as; + dm_consume_args(&as_nrd, num_raid_params); + _args[1].max = (as_nrd.argc - 1) / 2; + if (dm_read_arg(_args + 1, &as_nrd, &num_raid_devs, &ti->error)) return -EINVAL; - } - argc -= num_raid_params + 1; /* +1: we already have num_raid_devs */ - if (argc != (num_raid_devs * 2)) { - ti->error = "Supplied RAID devices does not match the count given"; + if (!__within_range(num_raid_devs, 1, MAX_RAID_DEVICES)) { + ti->error = "Invalid number of supplied raid devices"; return -EINVAL; } - rs = context_alloc(ti, rt, (unsigned)num_raid_devs); + rs = raid_set_alloc(ti, rt, num_raid_devs); if (IS_ERR(rs)) return PTR_ERR(rs); - ret = parse_raid_params(rs, argv, (unsigned)num_raid_params); - if (ret) + r = parse_raid_params(rs, &as, num_raid_params); + if (r) goto bad; - argv += num_raid_params + 1; - - ret = dev_parms(rs, argv); - if (ret) + r = parse_dev_params(rs, &as); + if (r) goto bad; rs->md.sync_super = super_sync; - ret = analyse_superblocks(ti, rs); - if (ret) + + /* + * Calculate ctr requested array and device sizes to allow + * for superblock analysis needing device sizes defined. + * + * Any existing superblock will overwrite the array and device sizes + */ + r = rs_set_dev_and_array_sectors(rs, false); + if (r) + goto bad; + + calculated_dev_sectors = rs->dev[0].rdev.sectors; + + /* + * Backup any new raid set level, layout, ... + * requested to be able to compare to superblock + * members for conversion decisions. + */ + rs_config_backup(rs, &rs_layout); + + r = analyse_superblocks(ti, rs); + if (r) goto bad; + resize = calculated_dev_sectors != rs->dev[0].rdev.sectors; + INIT_WORK(&rs->md.event_work, do_table_event); ti->private = rs; ti->num_flush_bios = 1; + /* Restore any requested new layout for conversion decision */ + rs_config_restore(rs, &rs_layout); + /* - * Disable/enable discard support on RAID set. + * Now that we have any superblock metadata available, + * check for new, recovering, reshaping, to be taken over, + * to be reshaped or an existing, unchanged raid set to + * run in sequence. */ - configure_discard_support(ti, rs); + if (test_bit(MD_ARRAY_FIRST_USE, &rs->md.flags)) { + /* A new raid6 set has to be recovered to ensure proper parity and Q-Syndrome */ + if (rs_is_raid6(rs) && + test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)) { + ti->error = "'nosync' not allowed for new raid6 set"; + r = -EINVAL; + goto bad; + } + rs_setup_recovery(rs, 0); + set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags); + rs_set_new(rs); + } else if (rs_is_recovering(rs)) { + /* A recovering raid set may be resized */ + ; /* skip setup rs */ + } else if (rs_is_reshaping(rs)) { + /* Have to reject size change request during reshape */ + if (resize) { + ti->error = "Can't resize a reshaping raid set"; + r = -EPERM; + goto bad; + } + /* skip setup rs */ + } else if (rs_takeover_requested(rs)) { + if (rs_is_reshaping(rs)) { + ti->error = "Can't takeover a reshaping raid set"; + r = -EPERM; + goto bad; + } + + /* + * If a takeover is needed, userspace sets any additional + * devices to rebuild and we can check for a valid request here. + * + * If acceptible, set the level to the new requested + * one, prohibit requesting recovery, allow the raid + * set to run and store superblocks during resume. + */ + r = rs_check_takeover(rs); + if (r) + goto bad; + + r = rs_setup_takeover(rs); + if (r) + goto bad; + + set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags); + set_bit(RT_FLAG_KEEP_RS_FROZEN, &rs->runtime_flags); + /* Takeover ain't recovery, so disable recovery */ + rs_setup_recovery(rs, MaxSector); + rs_set_new(rs); + } else if (rs_reshape_requested(rs)) { + /* + * We can only prepare for a reshape here, because the + * raid set needs to run to provide the repective reshape + * check functions via its MD personality instance. + * + * So do the reshape check after md_run() succeeded. + */ + r = rs_prepare_reshape(rs); + if (r) + return r; + + /* Reshaping ain't recovery, so disable recovery */ + rs_setup_recovery(rs, MaxSector); + rs_set_cur(rs); + } else { + /* May not set recovery when a device rebuild is requested */ + if (test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags)) { + rs_setup_recovery(rs, MaxSector); + set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags); + } else + rs_setup_recovery(rs, test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags) ? + 0 : (resize ? calculated_dev_sectors : MaxSector)); + rs_set_cur(rs); + } + + /* If constructor requested it, change data and new_data offsets */ + r = rs_adjust_data_offsets(rs); + if (r) + goto bad; + + /* Start raid set read-only and assumed clean to change in raid_resume() */ + rs->md.ro = 1; + rs->md.in_sync = 1; + set_bit(MD_RECOVERY_FROZEN, &rs->md.recovery); /* Has to be held on running the array */ mddev_lock_nointr(&rs->md); - ret = md_run(&rs->md); + r = md_run(&rs->md); rs->md.in_sync = 0; /* Assume already marked dirty */ - mddev_unlock(&rs->md); - if (ret) { - ti->error = "Fail to run raid array"; + if (r) { + ti->error = "Failed to run raid array"; + mddev_unlock(&rs->md); goto bad; } - if (ti->len != rs->md.array_sectors) { - ti->error = "Array size does not match requested target length"; - ret = -EINVAL; - goto size_mismatch; - } rs->callbacks.congested_fn = raid_is_congested; dm_table_add_target_callbacks(ti->table, &rs->callbacks); mddev_suspend(&rs->md); + + /* Try to adjust the raid4/5/6 stripe cache size to the stripe size */ + if (rs_is_raid456(rs)) { + r = rs_set_raid456_stripe_cache(rs); + if (r) + goto bad_stripe_cache; + } + + /* Now do an early reshape check */ + if (test_bit(RT_FLAG_RESHAPE_RS, &rs->runtime_flags)) { + r = rs_check_reshape(rs); + if (r) + goto bad_check_reshape; + + /* Restore new, ctr requested layout to perform check */ + rs_config_restore(rs, &rs_layout); + + if (rs->md.pers->start_reshape) { + r = rs->md.pers->check_reshape(&rs->md); + if (r) { + ti->error = "Reshape check failed"; + goto bad_check_reshape; + } + } + } + + mddev_unlock(&rs->md); return 0; -size_mismatch: +bad_stripe_cache: +bad_check_reshape: md_stop(&rs->md); bad: - context_free(rs); + raid_set_free(rs); - return ret; + return r; } static void raid_dtr(struct dm_target *ti) @@ -1327,7 +3000,7 @@ static void raid_dtr(struct dm_target *ti) list_del_init(&rs->callbacks.list); md_stop(&rs->md); - context_free(rs); + raid_set_free(rs); } static int raid_map(struct dm_target *ti, struct bio *bio) @@ -1335,11 +3008,23 @@ static int raid_map(struct dm_target *ti, struct bio *bio) struct raid_set *rs = ti->private; struct mddev *mddev = &rs->md; + /* + * If we're reshaping to add disk(s)), ti->len and + * mddev->array_sectors will differ during the process + * (ti->len > mddev->array_sectors), so we have to requeue + * bios with addresses > mddev->array_sectors here or + * there will occur accesses past EOD of the component + * data images thus erroring the raid set. + */ + if (unlikely(bio_end_sector(bio) > mddev->array_sectors)) + return DM_MAPIO_REQUEUE; + mddev->pers->make_request(mddev, bio); return DM_MAPIO_SUBMITTED; } +/* Return string describing the current sync action of @mddev */ static const char *decipher_sync_action(struct mddev *mddev) { if (test_bit(MD_RECOVERY_FROZEN, &mddev->recovery)) @@ -1365,195 +3050,260 @@ static const char *decipher_sync_action(struct mddev *mddev) return "idle"; } -static void raid_status(struct dm_target *ti, status_type_t type, - unsigned status_flags, char *result, unsigned maxlen) +/* + * Return status string @rdev + * + * Status characters: + * + * 'D' = Dead/Failed device + * 'a' = Alive but not in-sync + * 'A' = Alive and in-sync + */ +static const char *__raid_dev_status(struct md_rdev *rdev, bool array_in_sync) { - struct raid_set *rs = ti->private; - unsigned raid_param_cnt = 1; /* at least 1 for chunksize */ - unsigned sz = 0; - int i, array_in_sync = 0; - sector_t sync; + if (test_bit(Faulty, &rdev->flags)) + return "D"; + else if (!array_in_sync || !test_bit(In_sync, &rdev->flags)) + return "a"; + else + return "A"; +} - switch (type) { - case STATUSTYPE_INFO: - DMEMIT("%s %d ", rs->raid_type->name, rs->md.raid_disks); +/* Helper to return resync/reshape progress for @rs and @array_in_sync */ +static sector_t rs_get_progress(struct raid_set *rs, + sector_t resync_max_sectors, bool *array_in_sync) +{ + sector_t r, recovery_cp, curr_resync_completed; + struct mddev *mddev = &rs->md; - if (rs->raid_type->level) { - if (test_bit(MD_RECOVERY_RUNNING, &rs->md.recovery)) - sync = rs->md.curr_resync_completed; - else - sync = rs->md.recovery_cp; - - if (sync >= rs->md.resync_max_sectors) { - /* - * Sync complete. - */ - array_in_sync = 1; - sync = rs->md.resync_max_sectors; - } else if (test_bit(MD_RECOVERY_REQUESTED, &rs->md.recovery)) { - /* - * If "check" or "repair" is occurring, the array has - * undergone and initial sync and the health characters - * should not be 'a' anymore. - */ - array_in_sync = 1; + curr_resync_completed = mddev->curr_resync_completed ?: mddev->recovery_cp; + recovery_cp = mddev->recovery_cp; + *array_in_sync = false; + + if (rs_is_raid0(rs)) { + r = resync_max_sectors; + *array_in_sync = true; + + } else { + r = mddev->reshape_position; + + /* Reshape is relative to the array size */ + if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) || + r != MaxSector) { + if (r == MaxSector) { + *array_in_sync = true; + r = resync_max_sectors; } else { - /* - * The array may be doing an initial sync, or it may - * be rebuilding individual components. If all the - * devices are In_sync, then it is the array that is - * being initialized. - */ - for (i = 0; i < rs->md.raid_disks; i++) - if (!test_bit(In_sync, &rs->dev[i].rdev.flags)) - array_in_sync = 1; + /* Got to reverse on backward reshape */ + if (mddev->reshape_backwards) + r = mddev->array_sectors - r; + + /* Devide by # of data stripes */ + sector_div(r, mddev_data_stripes(rs)); } + + /* Sync is relative to the component device size */ + } else if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) + r = curr_resync_completed; + else + r = recovery_cp; + + if (r == MaxSector) { + /* + * Sync complete. + */ + *array_in_sync = true; + r = resync_max_sectors; + } else if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { + /* + * If "check" or "repair" is occurring, the raid set has + * undergone an initial sync and the health characters + * should not be 'a' anymore. + */ + *array_in_sync = true; } else { - /* RAID0 */ - array_in_sync = 1; - sync = rs->md.resync_max_sectors; - } + struct md_rdev *rdev; - /* - * Status characters: - * 'D' = Dead/Failed device - * 'a' = Alive but not in-sync - * 'A' = Alive and in-sync - */ - for (i = 0; i < rs->md.raid_disks; i++) { - if (test_bit(Faulty, &rs->dev[i].rdev.flags)) - DMEMIT("D"); - else if (!array_in_sync || - !test_bit(In_sync, &rs->dev[i].rdev.flags)) - DMEMIT("a"); - else - DMEMIT("A"); + /* + * The raid set may be doing an initial sync, or it may + * be rebuilding individual components. If all the + * devices are In_sync, then it is the raid set that is + * being initialized. + */ + rdev_for_each(rdev, mddev) + if (!test_bit(In_sync, &rdev->flags)) + *array_in_sync = true; +#if 0 + r = 0; /* HM FIXME: TESTME: https://bugzilla.redhat.com/show_bug.cgi?id=1210637 ? */ +#endif } + } + + return r; +} + +/* Helper to return @dev name or "-" if !@dev */ +static const char *__get_dev_name(struct dm_dev *dev) +{ + return dev ? dev->name : "-"; +} + +static void raid_status(struct dm_target *ti, status_type_t type, + unsigned int status_flags, char *result, unsigned int maxlen) +{ + struct raid_set *rs = ti->private; + struct mddev *mddev = &rs->md; + struct r5conf *conf = mddev->private; + int i, max_nr_stripes = conf ? conf->max_nr_stripes : 0; + bool array_in_sync; + unsigned int raid_param_cnt = 1; /* at least 1 for chunksize */ + unsigned int sz = 0; + unsigned int rebuild_disks; + unsigned int write_mostly_params = 0; + sector_t progress, resync_max_sectors, resync_mismatches; + const char *sync_action; + struct raid_type *rt; + struct md_rdev *rdev; + + switch (type) { + case STATUSTYPE_INFO: + /* *Should* always succeed */ + rt = get_raid_type_by_ll(mddev->new_level, mddev->new_layout); + if (!rt) + return; + + DMEMIT("%s %d ", rt->name, mddev->raid_disks); + + /* Access most recent mddev properties for status output */ + smp_rmb(); + /* Get sensible max sectors even if raid set not yet started */ + resync_max_sectors = test_bit(RT_FLAG_RS_PRERESUMED, &rs->runtime_flags) ? + mddev->resync_max_sectors : mddev->dev_sectors; + progress = rs_get_progress(rs, resync_max_sectors, &array_in_sync); + resync_mismatches = (mddev->last_sync_action && !strcasecmp(mddev->last_sync_action, "check")) ? + atomic64_read(&mddev->resync_mismatches) : 0; + sync_action = decipher_sync_action(&rs->md); + + /* HM FIXME: do we want another state char for raid0? It shows 'D' or 'A' now */ + rdev_for_each(rdev, mddev) + DMEMIT(__raid_dev_status(rdev, array_in_sync)); /* - * In-sync ratio: + * In-sync/Reshape ratio: * The in-sync ratio shows the progress of: - * - Initializing the array - * - Rebuilding a subset of devices of the array + * - Initializing the raid set + * - Rebuilding a subset of devices of the raid set * The user can distinguish between the two by referring * to the status characters. + * + * The reshape ratio shows the progress of + * changing the raid layout or the number of + * disks of a raid set */ - DMEMIT(" %llu/%llu", - (unsigned long long) sync, - (unsigned long long) rs->md.resync_max_sectors); + DMEMIT(" %llu/%llu", (unsigned long long) progress, + (unsigned long long) resync_max_sectors); /* + * v1.5.0+: + * * Sync action: - * See Documentation/device-mapper/dm-raid.c for + * See Documentation/device-mapper/dm-raid.txt for * information on each of these states. */ - DMEMIT(" %s", decipher_sync_action(&rs->md)); + DMEMIT(" %s", sync_action); /* + * v1.5.0+: + * * resync_mismatches/mismatch_cnt * This field shows the number of discrepancies found when - * performing a "check" of the array. + * performing a "check" of the raid set. */ - DMEMIT(" %llu", - (strcmp(rs->md.last_sync_action, "check")) ? 0 : - (unsigned long long) - atomic64_read(&rs->md.resync_mismatches)); - break; - case STATUSTYPE_TABLE: - /* The string you would use to construct this array */ - for (i = 0; i < rs->md.raid_disks; i++) { - if ((rs->ctr_flags & CTR_FLAG_REBUILD) && - rs->dev[i].data_dev && - !test_bit(In_sync, &rs->dev[i].rdev.flags)) - raid_param_cnt += 2; /* for rebuilds */ - if (rs->dev[i].data_dev && - test_bit(WriteMostly, &rs->dev[i].rdev.flags)) - raid_param_cnt += 2; - } - - raid_param_cnt += (hweight32(rs->ctr_flags & ~CTR_FLAG_REBUILD) * 2); - if (rs->ctr_flags & (CTR_FLAG_SYNC | CTR_FLAG_NOSYNC)) - raid_param_cnt--; - - DMEMIT("%s %u %u", rs->raid_type->name, - raid_param_cnt, rs->md.chunk_sectors); - - if ((rs->ctr_flags & CTR_FLAG_SYNC) && - (rs->md.recovery_cp == MaxSector)) - DMEMIT(" sync"); - if (rs->ctr_flags & CTR_FLAG_NOSYNC) - DMEMIT(" nosync"); - - for (i = 0; i < rs->md.raid_disks; i++) - if ((rs->ctr_flags & CTR_FLAG_REBUILD) && - rs->dev[i].data_dev && - !test_bit(In_sync, &rs->dev[i].rdev.flags)) - DMEMIT(" rebuild %u", i); - - if (rs->ctr_flags & CTR_FLAG_DAEMON_SLEEP) - DMEMIT(" daemon_sleep %lu", - rs->md.bitmap_info.daemon_sleep); - - if (rs->ctr_flags & CTR_FLAG_MIN_RECOVERY_RATE) - DMEMIT(" min_recovery_rate %d", rs->md.sync_speed_min); + DMEMIT(" %llu", (unsigned long long) resync_mismatches); - if (rs->ctr_flags & CTR_FLAG_MAX_RECOVERY_RATE) - DMEMIT(" max_recovery_rate %d", rs->md.sync_speed_max); - - for (i = 0; i < rs->md.raid_disks; i++) - if (rs->dev[i].data_dev && - test_bit(WriteMostly, &rs->dev[i].rdev.flags)) - DMEMIT(" write_mostly %u", i); - - if (rs->ctr_flags & CTR_FLAG_MAX_WRITE_BEHIND) - DMEMIT(" max_write_behind %lu", - rs->md.bitmap_info.max_write_behind); - - if (rs->ctr_flags & CTR_FLAG_STRIPE_CACHE) { - struct r5conf *conf = rs->md.private; - - /* convert from kiB to sectors */ - DMEMIT(" stripe_cache %d", - conf ? conf->max_nr_stripes * 2 : 0); - } - - if (rs->ctr_flags & CTR_FLAG_REGION_SIZE) - DMEMIT(" region_size %lu", - rs->md.bitmap_info.chunksize >> 9); - - if (rs->ctr_flags & CTR_FLAG_RAID10_COPIES) - DMEMIT(" raid10_copies %u", - raid10_md_layout_to_copies(rs->md.layout)); - - if (rs->ctr_flags & CTR_FLAG_RAID10_FORMAT) - DMEMIT(" raid10_format %s", - raid10_md_layout_to_format(rs->md.layout)); - - DMEMIT(" %d", rs->md.raid_disks); - for (i = 0; i < rs->md.raid_disks; i++) { - if (rs->dev[i].meta_dev) - DMEMIT(" %s", rs->dev[i].meta_dev->name); - else - DMEMIT(" -"); + /* + * v1.9.0+: + * + * data_offset (needed for out of space reshaping) + * This field shows the data offset into the data + * image LV where the first stripes data starts. + * + * We keep data_offset equal on all raid disks of the set, + * so retrieving it from the first raid disk is sufficient. + */ + DMEMIT(" %llu", (unsigned long long) rs->dev[0].rdev.data_offset); + break; - if (rs->dev[i].data_dev) - DMEMIT(" %s", rs->dev[i].data_dev->name); - else - DMEMIT(" -"); - } + case STATUSTYPE_TABLE: + /* Report the table line string you would use to construct this raid set */ + + /* Calculate raid parameter count */ + for (i = 0; i < rs->raid_disks; i++) + if (test_bit(WriteMostly, &rs->dev[i].rdev.flags)) + write_mostly_params += 2; + rebuild_disks = memweight(rs->rebuild_disks, DISKS_ARRAY_ELEMS * sizeof(*rs->rebuild_disks)); + raid_param_cnt += rebuild_disks * 2 + + write_mostly_params + + hweight32(rs->ctr_flags & CTR_FLAG_OPTIONS_NO_ARGS) + + hweight32(rs->ctr_flags & CTR_FLAG_OPTIONS_ONE_ARG) * 2; + /* Emit table line */ + DMEMIT("%s %u %u", rs->raid_type->name, raid_param_cnt, mddev->new_chunk_sectors); + if (test_bit(__CTR_FLAG_RAID10_FORMAT, &rs->ctr_flags)) + DMEMIT(" %s %s", dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_FORMAT), + raid10_md_layout_to_format(mddev->layout)); + if (test_bit(__CTR_FLAG_RAID10_COPIES, &rs->ctr_flags)) + DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_COPIES), + raid10_md_layout_to_copies(mddev->layout)); + if (test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)) + DMEMIT(" %s", dm_raid_arg_name_by_flag(CTR_FLAG_NOSYNC)); + if (test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags)) + DMEMIT(" %s", dm_raid_arg_name_by_flag(CTR_FLAG_SYNC)); + if (test_bit(__CTR_FLAG_REGION_SIZE, &rs->ctr_flags)) + DMEMIT(" %s %llu", dm_raid_arg_name_by_flag(CTR_FLAG_REGION_SIZE), + (unsigned long long) to_sector(mddev->bitmap_info.chunksize)); + if (test_bit(__CTR_FLAG_DATA_OFFSET, &rs->ctr_flags)) + DMEMIT(" %s %llu", dm_raid_arg_name_by_flag(CTR_FLAG_DATA_OFFSET), + (unsigned long long) rs->data_offset); + if (test_bit(__CTR_FLAG_DAEMON_SLEEP, &rs->ctr_flags)) + DMEMIT(" %s %lu", dm_raid_arg_name_by_flag(CTR_FLAG_DAEMON_SLEEP), + mddev->bitmap_info.daemon_sleep); + if (test_bit(__CTR_FLAG_DELTA_DISKS, &rs->ctr_flags)) + DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_DELTA_DISKS), + max(rs->delta_disks, mddev->delta_disks)); + if (test_bit(__CTR_FLAG_STRIPE_CACHE, &rs->ctr_flags)) + DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_STRIPE_CACHE), + max_nr_stripes); + if (rebuild_disks) + for (i = 0; i < rs->raid_disks; i++) + if (test_bit(rs->dev[i].rdev.raid_disk, (void *) rs->rebuild_disks)) + DMEMIT(" %s %u", dm_raid_arg_name_by_flag(CTR_FLAG_REBUILD), + rs->dev[i].rdev.raid_disk); + if (write_mostly_params) + for (i = 0; i < rs->raid_disks; i++) + if (test_bit(WriteMostly, &rs->dev[i].rdev.flags)) + DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_WRITE_MOSTLY), + rs->dev[i].rdev.raid_disk); + if (test_bit(__CTR_FLAG_MAX_WRITE_BEHIND, &rs->ctr_flags)) + DMEMIT(" %s %lu", dm_raid_arg_name_by_flag(CTR_FLAG_MAX_WRITE_BEHIND), + mddev->bitmap_info.max_write_behind); + if (test_bit(__CTR_FLAG_MAX_RECOVERY_RATE, &rs->ctr_flags)) + DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_MAX_RECOVERY_RATE), + mddev->sync_speed_max); + if (test_bit(__CTR_FLAG_MIN_RECOVERY_RATE, &rs->ctr_flags)) + DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_MIN_RECOVERY_RATE), + mddev->sync_speed_min); + DMEMIT(" %d", rs->raid_disks); + for (i = 0; i < rs->raid_disks; i++) + DMEMIT(" %s %s", __get_dev_name(rs->dev[i].meta_dev), + __get_dev_name(rs->dev[i].data_dev)); } } -static int raid_message(struct dm_target *ti, unsigned argc, char **argv) +static int raid_message(struct dm_target *ti, unsigned int argc, char **argv) { struct raid_set *rs = ti->private; struct mddev *mddev = &rs->md; - if (!strcasecmp(argv[0], "reshape")) { - DMERR("Reshape not supported."); - return -EINVAL; - } - if (!mddev->pers || !mddev->pers->sync_request) return -EINVAL; @@ -1571,11 +3321,10 @@ static int raid_message(struct dm_target *ti, unsigned argc, char **argv) test_bit(MD_RECOVERY_NEEDED, &mddev->recovery)) return -EBUSY; else if (!strcasecmp(argv[0], "resync")) - set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); - else if (!strcasecmp(argv[0], "recover")) { + ; /* MD_RECOVERY_NEEDED set below */ + else if (!strcasecmp(argv[0], "recover")) set_bit(MD_RECOVERY_RECOVER, &mddev->recovery); - set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); - } else { + else { if (!strcasecmp(argv[0], "check")) set_bit(MD_RECOVERY_CHECK, &mddev->recovery); else if (!!strcasecmp(argv[0], "repair")) @@ -1588,11 +3337,11 @@ static int raid_message(struct dm_target *ti, unsigned argc, char **argv) * canceling read-auto mode */ mddev->ro = 0; - if (!mddev->suspended) + if (!mddev->suspended && mddev->sync_thread) md_wakeup_thread(mddev->sync_thread); } set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); - if (!mddev->suspended) + if (!mddev->suspended && mddev->thread) md_wakeup_thread(mddev->thread); return 0; @@ -1602,28 +3351,27 @@ static int raid_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) { struct raid_set *rs = ti->private; - unsigned i; - int ret = 0; + unsigned int i; + int r = 0; - for (i = 0; !ret && i < rs->md.raid_disks; i++) + for (i = 0; !r && i < rs->md.raid_disks; i++) if (rs->dev[i].data_dev) - ret = fn(ti, + r = fn(ti, rs->dev[i].data_dev, 0, /* No offset on data devs */ rs->md.dev_sectors, data); - return ret; + return r; } static void raid_io_hints(struct dm_target *ti, struct queue_limits *limits) { struct raid_set *rs = ti->private; - unsigned chunk_size = rs->md.chunk_sectors << 9; - struct r5conf *conf = rs->md.private; + unsigned int chunk_size = to_bytes(rs->md.chunk_sectors); blk_limits_io_min(limits, chunk_size); - blk_limits_io_opt(limits, chunk_size * (conf->raid_disks - conf->max_degraded)); + blk_limits_io_opt(limits, chunk_size * mddev_data_stripes(rs)); } static void raid_presuspend(struct dm_target *ti) @@ -1637,7 +3385,11 @@ static void raid_postsuspend(struct dm_target *ti) { struct raid_set *rs = ti->private; - mddev_suspend(&rs->md); + if (test_and_clear_bit(RT_FLAG_RS_RESUMED, &rs->runtime_flags)) { + if (!rs->md.suspended) + mddev_suspend(&rs->md); + rs->md.ro = 1; + } } static void attempt_restore_of_faulty_devices(struct raid_set *rs) @@ -1651,7 +3403,8 @@ static void attempt_restore_of_faulty_devices(struct raid_set *rs) for (i = 0; i < rs->md.raid_disks; i++) { r = &rs->dev[i].rdev; if (test_bit(Faulty, &r->flags) && r->sb_page && - sync_page_io(r, 0, r->sb_size, r->sb_page, READ, 1)) { + sync_page_io(r, 0, r->sb_size, r->sb_page, + REQ_OP_READ, 0, true)) { DMINFO("Faulty %s device #%d has readable super block." " Attempting to revive it.", rs->raid_type->name, i); @@ -1660,7 +3413,7 @@ static void attempt_restore_of_faulty_devices(struct raid_set *rs) * Faulty bit may be set, but sometimes the array can * be suspended before the personalities can respond * by removing the device from the array (i.e. calling - * 'hot_remove_disk'). If they haven't yet removed + * 'hot_remove_disk'). If they haven't yet removed * the failed device, its 'raid_disk' number will be * '>= 0' - meaning we must call this function * ourselves. @@ -1696,34 +3449,192 @@ static void attempt_restore_of_faulty_devices(struct raid_set *rs) } } -static void raid_resume(struct dm_target *ti) +static int __load_dirty_region_bitmap(struct raid_set *rs) { - struct raid_set *rs = ti->private; + int r = 0; + + /* Try loading the bitmap unless "raid0", which does not have one */ + if (!rs_is_raid0(rs) && + !test_and_set_bit(RT_FLAG_RS_BITMAP_LOADED, &rs->runtime_flags)) { + r = bitmap_load(&rs->md); + if (r) + DMERR("Failed to load bitmap"); + } - if (rs->raid_type->level) { - set_bit(MD_CHANGE_DEVS, &rs->md.flags); + return r; +} - if (!rs->bitmap_loaded) { - bitmap_load(&rs->md); - rs->bitmap_loaded = 1; - } else { - /* - * A secondary resume while the device is active. - * Take this opportunity to check whether any failed - * devices are reachable again. - */ - attempt_restore_of_faulty_devices(rs); +/* Enforce updating all superblocks */ +static void rs_update_sbs(struct raid_set *rs) +{ + struct mddev *mddev = &rs->md; + int ro = mddev->ro; + + set_bit(MD_CHANGE_DEVS, &mddev->flags); + mddev->ro = 0; + md_update_sb(mddev, 1); + mddev->ro = ro; +} + +/* + * Reshape changes raid algorithm of @rs to new one within personality + * (e.g. raid6_zr -> raid6_nc), changes stripe size, adds/removes + * disks from a raid set thus growing/shrinking it or resizes the set + * + * Call mddev_lock_nointr() before! + */ +static int rs_start_reshape(struct raid_set *rs) +{ + int r; + struct mddev *mddev = &rs->md; + struct md_personality *pers = mddev->pers; + + r = rs_setup_reshape(rs); + if (r) + return r; + + /* Need to be resumed to be able to start reshape, recovery is frozen until raid_resume() though */ + if (mddev->suspended) + mddev_resume(mddev); + + /* + * Check any reshape constraints enforced by the personalility + * + * May as well already kick the reshape off so that * pers->start_reshape() becomes optional. + */ + r = pers->check_reshape(mddev); + if (r) { + rs->ti->error = "pers->check_reshape() failed"; + return r; + } + + /* + * Personality may not provide start reshape method in which + * case check_reshape above has already covered everything + */ + if (pers->start_reshape) { + r = pers->start_reshape(mddev); + if (r) { + rs->ti->error = "pers->start_reshape() failed"; + return r; } + } + + /* Suspend because a resume will happen in raid_resume() */ + if (!mddev->suspended) + mddev_suspend(mddev); + + /* + * Now reshape got set up, update superblocks to + * reflect the fact so that a table reload will + * access proper superblock content in the ctr. + */ + rs_update_sbs(rs); + + return 0; +} + +static int raid_preresume(struct dm_target *ti) +{ + int r; + struct raid_set *rs = ti->private; + struct mddev *mddev = &rs->md; + + /* This is a resume after a suspend of the set -> it's already started */ + if (test_and_set_bit(RT_FLAG_RS_PRERESUMED, &rs->runtime_flags)) + return 0; + + /* + * The superblocks need to be updated on disk if the + * array is new or new devices got added (thus zeroed + * out by userspace) or __load_dirty_region_bitmap + * will overwrite them in core with old data or fail. + */ + if (test_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags)) + rs_update_sbs(rs); + + /* + * Disable/enable discard support on raid set after any + * conversion, because devices can have been added + */ + configure_discard_support(rs); + + /* Load the bitmap from disk unless raid0 */ + r = __load_dirty_region_bitmap(rs); + if (r) + return r; + + /* Resize bitmap to adjust to changed region size (aka MD bitmap chunksize) */ + if (test_bit(RT_FLAG_RS_BITMAP_LOADED, &rs->runtime_flags) && + mddev->bitmap_info.chunksize != to_bytes(rs->requested_bitmap_chunk_sectors)) { + r = bitmap_resize(mddev->bitmap, mddev->dev_sectors, + to_bytes(rs->requested_bitmap_chunk_sectors), 0); + if (r) + DMERR("Failed to resize bitmap"); + } + + /* Check for any resize/reshape on @rs and adjust/initiate */ + /* Be prepared for mddev_resume() in raid_resume() */ + set_bit(MD_RECOVERY_FROZEN, &mddev->recovery); + if (mddev->recovery_cp && mddev->recovery_cp < MaxSector) { + set_bit(MD_RECOVERY_REQUESTED, &mddev->recovery); + set_bit(MD_RECOVERY_SYNC, &mddev->recovery); + mddev->resync_min = mddev->recovery_cp; + } - clear_bit(MD_RECOVERY_FROZEN, &rs->md.recovery); + rs_set_capacity(rs); + + /* Check for any reshape request unless new raid set */ + if (test_and_clear_bit(RT_FLAG_RESHAPE_RS, &rs->runtime_flags)) { + /* Initiate a reshape. */ + mddev_lock_nointr(mddev); + r = rs_start_reshape(rs); + mddev_unlock(mddev); + if (r) + DMWARN("Failed to check/start reshape, continuing without change"); + r = 0; } - mddev_resume(&rs->md); + return r; +} + +static void raid_resume(struct dm_target *ti) +{ + struct raid_set *rs = ti->private; + struct mddev *mddev = &rs->md; + + if (test_and_set_bit(RT_FLAG_RS_RESUMED, &rs->runtime_flags)) { + /* + * A secondary resume while the device is active. + * Take this opportunity to check whether any failed + * devices are reachable again. + */ + attempt_restore_of_faulty_devices(rs); + } else { + mddev->ro = 0; + mddev->in_sync = 0; + + /* + * When passing in flags to the ctr, we expect userspace + * to reset them because they made it to the superblocks + * and reload the mapping anyway. + * + * -> only unfreeze recovery in case of a table reload or + * we'll have a bogus recovery/reshape position + * retrieved from the superblock by the ctr because + * the ongoing recovery/reshape will change it after read. + */ + if (!test_bit(RT_FLAG_KEEP_RS_FROZEN, &rs->runtime_flags)) + clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery); + + if (mddev->suspended) + mddev_resume(mddev); + } } static struct target_type raid_target = { .name = "raid", - .version = {1, 8, 0}, + .version = {1, 9, 0}, .module = THIS_MODULE, .ctr = raid_ctr, .dtr = raid_dtr, @@ -1734,6 +3645,7 @@ static struct target_type raid_target = { .io_hints = raid_io_hints, .presuspend = raid_presuspend, .postsuspend = raid_postsuspend, + .preresume = raid_preresume, .resume = raid_resume, }; @@ -1758,11 +3670,13 @@ module_param(devices_handle_discard_safely, bool, 0644); MODULE_PARM_DESC(devices_handle_discard_safely, "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); -MODULE_DESCRIPTION(DM_NAME " raid4/5/6 target"); +MODULE_DESCRIPTION(DM_NAME " raid0/1/10/4/5/6 target"); +MODULE_ALIAS("dm-raid0"); MODULE_ALIAS("dm-raid1"); MODULE_ALIAS("dm-raid10"); MODULE_ALIAS("dm-raid4"); MODULE_ALIAS("dm-raid5"); MODULE_ALIAS("dm-raid6"); MODULE_AUTHOR("Neil Brown "); +MODULE_AUTHOR("Heinz Mauelshagen "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index b3ccf1e0d4f2..dac55b254a09 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -260,7 +260,8 @@ static int mirror_flush(struct dm_target *ti) struct dm_io_region io[ms->nr_mirrors]; struct mirror *m; struct dm_io_request io_req = { - .bi_rw = WRITE_FLUSH, + .bi_op = REQ_OP_WRITE, + .bi_op_flags = WRITE_FLUSH, .mem.type = DM_IO_KMEM, .mem.ptr.addr = NULL, .client = ms->io_client, @@ -527,7 +528,7 @@ static void read_callback(unsigned long error, void *context) DMWARN_LIMIT("Read failure on mirror device %s. " "Trying alternative device.", m->dev->name); - queue_bio(m->ms, bio, bio_rw(bio)); + queue_bio(m->ms, bio, bio_data_dir(bio)); return; } @@ -541,7 +542,8 @@ static void read_async_bio(struct mirror *m, struct bio *bio) { struct dm_io_region io; struct dm_io_request io_req = { - .bi_rw = READ, + .bi_op = REQ_OP_READ, + .bi_op_flags = 0, .mem.type = DM_IO_BIO, .mem.ptr.bio = bio, .notify.fn = read_callback, @@ -624,7 +626,7 @@ static void write_callback(unsigned long error, void *context) * If the bio is discard, return an error, but do not * degrade the array. */ - if (bio->bi_rw & REQ_DISCARD) { + if (bio_op(bio) == REQ_OP_DISCARD) { bio->bi_error = -EOPNOTSUPP; bio_endio(bio); return; @@ -654,7 +656,8 @@ static void do_write(struct mirror_set *ms, struct bio *bio) struct dm_io_region io[ms->nr_mirrors], *dest = io; struct mirror *m; struct dm_io_request io_req = { - .bi_rw = WRITE | (bio->bi_rw & WRITE_FLUSH_FUA), + .bi_op = REQ_OP_WRITE, + .bi_op_flags = bio->bi_rw & WRITE_FLUSH_FUA, .mem.type = DM_IO_BIO, .mem.ptr.bio = bio, .notify.fn = write_callback, @@ -662,8 +665,8 @@ static void do_write(struct mirror_set *ms, struct bio *bio) .client = ms->io_client, }; - if (bio->bi_rw & REQ_DISCARD) { - io_req.bi_rw |= REQ_DISCARD; + if (bio_op(bio) == REQ_OP_DISCARD) { + io_req.bi_op = REQ_OP_DISCARD; io_req.mem.type = DM_IO_KMEM; io_req.mem.ptr.addr = NULL; } @@ -701,8 +704,8 @@ static void do_writes(struct mirror_set *ms, struct bio_list *writes) bio_list_init(&requeue); while ((bio = bio_list_pop(writes))) { - if ((bio->bi_rw & REQ_FLUSH) || - (bio->bi_rw & REQ_DISCARD)) { + if ((bio->bi_rw & REQ_PREFLUSH) || + (bio_op(bio) == REQ_OP_DISCARD)) { bio_list_add(&sync, bio); continue; } @@ -1190,7 +1193,7 @@ static void mirror_dtr(struct dm_target *ti) */ static int mirror_map(struct dm_target *ti, struct bio *bio) { - int r, rw = bio_rw(bio); + int r, rw = bio_data_dir(bio); struct mirror *m; struct mirror_set *ms = ti->private; struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); @@ -1214,7 +1217,7 @@ static int mirror_map(struct dm_target *ti, struct bio *bio) * If region is not in-sync queue the bio. */ if (!r || (r == -EWOULDBLOCK)) { - if (rw == READA) + if (bio->bi_rw & REQ_RAHEAD) return -EWOULDBLOCK; queue_bio(ms, bio, rw); @@ -1239,7 +1242,7 @@ static int mirror_map(struct dm_target *ti, struct bio *bio) static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error) { - int rw = bio_rw(bio); + int rw = bio_data_dir(bio); struct mirror_set *ms = (struct mirror_set *) ti->private; struct mirror *m = NULL; struct dm_bio_details *bd = NULL; @@ -1250,7 +1253,8 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error) * We need to dec pending if this was a write. */ if (rw == WRITE) { - if (!(bio->bi_rw & (REQ_FLUSH | REQ_DISCARD))) + if (!(bio->bi_rw & REQ_PREFLUSH) && + bio_op(bio) != REQ_OP_DISCARD) dm_rh_dec(ms->rh, bio_record->write_region); return error; } diff --git a/drivers/md/dm-region-hash.c b/drivers/md/dm-region-hash.c index 74cb7b991d41..b11813431f31 100644 --- a/drivers/md/dm-region-hash.c +++ b/drivers/md/dm-region-hash.c @@ -398,12 +398,12 @@ void dm_rh_mark_nosync(struct dm_region_hash *rh, struct bio *bio) region_t region = dm_rh_bio_to_region(rh, bio); int recovering = 0; - if (bio->bi_rw & REQ_FLUSH) { + if (bio->bi_rw & REQ_PREFLUSH) { rh->flush_failure = 1; return; } - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) return; /* We must inform the log that the sync count has changed. */ @@ -526,7 +526,7 @@ void dm_rh_inc_pending(struct dm_region_hash *rh, struct bio_list *bios) struct bio *bio; for (bio = bios->head; bio; bio = bio->bi_next) { - if (bio->bi_rw & (REQ_FLUSH | REQ_DISCARD)) + if (bio->bi_rw & REQ_PREFLUSH || bio_op(bio) == REQ_OP_DISCARD) continue; rh_inc(rh, dm_rh_bio_to_region(rh, bio)); } diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c new file mode 100644 index 000000000000..7a9661868496 --- /dev/null +++ b/drivers/md/dm-rq.c @@ -0,0 +1,970 @@ +/* + * Copyright (C) 2016 Red Hat, Inc. All rights reserved. + * + * This file is released under the GPL. + */ + +#include "dm-core.h" +#include "dm-rq.h" + +#include /* for rq_end_sector() */ +#include + +#define DM_MSG_PREFIX "core-rq" + +#define DM_MQ_NR_HW_QUEUES 1 +#define DM_MQ_QUEUE_DEPTH 2048 +static unsigned dm_mq_nr_hw_queues = DM_MQ_NR_HW_QUEUES; +static unsigned dm_mq_queue_depth = DM_MQ_QUEUE_DEPTH; + +/* + * Request-based DM's mempools' reserved IOs set by the user. + */ +#define RESERVED_REQUEST_BASED_IOS 256 +static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS; + +#ifdef CONFIG_DM_MQ_DEFAULT +static bool use_blk_mq = true; +#else +static bool use_blk_mq = false; +#endif + +bool dm_use_blk_mq_default(void) +{ + return use_blk_mq; +} + +bool dm_use_blk_mq(struct mapped_device *md) +{ + return md->use_blk_mq; +} +EXPORT_SYMBOL_GPL(dm_use_blk_mq); + +unsigned dm_get_reserved_rq_based_ios(void) +{ + return __dm_get_module_param(&reserved_rq_based_ios, + RESERVED_REQUEST_BASED_IOS, DM_RESERVED_MAX_IOS); +} +EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios); + +static unsigned dm_get_blk_mq_nr_hw_queues(void) +{ + return __dm_get_module_param(&dm_mq_nr_hw_queues, 1, 32); +} + +static unsigned dm_get_blk_mq_queue_depth(void) +{ + return __dm_get_module_param(&dm_mq_queue_depth, + DM_MQ_QUEUE_DEPTH, BLK_MQ_MAX_DEPTH); +} + +int dm_request_based(struct mapped_device *md) +{ + return blk_queue_stackable(md->queue); +} + +static void dm_old_start_queue(struct request_queue *q) +{ + unsigned long flags; + + spin_lock_irqsave(q->queue_lock, flags); + if (blk_queue_stopped(q)) + blk_start_queue(q); + spin_unlock_irqrestore(q->queue_lock, flags); +} + +void dm_start_queue(struct request_queue *q) +{ + if (!q->mq_ops) + dm_old_start_queue(q); + else { + blk_mq_start_stopped_hw_queues(q, true); + blk_mq_kick_requeue_list(q); + } +} + +static void dm_old_stop_queue(struct request_queue *q) +{ + unsigned long flags; + + spin_lock_irqsave(q->queue_lock, flags); + if (blk_queue_stopped(q)) { + spin_unlock_irqrestore(q->queue_lock, flags); + return; + } + + blk_stop_queue(q); + spin_unlock_irqrestore(q->queue_lock, flags); +} + +void dm_stop_queue(struct request_queue *q) +{ + if (!q->mq_ops) + dm_old_stop_queue(q); + else + blk_mq_stop_hw_queues(q); +} + +static struct dm_rq_target_io *alloc_old_rq_tio(struct mapped_device *md, + gfp_t gfp_mask) +{ + return mempool_alloc(md->io_pool, gfp_mask); +} + +static void free_old_rq_tio(struct dm_rq_target_io *tio) +{ + mempool_free(tio, tio->md->io_pool); +} + +static struct request *alloc_old_clone_request(struct mapped_device *md, + gfp_t gfp_mask) +{ + return mempool_alloc(md->rq_pool, gfp_mask); +} + +static void free_old_clone_request(struct mapped_device *md, struct request *rq) +{ + mempool_free(rq, md->rq_pool); +} + +/* + * Partial completion handling for request-based dm + */ +static void end_clone_bio(struct bio *clone) +{ + struct dm_rq_clone_bio_info *info = + container_of(clone, struct dm_rq_clone_bio_info, clone); + struct dm_rq_target_io *tio = info->tio; + struct bio *bio = info->orig; + unsigned int nr_bytes = info->orig->bi_iter.bi_size; + int error = clone->bi_error; + + bio_put(clone); + + if (tio->error) + /* + * An error has already been detected on the request. + * Once error occurred, just let clone->end_io() handle + * the remainder. + */ + return; + else if (error) { + /* + * Don't notice the error to the upper layer yet. + * The error handling decision is made by the target driver, + * when the request is completed. + */ + tio->error = error; + return; + } + + /* + * I/O for the bio successfully completed. + * Notice the data completion to the upper layer. + */ + + /* + * bios are processed from the head of the list. + * So the completing bio should always be rq->bio. + * If it's not, something wrong is happening. + */ + if (tio->orig->bio != bio) + DMERR("bio completion is going in the middle of the request"); + + /* + * Update the original request. + * Do not use blk_end_request() here, because it may complete + * the original request before the clone, and break the ordering. + */ + blk_update_request(tio->orig, 0, nr_bytes); +} + +static struct dm_rq_target_io *tio_from_request(struct request *rq) +{ + return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special); +} + +static void rq_end_stats(struct mapped_device *md, struct request *orig) +{ + if (unlikely(dm_stats_used(&md->stats))) { + struct dm_rq_target_io *tio = tio_from_request(orig); + tio->duration_jiffies = jiffies - tio->duration_jiffies; + dm_stats_account_io(&md->stats, rq_data_dir(orig), + blk_rq_pos(orig), tio->n_sectors, true, + tio->duration_jiffies, &tio->stats_aux); + } +} + +/* + * Don't touch any member of the md after calling this function because + * the md may be freed in dm_put() at the end of this function. + * Or do dm_get() before calling this function and dm_put() later. + */ +static void rq_completed(struct mapped_device *md, int rw, bool run_queue) +{ + atomic_dec(&md->pending[rw]); + + /* nudge anyone waiting on suspend queue */ + if (!md_in_flight(md)) + wake_up(&md->wait); + + /* + * Run this off this callpath, as drivers could invoke end_io while + * inside their request_fn (and holding the queue lock). Calling + * back into ->request_fn() could deadlock attempting to grab the + * queue lock again. + */ + if (!md->queue->mq_ops && run_queue) + blk_run_queue_async(md->queue); + + /* + * dm_put() must be at the end of this function. See the comment above + */ + dm_put(md); +} + +static void free_rq_clone(struct request *clone) +{ + struct dm_rq_target_io *tio = clone->end_io_data; + struct mapped_device *md = tio->md; + + blk_rq_unprep_clone(clone); + + /* + * It is possible for a clone_old_rq() allocated clone to + * get passed in -- it may not yet have a request_queue. + * This is known to occur if the error target replaces + * a multipath target that has a request_fn queue stacked + * on blk-mq queue(s). + */ + if (clone->q && clone->q->mq_ops) + /* stacked on blk-mq queue(s) */ + tio->ti->type->release_clone_rq(clone); + else if (!md->queue->mq_ops) + /* request_fn queue stacked on request_fn queue(s) */ + free_old_clone_request(md, clone); + + if (!md->queue->mq_ops) + free_old_rq_tio(tio); +} + +/* + * Complete the clone and the original request. + * Must be called without clone's queue lock held, + * see end_clone_request() for more details. + */ +static void dm_end_request(struct request *clone, int error) +{ + int rw = rq_data_dir(clone); + struct dm_rq_target_io *tio = clone->end_io_data; + struct mapped_device *md = tio->md; + struct request *rq = tio->orig; + + if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { + rq->errors = clone->errors; + rq->resid_len = clone->resid_len; + + if (rq->sense) + /* + * We are using the sense buffer of the original + * request. + * So setting the length of the sense data is enough. + */ + rq->sense_len = clone->sense_len; + } + + free_rq_clone(clone); + rq_end_stats(md, rq); + if (!rq->q->mq_ops) + blk_end_request_all(rq, error); + else + blk_mq_end_request(rq, error); + rq_completed(md, rw, true); +} + +static void dm_unprep_request(struct request *rq) +{ + struct dm_rq_target_io *tio = tio_from_request(rq); + struct request *clone = tio->clone; + + if (!rq->q->mq_ops) { + rq->special = NULL; + rq->cmd_flags &= ~REQ_DONTPREP; + } + + if (clone) + free_rq_clone(clone); + else if (!tio->md->queue->mq_ops) + free_old_rq_tio(tio); +} + +/* + * Requeue the original request of a clone. + */ +static void dm_old_requeue_request(struct request *rq) +{ + struct request_queue *q = rq->q; + unsigned long flags; + + spin_lock_irqsave(q->queue_lock, flags); + blk_requeue_request(q, rq); + blk_run_queue_async(q); + spin_unlock_irqrestore(q->queue_lock, flags); +} + +static void dm_mq_requeue_request(struct request *rq) +{ + struct request_queue *q = rq->q; + unsigned long flags; + + blk_mq_requeue_request(rq); + spin_lock_irqsave(q->queue_lock, flags); + if (!blk_queue_stopped(q)) + blk_mq_kick_requeue_list(q); + spin_unlock_irqrestore(q->queue_lock, flags); +} + +static void dm_requeue_original_request(struct mapped_device *md, + struct request *rq) +{ + int rw = rq_data_dir(rq); + + rq_end_stats(md, rq); + dm_unprep_request(rq); + + if (!rq->q->mq_ops) + dm_old_requeue_request(rq); + else + dm_mq_requeue_request(rq); + + rq_completed(md, rw, false); +} + +static void dm_done(struct request *clone, int error, bool mapped) +{ + int r = error; + struct dm_rq_target_io *tio = clone->end_io_data; + dm_request_endio_fn rq_end_io = NULL; + + if (tio->ti) { + rq_end_io = tio->ti->type->rq_end_io; + + if (mapped && rq_end_io) + r = rq_end_io(tio->ti, clone, error, &tio->info); + } + + if (unlikely(r == -EREMOTEIO && (req_op(clone) == REQ_OP_WRITE_SAME) && + !clone->q->limits.max_write_same_sectors)) + disable_write_same(tio->md); + + if (r <= 0) + /* The target wants to complete the I/O */ + dm_end_request(clone, r); + else if (r == DM_ENDIO_INCOMPLETE) + /* The target will handle the I/O */ + return; + else if (r == DM_ENDIO_REQUEUE) + /* The target wants to requeue the I/O */ + dm_requeue_original_request(tio->md, tio->orig); + else { + DMWARN("unimplemented target endio return value: %d", r); + BUG(); + } +} + +/* + * Request completion handler for request-based dm + */ +static void dm_softirq_done(struct request *rq) +{ + bool mapped = true; + struct dm_rq_target_io *tio = tio_from_request(rq); + struct request *clone = tio->clone; + int rw; + + if (!clone) { + rq_end_stats(tio->md, rq); + rw = rq_data_dir(rq); + if (!rq->q->mq_ops) { + blk_end_request_all(rq, tio->error); + rq_completed(tio->md, rw, false); + free_old_rq_tio(tio); + } else { + blk_mq_end_request(rq, tio->error); + rq_completed(tio->md, rw, false); + } + return; + } + + if (rq->cmd_flags & REQ_FAILED) + mapped = false; + + dm_done(clone, tio->error, mapped); +} + +/* + * Complete the clone and the original request with the error status + * through softirq context. + */ +static void dm_complete_request(struct request *rq, int error) +{ + struct dm_rq_target_io *tio = tio_from_request(rq); + + tio->error = error; + if (!rq->q->mq_ops) + blk_complete_request(rq); + else + blk_mq_complete_request(rq, error); +} + +/* + * Complete the not-mapped clone and the original request with the error status + * through softirq context. + * Target's rq_end_io() function isn't called. + * This may be used when the target's map_rq() or clone_and_map_rq() functions fail. + */ +static void dm_kill_unmapped_request(struct request *rq, int error) +{ + rq->cmd_flags |= REQ_FAILED; + dm_complete_request(rq, error); +} + +/* + * Called with the clone's queue lock held (in the case of .request_fn) + */ +static void end_clone_request(struct request *clone, int error) +{ + struct dm_rq_target_io *tio = clone->end_io_data; + + if (!clone->q->mq_ops) { + /* + * For just cleaning up the information of the queue in which + * the clone was dispatched. + * The clone is *NOT* freed actually here because it is alloced + * from dm own mempool (REQ_ALLOCED isn't set). + */ + __blk_put_request(clone->q, clone); + } + + /* + * Actual request completion is done in a softirq context which doesn't + * hold the clone's queue lock. Otherwise, deadlock could occur because: + * - another request may be submitted by the upper level driver + * of the stacking during the completion + * - the submission which requires queue lock may be done + * against this clone's queue + */ + dm_complete_request(tio->orig, error); +} + +static void dm_dispatch_clone_request(struct request *clone, struct request *rq) +{ + int r; + + if (blk_queue_io_stat(clone->q)) + clone->cmd_flags |= REQ_IO_STAT; + + clone->start_time = jiffies; + r = blk_insert_cloned_request(clone->q, clone); + if (r) + /* must complete clone in terms of original request */ + dm_complete_request(rq, r); +} + +static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig, + void *data) +{ + struct dm_rq_target_io *tio = data; + struct dm_rq_clone_bio_info *info = + container_of(bio, struct dm_rq_clone_bio_info, clone); + + info->orig = bio_orig; + info->tio = tio; + bio->bi_end_io = end_clone_bio; + + return 0; +} + +static int setup_clone(struct request *clone, struct request *rq, + struct dm_rq_target_io *tio, gfp_t gfp_mask) +{ + int r; + + r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask, + dm_rq_bio_constructor, tio); + if (r) + return r; + + clone->cmd = rq->cmd; + clone->cmd_len = rq->cmd_len; + clone->sense = rq->sense; + clone->end_io = end_clone_request; + clone->end_io_data = tio; + + tio->clone = clone; + + return 0; +} + +static struct request *clone_old_rq(struct request *rq, struct mapped_device *md, + struct dm_rq_target_io *tio, gfp_t gfp_mask) +{ + /* + * Create clone for use with .request_fn request_queue + */ + struct request *clone; + + clone = alloc_old_clone_request(md, gfp_mask); + if (!clone) + return NULL; + + blk_rq_init(NULL, clone); + if (setup_clone(clone, rq, tio, gfp_mask)) { + /* -ENOMEM */ + free_old_clone_request(md, clone); + return NULL; + } + + return clone; +} + +static void map_tio_request(struct kthread_work *work); + +static void init_tio(struct dm_rq_target_io *tio, struct request *rq, + struct mapped_device *md) +{ + tio->md = md; + tio->ti = NULL; + tio->clone = NULL; + tio->orig = rq; + tio->error = 0; + /* + * Avoid initializing info for blk-mq; it passes + * target-specific data through info.ptr + * (see: dm_mq_init_request) + */ + if (!md->init_tio_pdu) + memset(&tio->info, 0, sizeof(tio->info)); + if (md->kworker_task) + init_kthread_work(&tio->work, map_tio_request); +} + +static struct dm_rq_target_io *dm_old_prep_tio(struct request *rq, + struct mapped_device *md, + gfp_t gfp_mask) +{ + struct dm_rq_target_io *tio; + int srcu_idx; + struct dm_table *table; + + tio = alloc_old_rq_tio(md, gfp_mask); + if (!tio) + return NULL; + + init_tio(tio, rq, md); + + table = dm_get_live_table(md, &srcu_idx); + /* + * Must clone a request if this .request_fn DM device + * is stacked on .request_fn device(s). + */ + if (!dm_table_all_blk_mq_devices(table)) { + if (!clone_old_rq(rq, md, tio, gfp_mask)) { + dm_put_live_table(md, srcu_idx); + free_old_rq_tio(tio); + return NULL; + } + } + dm_put_live_table(md, srcu_idx); + + return tio; +} + +/* + * Called with the queue lock held. + */ +static int dm_old_prep_fn(struct request_queue *q, struct request *rq) +{ + struct mapped_device *md = q->queuedata; + struct dm_rq_target_io *tio; + + if (unlikely(rq->special)) { + DMWARN("Already has something in rq->special."); + return BLKPREP_KILL; + } + + tio = dm_old_prep_tio(rq, md, GFP_ATOMIC); + if (!tio) + return BLKPREP_DEFER; + + rq->special = tio; + rq->cmd_flags |= REQ_DONTPREP; + + return BLKPREP_OK; +} + +/* + * Returns: + * 0 : the request has been processed + * DM_MAPIO_REQUEUE : the original request needs to be requeued + * < 0 : the request was completed due to failure + */ +static int map_request(struct dm_rq_target_io *tio, struct request *rq, + struct mapped_device *md) +{ + int r; + struct dm_target *ti = tio->ti; + struct request *clone = NULL; + + if (tio->clone) { + clone = tio->clone; + r = ti->type->map_rq(ti, clone, &tio->info); + } else { + r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone); + if (r < 0) { + /* The target wants to complete the I/O */ + dm_kill_unmapped_request(rq, r); + return r; + } + if (r != DM_MAPIO_REMAPPED) + return r; + if (setup_clone(clone, rq, tio, GFP_ATOMIC)) { + /* -ENOMEM */ + ti->type->release_clone_rq(clone); + return DM_MAPIO_REQUEUE; + } + } + + switch (r) { + case DM_MAPIO_SUBMITTED: + /* The target has taken the I/O to submit by itself later */ + break; + case DM_MAPIO_REMAPPED: + /* The target has remapped the I/O so dispatch it */ + trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)), + blk_rq_pos(rq)); + dm_dispatch_clone_request(clone, rq); + break; + case DM_MAPIO_REQUEUE: + /* The target wants to requeue the I/O */ + dm_requeue_original_request(md, tio->orig); + break; + default: + if (r > 0) { + DMWARN("unimplemented target map return value: %d", r); + BUG(); + } + + /* The target wants to complete the I/O */ + dm_kill_unmapped_request(rq, r); + return r; + } + + return 0; +} + +static void dm_start_request(struct mapped_device *md, struct request *orig) +{ + if (!orig->q->mq_ops) + blk_start_request(orig); + else + blk_mq_start_request(orig); + atomic_inc(&md->pending[rq_data_dir(orig)]); + + if (md->seq_rq_merge_deadline_usecs) { + md->last_rq_pos = rq_end_sector(orig); + md->last_rq_rw = rq_data_dir(orig); + md->last_rq_start_time = ktime_get(); + } + + if (unlikely(dm_stats_used(&md->stats))) { + struct dm_rq_target_io *tio = tio_from_request(orig); + tio->duration_jiffies = jiffies; + tio->n_sectors = blk_rq_sectors(orig); + dm_stats_account_io(&md->stats, rq_data_dir(orig), + blk_rq_pos(orig), tio->n_sectors, false, 0, + &tio->stats_aux); + } + + /* + * Hold the md reference here for the in-flight I/O. + * We can't rely on the reference count by device opener, + * because the device may be closed during the request completion + * when all bios are completed. + * See the comment in rq_completed() too. + */ + dm_get(md); +} + +static void map_tio_request(struct kthread_work *work) +{ + struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work); + struct request *rq = tio->orig; + struct mapped_device *md = tio->md; + + if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) + dm_requeue_original_request(md, rq); +} + +ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf) +{ + return sprintf(buf, "%u\n", md->seq_rq_merge_deadline_usecs); +} + +#define MAX_SEQ_RQ_MERGE_DEADLINE_USECS 100000 + +ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md, + const char *buf, size_t count) +{ + unsigned deadline; + + if (dm_get_md_type(md) != DM_TYPE_REQUEST_BASED) + return count; + + if (kstrtouint(buf, 10, &deadline)) + return -EINVAL; + + if (deadline > MAX_SEQ_RQ_MERGE_DEADLINE_USECS) + deadline = MAX_SEQ_RQ_MERGE_DEADLINE_USECS; + + md->seq_rq_merge_deadline_usecs = deadline; + + return count; +} + +static bool dm_old_request_peeked_before_merge_deadline(struct mapped_device *md) +{ + ktime_t kt_deadline; + + if (!md->seq_rq_merge_deadline_usecs) + return false; + + kt_deadline = ns_to_ktime((u64)md->seq_rq_merge_deadline_usecs * NSEC_PER_USEC); + kt_deadline = ktime_add_safe(md->last_rq_start_time, kt_deadline); + + return !ktime_after(ktime_get(), kt_deadline); +} + +/* + * q->request_fn for old request-based dm. + * Called with the queue lock held. + */ +static void dm_old_request_fn(struct request_queue *q) +{ + struct mapped_device *md = q->queuedata; + struct dm_target *ti = md->immutable_target; + struct request *rq; + struct dm_rq_target_io *tio; + sector_t pos = 0; + + if (unlikely(!ti)) { + int srcu_idx; + struct dm_table *map = dm_get_live_table(md, &srcu_idx); + + ti = dm_table_find_target(map, pos); + dm_put_live_table(md, srcu_idx); + } + + /* + * For suspend, check blk_queue_stopped() and increment + * ->pending within a single queue_lock not to increment the + * number of in-flight I/Os after the queue is stopped in + * dm_suspend(). + */ + while (!blk_queue_stopped(q)) { + rq = blk_peek_request(q); + if (!rq) + return; + + /* always use block 0 to find the target for flushes for now */ + pos = 0; + if (req_op(rq) != REQ_OP_FLUSH) + pos = blk_rq_pos(rq); + + if ((dm_old_request_peeked_before_merge_deadline(md) && + md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 && + md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq)) || + (ti->type->busy && ti->type->busy(ti))) { + blk_delay_queue(q, 10); + return; + } + + dm_start_request(md, rq); + + tio = tio_from_request(rq); + /* Establish tio->ti before queuing work (map_tio_request) */ + tio->ti = ti; + queue_kthread_work(&md->kworker, &tio->work); + BUG_ON(!irqs_disabled()); + } +} + +/* + * Fully initialize a .request_fn request-based queue. + */ +int dm_old_init_request_queue(struct mapped_device *md) +{ + /* Fully initialize the queue */ + if (!blk_init_allocated_queue(md->queue, dm_old_request_fn, NULL)) + return -EINVAL; + + /* disable dm_old_request_fn's merge heuristic by default */ + md->seq_rq_merge_deadline_usecs = 0; + + dm_init_normal_md_queue(md); + blk_queue_softirq_done(md->queue, dm_softirq_done); + blk_queue_prep_rq(md->queue, dm_old_prep_fn); + + /* Initialize the request-based DM worker thread */ + init_kthread_worker(&md->kworker); + md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker, + "kdmwork-%s", dm_device_name(md)); + if (IS_ERR(md->kworker_task)) + return PTR_ERR(md->kworker_task); + + elv_register_queue(md->queue); + + return 0; +} + +static int dm_mq_init_request(void *data, struct request *rq, + unsigned int hctx_idx, unsigned int request_idx, + unsigned int numa_node) +{ + struct mapped_device *md = data; + struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); + + /* + * Must initialize md member of tio, otherwise it won't + * be available in dm_mq_queue_rq. + */ + tio->md = md; + + if (md->init_tio_pdu) { + /* target-specific per-io data is immediately after the tio */ + tio->info.ptr = tio + 1; + } + + return 0; +} + +static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) +{ + struct request *rq = bd->rq; + struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); + struct mapped_device *md = tio->md; + struct dm_target *ti = md->immutable_target; + + if (unlikely(!ti)) { + int srcu_idx; + struct dm_table *map = dm_get_live_table(md, &srcu_idx); + + ti = dm_table_find_target(map, 0); + dm_put_live_table(md, srcu_idx); + } + + if (ti->type->busy && ti->type->busy(ti)) + return BLK_MQ_RQ_QUEUE_BUSY; + + dm_start_request(md, rq); + + /* Init tio using md established in .init_request */ + init_tio(tio, rq, md); + + /* + * Establish tio->ti before calling map_request(). + */ + tio->ti = ti; + + /* Direct call is fine since .queue_rq allows allocations */ + if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) { + /* Undo dm_start_request() before requeuing */ + rq_end_stats(md, rq); + rq_completed(md, rq_data_dir(rq), false); + return BLK_MQ_RQ_QUEUE_BUSY; + } + + return BLK_MQ_RQ_QUEUE_OK; +} + +static struct blk_mq_ops dm_mq_ops = { + .queue_rq = dm_mq_queue_rq, + .map_queue = blk_mq_map_queue, + .complete = dm_softirq_done, + .init_request = dm_mq_init_request, +}; + +int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t) +{ + struct request_queue *q; + struct dm_target *immutable_tgt; + int err; + + if (!dm_table_all_blk_mq_devices(t)) { + DMERR("request-based dm-mq may only be stacked on blk-mq device(s)"); + return -EINVAL; + } + + md->tag_set = kzalloc_node(sizeof(struct blk_mq_tag_set), GFP_KERNEL, md->numa_node_id); + if (!md->tag_set) + return -ENOMEM; + + md->tag_set->ops = &dm_mq_ops; + md->tag_set->queue_depth = dm_get_blk_mq_queue_depth(); + md->tag_set->numa_node = md->numa_node_id; + md->tag_set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; + md->tag_set->nr_hw_queues = dm_get_blk_mq_nr_hw_queues(); + md->tag_set->driver_data = md; + + md->tag_set->cmd_size = sizeof(struct dm_rq_target_io); + immutable_tgt = dm_table_get_immutable_target(t); + if (immutable_tgt && immutable_tgt->per_io_data_size) { + /* any target-specific per-io data is immediately after the tio */ + md->tag_set->cmd_size += immutable_tgt->per_io_data_size; + md->init_tio_pdu = true; + } + + err = blk_mq_alloc_tag_set(md->tag_set); + if (err) + goto out_kfree_tag_set; + + q = blk_mq_init_allocated_queue(md->tag_set, md->queue); + if (IS_ERR(q)) { + err = PTR_ERR(q); + goto out_tag_set; + } + dm_init_md_queue(md); + + /* backfill 'mq' sysfs registration normally done in blk_register_queue */ + blk_mq_register_disk(md->disk); + + return 0; + +out_tag_set: + blk_mq_free_tag_set(md->tag_set); +out_kfree_tag_set: + kfree(md->tag_set); + + return err; +} + +void dm_mq_cleanup_mapped_device(struct mapped_device *md) +{ + if (md->tag_set) { + blk_mq_free_tag_set(md->tag_set); + kfree(md->tag_set); + } +} + +module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools"); + +module_param(use_blk_mq, bool, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(use_blk_mq, "Use block multiqueue for request-based DM devices"); + +module_param(dm_mq_nr_hw_queues, uint, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(dm_mq_nr_hw_queues, "Number of hardware queues for request-based dm-mq devices"); + +module_param(dm_mq_queue_depth, uint, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(dm_mq_queue_depth, "Queue depth for request-based dm-mq devices"); diff --git a/drivers/md/dm-rq.h b/drivers/md/dm-rq.h new file mode 100644 index 000000000000..9e6f0a3773d4 --- /dev/null +++ b/drivers/md/dm-rq.h @@ -0,0 +1,64 @@ +/* + * Internal header file for device mapper + * + * Copyright (C) 2016 Red Hat, Inc. All rights reserved. + * + * This file is released under the LGPL. + */ + +#ifndef DM_RQ_INTERNAL_H +#define DM_RQ_INTERNAL_H + +#include +#include + +#include "dm-stats.h" + +struct mapped_device; + +/* + * One of these is allocated per request. + */ +struct dm_rq_target_io { + struct mapped_device *md; + struct dm_target *ti; + struct request *orig, *clone; + struct kthread_work work; + int error; + union map_info info; + struct dm_stats_aux stats_aux; + unsigned long duration_jiffies; + unsigned n_sectors; +}; + +/* + * For request-based dm - the bio clones we allocate are embedded in these + * structs. + * + * We allocate these with bio_alloc_bioset, using the front_pad parameter when + * the bioset is created - this means the bio has to come at the end of the + * struct. + */ +struct dm_rq_clone_bio_info { + struct bio *orig; + struct dm_rq_target_io *tio; + struct bio clone; +}; + +bool dm_use_blk_mq_default(void); +bool dm_use_blk_mq(struct mapped_device *md); + +int dm_old_init_request_queue(struct mapped_device *md); +int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t); +void dm_mq_cleanup_mapped_device(struct mapped_device *md); + +void dm_start_queue(struct request_queue *q); +void dm_stop_queue(struct request_queue *q); + +unsigned dm_get_reserved_rq_based_ios(void); + +ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf); +ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md, + const char *buf, size_t count); + +#endif diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c index 4d3909393f2c..b8cf956b577b 100644 --- a/drivers/md/dm-snap-persistent.c +++ b/drivers/md/dm-snap-persistent.c @@ -226,8 +226,8 @@ static void do_metadata(struct work_struct *work) /* * Read or write a chunk aligned and sized block of data from a device. */ -static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, int rw, - int metadata) +static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, int op, + int op_flags, int metadata) { struct dm_io_region where = { .bdev = dm_snap_cow(ps->store->snap)->bdev, @@ -235,7 +235,8 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, int rw, .count = ps->store->chunk_size, }; struct dm_io_request io_req = { - .bi_rw = rw, + .bi_op = op, + .bi_op_flags = op_flags, .mem.type = DM_IO_VMA, .mem.ptr.vma = area, .client = ps->io_client, @@ -281,14 +282,14 @@ static void skip_metadata(struct pstore *ps) * Read or write a metadata area. Remembering to skip the first * chunk which holds the header. */ -static int area_io(struct pstore *ps, int rw) +static int area_io(struct pstore *ps, int op, int op_flags) { int r; chunk_t chunk; chunk = area_location(ps, ps->current_area); - r = chunk_io(ps, ps->area, chunk, rw, 0); + r = chunk_io(ps, ps->area, chunk, op, op_flags, 0); if (r) return r; @@ -302,7 +303,8 @@ static void zero_memory_area(struct pstore *ps) static int zero_disk_area(struct pstore *ps, chunk_t area) { - return chunk_io(ps, ps->zero_area, area_location(ps, area), WRITE, 0); + return chunk_io(ps, ps->zero_area, area_location(ps, area), + REQ_OP_WRITE, 0, 0); } static int read_header(struct pstore *ps, int *new_snapshot) @@ -334,7 +336,7 @@ static int read_header(struct pstore *ps, int *new_snapshot) if (r) return r; - r = chunk_io(ps, ps->header_area, 0, READ, 1); + r = chunk_io(ps, ps->header_area, 0, REQ_OP_READ, 0, 1); if (r) goto bad; @@ -395,7 +397,7 @@ static int write_header(struct pstore *ps) dh->version = cpu_to_le32(ps->version); dh->chunk_size = cpu_to_le32(ps->store->chunk_size); - return chunk_io(ps, ps->header_area, 0, WRITE, 1); + return chunk_io(ps, ps->header_area, 0, REQ_OP_WRITE, 0, 1); } /* @@ -739,7 +741,7 @@ static void persistent_commit_exception(struct dm_exception_store *store, /* * Commit exceptions to disk. */ - if (ps->valid && area_io(ps, WRITE_FLUSH_FUA)) + if (ps->valid && area_io(ps, REQ_OP_WRITE, WRITE_FLUSH_FUA)) ps->valid = 0; /* @@ -779,7 +781,7 @@ static int persistent_prepare_merge(struct dm_exception_store *store, return 0; ps->current_area--; - r = area_io(ps, READ); + r = area_io(ps, REQ_OP_READ, 0); if (r < 0) return r; ps->current_committed = ps->exceptions_per_area; @@ -816,7 +818,7 @@ static int persistent_commit_merge(struct dm_exception_store *store, for (i = 0; i < nr_merged; i++) clear_exception(ps, ps->current_committed - 1 - i); - r = area_io(ps, WRITE_FLUSH_FUA); + r = area_io(ps, REQ_OP_WRITE, WRITE_FLUSH_FUA); if (r < 0) return r; diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 70bb0e8b62ce..731e1f5bd895 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -1680,7 +1680,7 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio) init_tracked_chunk(bio); - if (bio->bi_rw & REQ_FLUSH) { + if (bio->bi_rw & REQ_PREFLUSH) { bio->bi_bdev = s->cow->bdev; return DM_MAPIO_REMAPPED; } @@ -1696,7 +1696,8 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio) * to copy an exception */ down_write(&s->lock); - if (!s->valid || (unlikely(s->snapshot_overflowed) && bio_rw(bio) == WRITE)) { + if (!s->valid || (unlikely(s->snapshot_overflowed) && + bio_data_dir(bio) == WRITE)) { r = -EIO; goto out_unlock; } @@ -1713,7 +1714,7 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio) * flags so we should only get this if we are * writeable. */ - if (bio_rw(bio) == WRITE) { + if (bio_data_dir(bio) == WRITE) { pe = __lookup_pending_exception(s, chunk); if (!pe) { up_write(&s->lock); @@ -1799,7 +1800,7 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio) init_tracked_chunk(bio); - if (bio->bi_rw & REQ_FLUSH) { + if (bio->bi_rw & REQ_PREFLUSH) { if (!dm_bio_get_target_bio_nr(bio)) bio->bi_bdev = s->origin->bdev; else @@ -1819,7 +1820,7 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio) e = dm_lookup_exception(&s->complete, chunk); if (e) { /* Queue writes overlapping with chunks being merged */ - if (bio_rw(bio) == WRITE && + if (bio_data_dir(bio) == WRITE && chunk >= s->first_merging_chunk && chunk < (s->first_merging_chunk + s->num_merging_chunks)) { @@ -1831,7 +1832,7 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio) remap_exception(s, e, bio, chunk); - if (bio_rw(bio) == WRITE) + if (bio_data_dir(bio) == WRITE) track_chunk(s, bio, chunk); goto out_unlock; } @@ -1839,7 +1840,7 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio) redirect_to_origin: bio->bi_bdev = s->origin->bdev; - if (bio_rw(bio) == WRITE) { + if (bio_data_dir(bio) == WRITE) { up_write(&s->lock); return do_origin(s->origin, bio); } @@ -2285,10 +2286,10 @@ static int origin_map(struct dm_target *ti, struct bio *bio) bio->bi_bdev = o->dev->bdev; - if (unlikely(bio->bi_rw & REQ_FLUSH)) + if (unlikely(bio->bi_rw & REQ_PREFLUSH)) return DM_MAPIO_REMAPPED; - if (bio_rw(bio) != WRITE) + if (bio_data_dir(bio) != WRITE) return DM_MAPIO_REMAPPED; available_sectors = o->split_boundary - @@ -2301,6 +2302,13 @@ static int origin_map(struct dm_target *ti, struct bio *bio) return do_origin(o->dev, bio); } +static long origin_direct_access(struct dm_target *ti, sector_t sector, + void __pmem **kaddr, pfn_t *pfn, long size) +{ + DMWARN("device does not support dax."); + return -EIO; +} + /* * Set the target "max_io_len" field to the minimum of all the snapshots' * chunk sizes. @@ -2360,6 +2368,7 @@ static struct target_type origin_target = { .postsuspend = origin_postsuspend, .status = origin_status, .iterate_devices = origin_iterate_devices, + .direct_access = origin_direct_access, }; static struct target_type snapshot_target = { diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c index 8289804ccd99..38b05f23b96c 100644 --- a/drivers/md/dm-stats.c +++ b/drivers/md/dm-stats.c @@ -10,7 +10,7 @@ #include #include -#include "dm.h" +#include "dm-core.h" #include "dm-stats.h" #define DM_MSG_PREFIX "stats" @@ -514,11 +514,10 @@ static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared, } static void dm_stat_for_entry(struct dm_stat *s, size_t entry, - unsigned long bi_rw, sector_t len, + int idx, sector_t len, struct dm_stats_aux *stats_aux, bool end, unsigned long duration_jiffies) { - unsigned long idx = bi_rw & REQ_WRITE; struct dm_stat_shared *shared = &s->stat_shared[entry]; struct dm_stat_percpu *p; @@ -584,7 +583,7 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry, #endif } -static void __dm_stat_bio(struct dm_stat *s, unsigned long bi_rw, +static void __dm_stat_bio(struct dm_stat *s, int bi_rw, sector_t bi_sector, sector_t end_sector, bool end, unsigned long duration_jiffies, struct dm_stats_aux *stats_aux) @@ -645,8 +644,8 @@ void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw, last = raw_cpu_ptr(stats->last); stats_aux->merged = (bi_sector == (ACCESS_ONCE(last->last_sector) && - ((bi_rw & (REQ_WRITE | REQ_DISCARD)) == - (ACCESS_ONCE(last->last_rw) & (REQ_WRITE | REQ_DISCARD))) + ((bi_rw == WRITE) == + (ACCESS_ONCE(last->last_rw) == WRITE)) )); ACCESS_ONCE(last->last_sector) = end_sector; ACCESS_ONCE(last->last_rw) = bi_rw; diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c index 797ddb900b06..01bb9cf2a8c2 100644 --- a/drivers/md/dm-stripe.c +++ b/drivers/md/dm-stripe.c @@ -286,14 +286,14 @@ static int stripe_map(struct dm_target *ti, struct bio *bio) uint32_t stripe; unsigned target_bio_nr; - if (bio->bi_rw & REQ_FLUSH) { + if (bio->bi_rw & REQ_PREFLUSH) { target_bio_nr = dm_bio_get_target_bio_nr(bio); BUG_ON(target_bio_nr >= sc->stripes); bio->bi_bdev = sc->stripe[target_bio_nr].dev->bdev; return DM_MAPIO_REMAPPED; } - if (unlikely(bio->bi_rw & REQ_DISCARD) || - unlikely(bio->bi_rw & REQ_WRITE_SAME)) { + if (unlikely(bio_op(bio) == REQ_OP_DISCARD) || + unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) { target_bio_nr = dm_bio_get_target_bio_nr(bio); BUG_ON(target_bio_nr >= sc->stripes); return stripe_map_range(sc, bio, target_bio_nr); @@ -308,6 +308,29 @@ static int stripe_map(struct dm_target *ti, struct bio *bio) return DM_MAPIO_REMAPPED; } +static long stripe_direct_access(struct dm_target *ti, sector_t sector, + void __pmem **kaddr, pfn_t *pfn, long size) +{ + struct stripe_c *sc = ti->private; + uint32_t stripe; + struct block_device *bdev; + struct blk_dax_ctl dax = { + .size = size, + }; + long ret; + + stripe_map_sector(sc, sector, &stripe, &dax.sector); + + dax.sector += sc->stripe[stripe].physical_start; + bdev = sc->stripe[stripe].dev->bdev; + + ret = bdev_direct_access(bdev, &dax); + *kaddr = dax.addr; + *pfn = dax.pfn; + + return ret; +} + /* * Stripe status: * @@ -416,7 +439,7 @@ static void stripe_io_hints(struct dm_target *ti, static struct target_type stripe_target = { .name = "striped", - .version = {1, 5, 1}, + .version = {1, 6, 0}, .module = THIS_MODULE, .ctr = stripe_ctr, .dtr = stripe_dtr, @@ -425,6 +448,7 @@ static struct target_type stripe_target = { .status = stripe_status, .iterate_devices = stripe_iterate_devices, .io_hints = stripe_io_hints, + .direct_access = stripe_direct_access, }; int __init dm_stripe_init(void) diff --git a/drivers/md/dm-sysfs.c b/drivers/md/dm-sysfs.c index 7e818f5f1dc4..c209b8a19b84 100644 --- a/drivers/md/dm-sysfs.c +++ b/drivers/md/dm-sysfs.c @@ -6,7 +6,8 @@ #include #include -#include "dm.h" +#include "dm-core.h" +#include "dm-rq.h" struct dm_sysfs_attr { struct attribute attr; diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 626a5ec04466..3e407a9cde1f 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -5,7 +5,7 @@ * This file is released under the GPL. */ -#include "dm.h" +#include "dm-core.h" #include #include @@ -43,8 +43,10 @@ struct dm_table { struct dm_target *targets; struct target_type *immutable_target_type; - unsigned integrity_supported:1; - unsigned singleton:1; + + bool integrity_supported:1; + bool singleton:1; + bool all_blk_mq:1; /* * Indicates the rw permissions for the new logical @@ -206,6 +208,7 @@ int dm_table_create(struct dm_table **result, fmode_t mode, return -ENOMEM; } + t->type = DM_TYPE_NONE; t->mode = mode; t->md = md; *result = t; @@ -703,7 +706,7 @@ int dm_table_add_target(struct dm_table *t, const char *type, dm_device_name(t->md), type); return -EINVAL; } - t->singleton = 1; + t->singleton = true; } if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) { @@ -824,22 +827,70 @@ void dm_consume_args(struct dm_arg_set *as, unsigned num_args) } EXPORT_SYMBOL(dm_consume_args); +static bool __table_type_bio_based(unsigned table_type) +{ + return (table_type == DM_TYPE_BIO_BASED || + table_type == DM_TYPE_DAX_BIO_BASED); +} + static bool __table_type_request_based(unsigned table_type) { return (table_type == DM_TYPE_REQUEST_BASED || table_type == DM_TYPE_MQ_REQUEST_BASED); } -static int dm_table_set_type(struct dm_table *t) +void dm_table_set_type(struct dm_table *t, unsigned type) +{ + t->type = type; +} +EXPORT_SYMBOL_GPL(dm_table_set_type); + +static int device_supports_dax(struct dm_target *ti, struct dm_dev *dev, + sector_t start, sector_t len, void *data) +{ + struct request_queue *q = bdev_get_queue(dev->bdev); + + return q && blk_queue_dax(q); +} + +static bool dm_table_supports_dax(struct dm_table *t) +{ + struct dm_target *ti; + unsigned i = 0; + + /* Ensure that all targets support DAX. */ + while (i < dm_table_get_num_targets(t)) { + ti = dm_table_get_target(t, i++); + + if (!ti->type->direct_access) + return false; + + if (!ti->type->iterate_devices || + !ti->type->iterate_devices(ti, device_supports_dax, NULL)) + return false; + } + + return true; +} + +static int dm_table_determine_type(struct dm_table *t) { unsigned i; unsigned bio_based = 0, request_based = 0, hybrid = 0; - bool use_blk_mq = false; + bool verify_blk_mq = false; struct dm_target *tgt; struct dm_dev_internal *dd; - struct list_head *devices; + struct list_head *devices = dm_table_get_devices(t); unsigned live_md_type = dm_get_md_type(t->md); + if (t->type != DM_TYPE_NONE) { + /* target already set the table's type */ + if (t->type == DM_TYPE_BIO_BASED) + return 0; + BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED); + goto verify_rq_based; + } + for (i = 0; i < t->num_targets; i++) { tgt = t->targets + i; if (dm_target_hybrid(tgt)) @@ -871,11 +922,27 @@ static int dm_table_set_type(struct dm_table *t) if (bio_based) { /* We must use this table as bio-based */ t->type = DM_TYPE_BIO_BASED; + if (dm_table_supports_dax(t) || + (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) + t->type = DM_TYPE_DAX_BIO_BASED; return 0; } BUG_ON(!request_based); /* No targets in this table */ + if (list_empty(devices) && __table_type_request_based(live_md_type)) { + /* inherit live MD type */ + t->type = live_md_type; + return 0; + } + + /* + * The only way to establish DM_TYPE_MQ_REQUEST_BASED is by + * having a compatible target use dm_table_set_type. + */ + t->type = DM_TYPE_REQUEST_BASED; + +verify_rq_based: /* * Request-based dm supports only tables that have a single target now. * To support multiple targets, request splitting support is needed, @@ -888,7 +955,6 @@ static int dm_table_set_type(struct dm_table *t) } /* Non-request-stackable devices can't be used for request-based dm */ - devices = dm_table_get_devices(t); list_for_each_entry(dd, devices, list) { struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev); @@ -899,10 +965,10 @@ static int dm_table_set_type(struct dm_table *t) } if (q->mq_ops) - use_blk_mq = true; + verify_blk_mq = true; } - if (use_blk_mq) { + if (verify_blk_mq) { /* verify _all_ devices in the table are blk-mq devices */ list_for_each_entry(dd, devices, list) if (!bdev_get_queue(dd->dm_dev->bdev)->mq_ops) { @@ -910,14 +976,9 @@ static int dm_table_set_type(struct dm_table *t) " are blk-mq request-stackable"); return -EINVAL; } - t->type = DM_TYPE_MQ_REQUEST_BASED; - } else if (list_empty(devices) && __table_type_request_based(live_md_type)) { - /* inherit live MD type */ - t->type = live_md_type; - - } else - t->type = DM_TYPE_REQUEST_BASED; + t->all_blk_mq = true; + } return 0; } @@ -956,14 +1017,19 @@ struct dm_target *dm_table_get_wildcard_target(struct dm_table *t) return NULL; } +bool dm_table_bio_based(struct dm_table *t) +{ + return __table_type_bio_based(dm_table_get_type(t)); +} + bool dm_table_request_based(struct dm_table *t) { return __table_type_request_based(dm_table_get_type(t)); } -bool dm_table_mq_request_based(struct dm_table *t) +bool dm_table_all_blk_mq_devices(struct dm_table *t) { - return dm_table_get_type(t) == DM_TYPE_MQ_REQUEST_BASED; + return t->all_blk_mq; } static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md) @@ -978,7 +1044,7 @@ static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device * return -EINVAL; } - if (type == DM_TYPE_BIO_BASED) + if (__table_type_bio_based(type)) for (i = 0; i < t->num_targets; i++) { tgt = t->targets + i; per_io_data_size = max(per_io_data_size, tgt->per_io_data_size); @@ -1106,7 +1172,7 @@ static int dm_table_register_integrity(struct dm_table *t) return 0; if (!integrity_profile_exists(dm_disk(md))) { - t->integrity_supported = 1; + t->integrity_supported = true; /* * Register integrity profile during table load; we can do * this because the final profile must match during resume. @@ -1129,7 +1195,7 @@ static int dm_table_register_integrity(struct dm_table *t) } /* Preserve existing integrity profile */ - t->integrity_supported = 1; + t->integrity_supported = true; return 0; } @@ -1141,9 +1207,9 @@ int dm_table_complete(struct dm_table *t) { int r; - r = dm_table_set_type(t); + r = dm_table_determine_type(t); if (r) { - DMERR("unable to set table type"); + DMERR("unable to determine table type"); return r; } diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c index a317dd884ba6..6eecd6b36f76 100644 --- a/drivers/md/dm-target.c +++ b/drivers/md/dm-target.c @@ -4,7 +4,7 @@ * This file is released under the GPL. */ -#include "dm.h" +#include "dm-core.h" #include #include @@ -148,9 +148,15 @@ static void io_err_release_clone_rq(struct request *clone) { } +static long io_err_direct_access(struct dm_target *ti, sector_t sector, + void __pmem **kaddr, pfn_t *pfn, long size) +{ + return -EIO; +} + static struct target_type error_target = { .name = "error", - .version = {1, 4, 0}, + .version = {1, 5, 0}, .features = DM_TARGET_WILDCARD, .ctr = io_err_ctr, .dtr = io_err_dtr, @@ -158,6 +164,7 @@ static struct target_type error_target = { .map_rq = io_err_map_rq, .clone_and_map_rq = io_err_clone_and_map_rq, .release_clone_rq = io_err_release_clone_rq, + .direct_access = io_err_direct_access, }; int __init dm_target_init(void) diff --git a/drivers/md/dm-thin-metadata.c b/drivers/md/dm-thin-metadata.c index 43824d73366d..a15091a0d40c 100644 --- a/drivers/md/dm-thin-metadata.c +++ b/drivers/md/dm-thin-metadata.c @@ -1677,6 +1677,36 @@ int dm_pool_block_is_used(struct dm_pool_metadata *pmd, dm_block_t b, bool *resu return r; } +int dm_pool_inc_data_range(struct dm_pool_metadata *pmd, dm_block_t b, dm_block_t e) +{ + int r = 0; + + down_write(&pmd->root_lock); + for (; b != e; b++) { + r = dm_sm_inc_block(pmd->data_sm, b); + if (r) + break; + } + up_write(&pmd->root_lock); + + return r; +} + +int dm_pool_dec_data_range(struct dm_pool_metadata *pmd, dm_block_t b, dm_block_t e) +{ + int r = 0; + + down_write(&pmd->root_lock); + for (; b != e; b++) { + r = dm_sm_dec_block(pmd->data_sm, b); + if (r) + break; + } + up_write(&pmd->root_lock); + + return r; +} + bool dm_thin_changed_this_transaction(struct dm_thin_device *td) { int r; diff --git a/drivers/md/dm-thin-metadata.h b/drivers/md/dm-thin-metadata.h index a938babe4258..35e954ea20a9 100644 --- a/drivers/md/dm-thin-metadata.h +++ b/drivers/md/dm-thin-metadata.h @@ -197,6 +197,9 @@ int dm_pool_get_data_dev_size(struct dm_pool_metadata *pmd, dm_block_t *result); int dm_pool_block_is_used(struct dm_pool_metadata *pmd, dm_block_t b, bool *result); +int dm_pool_inc_data_range(struct dm_pool_metadata *pmd, dm_block_t b, dm_block_t e); +int dm_pool_dec_data_range(struct dm_pool_metadata *pmd, dm_block_t b, dm_block_t e); + /* * Returns -ENOSPC if the new size is too small and already allocated * blocks would be lost. diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index fc803d50f9f0..197ea2003400 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -253,6 +253,7 @@ struct pool { struct bio_list deferred_flush_bios; struct list_head prepared_mappings; struct list_head prepared_discards; + struct list_head prepared_discards_pt2; struct list_head active_thins; struct dm_deferred_set *shared_read_ds; @@ -269,6 +270,7 @@ struct pool { process_mapping_fn process_prepared_mapping; process_mapping_fn process_prepared_discard; + process_mapping_fn process_prepared_discard_pt2; struct dm_bio_prison_cell **cell_sort_array; }; @@ -360,7 +362,7 @@ static int issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t da sector_t len = block_to_sectors(tc->pool, data_e - data_b); return __blkdev_issue_discard(tc->pool_dev->bdev, s, len, - GFP_NOWAIT, REQ_WRITE | REQ_DISCARD, &op->bio); + GFP_NOWAIT, 0, &op->bio); } static void end_discard(struct discard_op *op, int r) @@ -371,7 +373,8 @@ static void end_discard(struct discard_op *op, int r) * need to wait for the chain to complete. */ bio_chain(op->bio, op->parent_bio); - submit_bio(REQ_WRITE | REQ_DISCARD, op->bio); + bio_set_op_attrs(op->bio, REQ_OP_DISCARD, 0); + submit_bio(op->bio); } blk_finish_plug(&op->plug); @@ -696,7 +699,7 @@ static void remap_to_origin(struct thin_c *tc, struct bio *bio) static int bio_triggers_commit(struct thin_c *tc, struct bio *bio) { - return (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && + return (bio->bi_rw & (REQ_PREFLUSH | REQ_FUA)) && dm_thin_changed_this_transaction(tc->td); } @@ -704,7 +707,7 @@ static void inc_all_io_entry(struct pool *pool, struct bio *bio) { struct dm_thin_endio_hook *h; - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) return; h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); @@ -867,7 +870,8 @@ static void __inc_remap_and_issue_cell(void *context, struct bio *bio; while ((bio = bio_list_pop(&cell->bios))) { - if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) + if (bio->bi_rw & (REQ_PREFLUSH | REQ_FUA) || + bio_op(bio) == REQ_OP_DISCARD) bio_list_add(&info->defer_bios, bio); else { inc_all_io_entry(info->tc->pool, bio); @@ -999,7 +1003,8 @@ static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m) /*----------------------------------------------------------------*/ -static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m) +static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m, + struct bio *discard_parent) { /* * We've already unmapped this range of blocks, but before we @@ -1012,7 +1017,7 @@ static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin; struct discard_op op; - begin_discard(&op, tc, m->bio); + begin_discard(&op, tc, discard_parent); while (b != end) { /* find start of unmapped run */ for (; b < end; b++) { @@ -1047,28 +1052,101 @@ out: end_discard(&op, r); } -static void process_prepared_discard_passdown(struct dm_thin_new_mapping *m) +static void queue_passdown_pt2(struct dm_thin_new_mapping *m) +{ + unsigned long flags; + struct pool *pool = m->tc->pool; + + spin_lock_irqsave(&pool->lock, flags); + list_add_tail(&m->list, &pool->prepared_discards_pt2); + spin_unlock_irqrestore(&pool->lock, flags); + wake_worker(pool); +} + +static void passdown_endio(struct bio *bio) +{ + /* + * It doesn't matter if the passdown discard failed, we still want + * to unmap (we ignore err). + */ + queue_passdown_pt2(bio->bi_private); +} + +static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m) { int r; struct thin_c *tc = m->tc; struct pool *pool = tc->pool; + struct bio *discard_parent; + dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin); + /* + * Only this thread allocates blocks, so we can be sure that the + * newly unmapped blocks will not be allocated before the end of + * the function. + */ r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end); if (r) { metadata_operation_failed(pool, "dm_thin_remove_range", r); bio_io_error(m->bio); + cell_defer_no_holder(tc, m->cell); + mempool_free(m, pool->mapping_pool); + return; + } - } else if (m->maybe_shared) { - passdown_double_checking_shared_status(m); + discard_parent = bio_alloc(GFP_NOIO, 1); + if (!discard_parent) { + DMWARN("%s: unable to allocate top level discard bio for passdown. Skipping passdown.", + dm_device_name(tc->pool->pool_md)); + queue_passdown_pt2(m); } else { - struct discard_op op; - begin_discard(&op, tc, m->bio); - r = issue_discard(&op, m->data_block, - m->data_block + (m->virt_end - m->virt_begin)); - end_discard(&op, r); + discard_parent->bi_end_io = passdown_endio; + discard_parent->bi_private = m; + + if (m->maybe_shared) + passdown_double_checking_shared_status(m, discard_parent); + else { + struct discard_op op; + + begin_discard(&op, tc, discard_parent); + r = issue_discard(&op, m->data_block, data_end); + end_discard(&op, r); + } } + /* + * Increment the unmapped blocks. This prevents a race between the + * passdown io and reallocation of freed blocks. + */ + r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end); + if (r) { + metadata_operation_failed(pool, "dm_pool_inc_data_range", r); + bio_io_error(m->bio); + cell_defer_no_holder(tc, m->cell); + mempool_free(m, pool->mapping_pool); + return; + } +} + +static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m) +{ + int r; + struct thin_c *tc = m->tc; + struct pool *pool = tc->pool; + + /* + * The passdown has completed, so now we can decrement all those + * unmapped blocks. + */ + r = dm_pool_dec_data_range(pool->pmd, m->data_block, + m->data_block + (m->virt_end - m->virt_begin)); + if (r) { + metadata_operation_failed(pool, "dm_pool_dec_data_range", r); + bio_io_error(m->bio); + } else + bio_endio(m->bio); + cell_defer_no_holder(tc, m->cell); mempool_free(m, pool->mapping_pool); } @@ -1639,7 +1717,8 @@ static void __remap_and_issue_shared_cell(void *context, while ((bio = bio_list_pop(&cell->bios))) { if ((bio_data_dir(bio) == WRITE) || - (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA))) + (bio->bi_rw & (REQ_PREFLUSH | REQ_FUA) || + bio_op(bio) == REQ_OP_DISCARD)) bio_list_add(&info->defer_bios, bio); else { struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));; @@ -2028,7 +2107,7 @@ static void process_thin_deferred_bios(struct thin_c *tc) break; } - if (bio->bi_rw & REQ_DISCARD) + if (bio_op(bio) == REQ_OP_DISCARD) pool->process_discard(tc, bio); else pool->process_bio(tc, bio); @@ -2115,7 +2194,7 @@ static void process_thin_deferred_cells(struct thin_c *tc) return; } - if (cell->holder->bi_rw & REQ_DISCARD) + if (bio_op(cell->holder) == REQ_OP_DISCARD) pool->process_discard_cell(tc, cell); else pool->process_cell(tc, cell); @@ -2212,6 +2291,8 @@ static void do_worker(struct work_struct *ws) throttle_work_update(&pool->throttle); process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard); throttle_work_update(&pool->throttle); + process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2); + throttle_work_update(&pool->throttle); process_deferred_bios(pool); throttle_work_complete(&pool->throttle); } @@ -2340,7 +2421,8 @@ static void set_discard_callbacks(struct pool *pool) if (passdown_enabled(pt)) { pool->process_discard_cell = process_discard_cell_passdown; - pool->process_prepared_discard = process_prepared_discard_passdown; + pool->process_prepared_discard = process_prepared_discard_passdown_pt1; + pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2; } else { pool->process_discard_cell = process_discard_cell_no_passdown; pool->process_prepared_discard = process_prepared_discard_no_passdown; @@ -2553,7 +2635,8 @@ static int thin_bio_map(struct dm_target *ti, struct bio *bio) return DM_MAPIO_SUBMITTED; } - if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) { + if (bio->bi_rw & (REQ_PREFLUSH | REQ_FUA) || + bio_op(bio) == REQ_OP_DISCARD) { thin_defer_bio_with_throttle(tc, bio); return DM_MAPIO_SUBMITTED; } @@ -2826,6 +2909,7 @@ static struct pool *pool_create(struct mapped_device *pool_md, bio_list_init(&pool->deferred_flush_bios); INIT_LIST_HEAD(&pool->prepared_mappings); INIT_LIST_HEAD(&pool->prepared_discards); + INIT_LIST_HEAD(&pool->prepared_discards_pt2); INIT_LIST_HEAD(&pool->active_thins); pool->low_water_triggered = false; pool->suspended = true; diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 459a9f8905ed..0f0eb8a3d922 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -453,9 +453,7 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, */ offset = block << v->data_dev_block_bits; - - res = offset; - div64_u64(res, v->fec->rounds << v->data_dev_block_bits); + res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); /* * The base RS block we can feed to the interleaver to find out all diff --git a/drivers/md/dm-zero.c b/drivers/md/dm-zero.c index 766bc93006e6..618b8752dcf1 100644 --- a/drivers/md/dm-zero.c +++ b/drivers/md/dm-zero.c @@ -35,16 +35,19 @@ static int zero_ctr(struct dm_target *ti, unsigned int argc, char **argv) */ static int zero_map(struct dm_target *ti, struct bio *bio) { - switch(bio_rw(bio)) { - case READ: + switch (bio_op(bio)) { + case REQ_OP_READ: + if (bio->bi_rw & REQ_RAHEAD) { + /* readahead of null bytes only wastes buffer cache */ + return -EIO; + } zero_fill_bio(bio); break; - case READA: - /* readahead of null bytes only wastes buffer cache */ - return -EIO; - case WRITE: + case REQ_OP_WRITE: /* writes get silently dropped */ break; + default: + return -EIO; } bio_endio(bio); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 1b2f96205361..ceb69fc0b10b 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -5,13 +5,13 @@ * This file is released under the GPL. */ -#include "dm.h" +#include "dm-core.h" +#include "dm-rq.h" #include "dm-uevent.h" #include #include #include -#include #include #include #include @@ -20,14 +20,8 @@ #include #include #include -#include -#include -#include /* for rq_end_sector() */ -#include #include -#include - #define DM_MSG_PREFIX "core" #ifdef CONFIG_PRINTK @@ -63,7 +57,6 @@ static DECLARE_WORK(deferred_remove_work, do_deferred_remove); static struct workqueue_struct *deferred_remove_workqueue; /* - * For bio-based dm. * One of these is allocated per bio. */ struct dm_io { @@ -76,36 +69,6 @@ struct dm_io { struct dm_stats_aux stats_aux; }; -/* - * For request-based dm. - * One of these is allocated per request. - */ -struct dm_rq_target_io { - struct mapped_device *md; - struct dm_target *ti; - struct request *orig, *clone; - struct kthread_work work; - int error; - union map_info info; - struct dm_stats_aux stats_aux; - unsigned long duration_jiffies; - unsigned n_sectors; -}; - -/* - * For request-based dm - the bio clones we allocate are embedded in these - * structs. - * - * We allocate these with bio_alloc_bioset, using the front_pad parameter when - * the bioset is created - this means the bio has to come at the end of the - * struct. - */ -struct dm_rq_clone_bio_info { - struct bio *orig; - struct dm_rq_target_io *tio; - struct bio clone; -}; - #define MINOR_ALLOCED ((void *)-1) /* @@ -120,130 +83,9 @@ struct dm_rq_clone_bio_info { #define DMF_DEFERRED_REMOVE 6 #define DMF_SUSPENDED_INTERNALLY 7 -/* - * Work processed by per-device workqueue. - */ -struct mapped_device { - struct srcu_struct io_barrier; - struct mutex suspend_lock; - - /* - * The current mapping (struct dm_table *). - * Use dm_get_live_table{_fast} or take suspend_lock for - * dereference. - */ - void __rcu *map; - - struct list_head table_devices; - struct mutex table_devices_lock; - - unsigned long flags; - - struct request_queue *queue; - int numa_node_id; - - unsigned type; - /* Protect queue and type against concurrent access. */ - struct mutex type_lock; - - atomic_t holders; - atomic_t open_count; - - struct dm_target *immutable_target; - struct target_type *immutable_target_type; - - struct gendisk *disk; - char name[16]; - - void *interface_ptr; - - /* - * A list of ios that arrived while we were suspended. - */ - atomic_t pending[2]; - wait_queue_head_t wait; - struct work_struct work; - spinlock_t deferred_lock; - struct bio_list deferred; - - /* - * Event handling. - */ - wait_queue_head_t eventq; - atomic_t event_nr; - atomic_t uevent_seq; - struct list_head uevent_list; - spinlock_t uevent_lock; /* Protect access to uevent_list */ - - /* the number of internal suspends */ - unsigned internal_suspend_count; - - /* - * Processing queue (flush) - */ - struct workqueue_struct *wq; - - /* - * io objects are allocated from here. - */ - mempool_t *io_pool; - mempool_t *rq_pool; - - struct bio_set *bs; - - /* - * freeze/thaw support require holding onto a super block - */ - struct super_block *frozen_sb; - - /* forced geometry settings */ - struct hd_geometry geometry; - - struct block_device *bdev; - - /* kobject and completion */ - struct dm_kobject_holder kobj_holder; - - /* zero-length flush that will be cloned and submitted to targets */ - struct bio flush_bio; - - struct dm_stats stats; - - struct kthread_worker kworker; - struct task_struct *kworker_task; - - /* for request-based merge heuristic in dm_request_fn() */ - unsigned seq_rq_merge_deadline_usecs; - int last_rq_rw; - sector_t last_rq_pos; - ktime_t last_rq_start_time; - - /* for blk-mq request-based DM support */ - struct blk_mq_tag_set *tag_set; - bool use_blk_mq:1; - bool init_tio_pdu:1; -}; - -#ifdef CONFIG_DM_MQ_DEFAULT -static bool use_blk_mq = true; -#else -static bool use_blk_mq = false; -#endif - -#define DM_MQ_NR_HW_QUEUES 1 -#define DM_MQ_QUEUE_DEPTH 2048 #define DM_NUMA_NODE NUMA_NO_NODE - -static unsigned dm_mq_nr_hw_queues = DM_MQ_NR_HW_QUEUES; -static unsigned dm_mq_queue_depth = DM_MQ_QUEUE_DEPTH; static int dm_numa_node = DM_NUMA_NODE; -bool dm_use_blk_mq(struct mapped_device *md) -{ - return md->use_blk_mq; -} -EXPORT_SYMBOL_GPL(dm_use_blk_mq); - /* * For mempools pre-allocation at the table loading time. */ @@ -259,9 +101,6 @@ struct table_device { struct dm_dev dm_dev; }; -#define RESERVED_BIO_BASED_IOS 16 -#define RESERVED_REQUEST_BASED_IOS 256 -#define RESERVED_MAX_IOS 1024 static struct kmem_cache *_io_cache; static struct kmem_cache *_rq_tio_cache; static struct kmem_cache *_rq_cache; @@ -269,13 +108,9 @@ static struct kmem_cache *_rq_cache; /* * Bio-based DM's mempools' reserved IOs set by the user. */ +#define RESERVED_BIO_BASED_IOS 16 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; -/* - * Request-based DM's mempools' reserved IOs set by the user. - */ -static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS; - static int __dm_get_module_param_int(int *module_param, int min, int max) { int param = ACCESS_ONCE(*module_param); @@ -297,8 +132,8 @@ static int __dm_get_module_param_int(int *module_param, int min, int max) return param; } -static unsigned __dm_get_module_param(unsigned *module_param, - unsigned def, unsigned max) +unsigned __dm_get_module_param(unsigned *module_param, + unsigned def, unsigned max) { unsigned param = ACCESS_ONCE(*module_param); unsigned modified_param = 0; @@ -319,28 +154,10 @@ static unsigned __dm_get_module_param(unsigned *module_param, unsigned dm_get_reserved_bio_based_ios(void) { return __dm_get_module_param(&reserved_bio_based_ios, - RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS); + RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); } EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); -unsigned dm_get_reserved_rq_based_ios(void) -{ - return __dm_get_module_param(&reserved_rq_based_ios, - RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS); -} -EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios); - -static unsigned dm_get_blk_mq_nr_hw_queues(void) -{ - return __dm_get_module_param(&dm_mq_nr_hw_queues, 1, 32); -} - -static unsigned dm_get_blk_mq_queue_depth(void) -{ - return __dm_get_module_param(&dm_mq_queue_depth, - DM_MQ_QUEUE_DEPTH, BLK_MQ_MAX_DEPTH); -} - static unsigned dm_get_numa_node(void) { return __dm_get_module_param_int(&dm_numa_node, @@ -679,29 +496,7 @@ static void free_tio(struct dm_target_io *tio) bio_put(&tio->clone); } -static struct dm_rq_target_io *alloc_old_rq_tio(struct mapped_device *md, - gfp_t gfp_mask) -{ - return mempool_alloc(md->io_pool, gfp_mask); -} - -static void free_old_rq_tio(struct dm_rq_target_io *tio) -{ - mempool_free(tio, tio->md->io_pool); -} - -static struct request *alloc_old_clone_request(struct mapped_device *md, - gfp_t gfp_mask) -{ - return mempool_alloc(md->rq_pool, gfp_mask); -} - -static void free_old_clone_request(struct mapped_device *md, struct request *rq) -{ - mempool_free(rq, md->rq_pool); -} - -static int md_in_flight(struct mapped_device *md) +int md_in_flight(struct mapped_device *md) { return atomic_read(&md->pending[READ]) + atomic_read(&md->pending[WRITE]); @@ -723,8 +518,9 @@ static void start_io_acct(struct dm_io *io) atomic_inc_return(&md->pending[rw])); if (unlikely(dm_stats_used(&md->stats))) - dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, - bio_sectors(bio), false, 0, &io->stats_aux); + dm_stats_account_io(&md->stats, bio_data_dir(bio), + bio->bi_iter.bi_sector, bio_sectors(bio), + false, 0, &io->stats_aux); } static void end_io_acct(struct dm_io *io) @@ -738,8 +534,9 @@ static void end_io_acct(struct dm_io *io) generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time); if (unlikely(dm_stats_used(&md->stats))) - dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, - bio_sectors(bio), true, duration, &io->stats_aux); + dm_stats_account_io(&md->stats, bio_data_dir(bio), + bio->bi_iter.bi_sector, bio_sectors(bio), + true, duration, &io->stats_aux); /* * After this is decremented the bio must not be touched if it is @@ -1001,12 +798,12 @@ static void dec_pending(struct dm_io *io, int error) if (io_error == DM_ENDIO_REQUEUE) return; - if ((bio->bi_rw & REQ_FLUSH) && bio->bi_iter.bi_size) { + if ((bio->bi_rw & REQ_PREFLUSH) && bio->bi_iter.bi_size) { /* * Preflush done for flush with data, reissue - * without REQ_FLUSH. + * without REQ_PREFLUSH. */ - bio->bi_rw &= ~REQ_FLUSH; + bio->bi_rw &= ~REQ_PREFLUSH; queue_io(md, bio); } else { /* done with normal IO or empty flush */ @@ -1017,7 +814,7 @@ static void dec_pending(struct dm_io *io, int error) } } -static void disable_write_same(struct mapped_device *md) +void disable_write_same(struct mapped_device *md) { struct queue_limits *limits = dm_get_queue_limits(md); @@ -1051,7 +848,7 @@ static void clone_endio(struct bio *bio) } } - if (unlikely(r == -EREMOTEIO && (bio->bi_rw & REQ_WRITE_SAME) && + if (unlikely(r == -EREMOTEIO && (bio_op(bio) == REQ_OP_WRITE_SAME) && !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors)) disable_write_same(md); @@ -1059,371 +856,6 @@ static void clone_endio(struct bio *bio) dec_pending(io, error); } -/* - * Partial completion handling for request-based dm - */ -static void end_clone_bio(struct bio *clone) -{ - struct dm_rq_clone_bio_info *info = - container_of(clone, struct dm_rq_clone_bio_info, clone); - struct dm_rq_target_io *tio = info->tio; - struct bio *bio = info->orig; - unsigned int nr_bytes = info->orig->bi_iter.bi_size; - int error = clone->bi_error; - - bio_put(clone); - - if (tio->error) - /* - * An error has already been detected on the request. - * Once error occurred, just let clone->end_io() handle - * the remainder. - */ - return; - else if (error) { - /* - * Don't notice the error to the upper layer yet. - * The error handling decision is made by the target driver, - * when the request is completed. - */ - tio->error = error; - return; - } - - /* - * I/O for the bio successfully completed. - * Notice the data completion to the upper layer. - */ - - /* - * bios are processed from the head of the list. - * So the completing bio should always be rq->bio. - * If it's not, something wrong is happening. - */ - if (tio->orig->bio != bio) - DMERR("bio completion is going in the middle of the request"); - - /* - * Update the original request. - * Do not use blk_end_request() here, because it may complete - * the original request before the clone, and break the ordering. - */ - blk_update_request(tio->orig, 0, nr_bytes); -} - -static struct dm_rq_target_io *tio_from_request(struct request *rq) -{ - return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special); -} - -static void rq_end_stats(struct mapped_device *md, struct request *orig) -{ - if (unlikely(dm_stats_used(&md->stats))) { - struct dm_rq_target_io *tio = tio_from_request(orig); - tio->duration_jiffies = jiffies - tio->duration_jiffies; - dm_stats_account_io(&md->stats, orig->cmd_flags, blk_rq_pos(orig), - tio->n_sectors, true, tio->duration_jiffies, - &tio->stats_aux); - } -} - -/* - * Don't touch any member of the md after calling this function because - * the md may be freed in dm_put() at the end of this function. - * Or do dm_get() before calling this function and dm_put() later. - */ -static void rq_completed(struct mapped_device *md, int rw, bool run_queue) -{ - atomic_dec(&md->pending[rw]); - - /* nudge anyone waiting on suspend queue */ - if (!md_in_flight(md)) - wake_up(&md->wait); - - /* - * Run this off this callpath, as drivers could invoke end_io while - * inside their request_fn (and holding the queue lock). Calling - * back into ->request_fn() could deadlock attempting to grab the - * queue lock again. - */ - if (!md->queue->mq_ops && run_queue) - blk_run_queue_async(md->queue); - - /* - * dm_put() must be at the end of this function. See the comment above - */ - dm_put(md); -} - -static void free_rq_clone(struct request *clone) -{ - struct dm_rq_target_io *tio = clone->end_io_data; - struct mapped_device *md = tio->md; - - blk_rq_unprep_clone(clone); - - if (md->type == DM_TYPE_MQ_REQUEST_BASED) - /* stacked on blk-mq queue(s) */ - tio->ti->type->release_clone_rq(clone); - else if (!md->queue->mq_ops) - /* request_fn queue stacked on request_fn queue(s) */ - free_old_clone_request(md, clone); - - if (!md->queue->mq_ops) - free_old_rq_tio(tio); -} - -/* - * Complete the clone and the original request. - * Must be called without clone's queue lock held, - * see end_clone_request() for more details. - */ -static void dm_end_request(struct request *clone, int error) -{ - int rw = rq_data_dir(clone); - struct dm_rq_target_io *tio = clone->end_io_data; - struct mapped_device *md = tio->md; - struct request *rq = tio->orig; - - if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { - rq->errors = clone->errors; - rq->resid_len = clone->resid_len; - - if (rq->sense) - /* - * We are using the sense buffer of the original - * request. - * So setting the length of the sense data is enough. - */ - rq->sense_len = clone->sense_len; - } - - free_rq_clone(clone); - rq_end_stats(md, rq); - if (!rq->q->mq_ops) - blk_end_request_all(rq, error); - else - blk_mq_end_request(rq, error); - rq_completed(md, rw, true); -} - -static void dm_unprep_request(struct request *rq) -{ - struct dm_rq_target_io *tio = tio_from_request(rq); - struct request *clone = tio->clone; - - if (!rq->q->mq_ops) { - rq->special = NULL; - rq->cmd_flags &= ~REQ_DONTPREP; - } - - if (clone) - free_rq_clone(clone); - else if (!tio->md->queue->mq_ops) - free_old_rq_tio(tio); -} - -/* - * Requeue the original request of a clone. - */ -static void dm_old_requeue_request(struct request *rq) -{ - struct request_queue *q = rq->q; - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - blk_requeue_request(q, rq); - blk_run_queue_async(q); - spin_unlock_irqrestore(q->queue_lock, flags); -} - -static void dm_mq_requeue_request(struct request *rq) -{ - struct request_queue *q = rq->q; - unsigned long flags; - - blk_mq_requeue_request(rq); - spin_lock_irqsave(q->queue_lock, flags); - if (!blk_queue_stopped(q)) - blk_mq_kick_requeue_list(q); - spin_unlock_irqrestore(q->queue_lock, flags); -} - -static void dm_requeue_original_request(struct mapped_device *md, - struct request *rq) -{ - int rw = rq_data_dir(rq); - - rq_end_stats(md, rq); - dm_unprep_request(rq); - - if (!rq->q->mq_ops) - dm_old_requeue_request(rq); - else - dm_mq_requeue_request(rq); - - rq_completed(md, rw, false); -} - -static void dm_old_stop_queue(struct request_queue *q) -{ - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - if (blk_queue_stopped(q)) { - spin_unlock_irqrestore(q->queue_lock, flags); - return; - } - - blk_stop_queue(q); - spin_unlock_irqrestore(q->queue_lock, flags); -} - -static void dm_stop_queue(struct request_queue *q) -{ - if (!q->mq_ops) - dm_old_stop_queue(q); - else - blk_mq_stop_hw_queues(q); -} - -static void dm_old_start_queue(struct request_queue *q) -{ - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - if (blk_queue_stopped(q)) - blk_start_queue(q); - spin_unlock_irqrestore(q->queue_lock, flags); -} - -static void dm_start_queue(struct request_queue *q) -{ - if (!q->mq_ops) - dm_old_start_queue(q); - else { - blk_mq_start_stopped_hw_queues(q, true); - blk_mq_kick_requeue_list(q); - } -} - -static void dm_done(struct request *clone, int error, bool mapped) -{ - int r = error; - struct dm_rq_target_io *tio = clone->end_io_data; - dm_request_endio_fn rq_end_io = NULL; - - if (tio->ti) { - rq_end_io = tio->ti->type->rq_end_io; - - if (mapped && rq_end_io) - r = rq_end_io(tio->ti, clone, error, &tio->info); - } - - if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) && - !clone->q->limits.max_write_same_sectors)) - disable_write_same(tio->md); - - if (r <= 0) - /* The target wants to complete the I/O */ - dm_end_request(clone, r); - else if (r == DM_ENDIO_INCOMPLETE) - /* The target will handle the I/O */ - return; - else if (r == DM_ENDIO_REQUEUE) - /* The target wants to requeue the I/O */ - dm_requeue_original_request(tio->md, tio->orig); - else { - DMWARN("unimplemented target endio return value: %d", r); - BUG(); - } -} - -/* - * Request completion handler for request-based dm - */ -static void dm_softirq_done(struct request *rq) -{ - bool mapped = true; - struct dm_rq_target_io *tio = tio_from_request(rq); - struct request *clone = tio->clone; - int rw; - - if (!clone) { - rq_end_stats(tio->md, rq); - rw = rq_data_dir(rq); - if (!rq->q->mq_ops) { - blk_end_request_all(rq, tio->error); - rq_completed(tio->md, rw, false); - free_old_rq_tio(tio); - } else { - blk_mq_end_request(rq, tio->error); - rq_completed(tio->md, rw, false); - } - return; - } - - if (rq->cmd_flags & REQ_FAILED) - mapped = false; - - dm_done(clone, tio->error, mapped); -} - -/* - * Complete the clone and the original request with the error status - * through softirq context. - */ -static void dm_complete_request(struct request *rq, int error) -{ - struct dm_rq_target_io *tio = tio_from_request(rq); - - tio->error = error; - if (!rq->q->mq_ops) - blk_complete_request(rq); - else - blk_mq_complete_request(rq, error); -} - -/* - * Complete the not-mapped clone and the original request with the error status - * through softirq context. - * Target's rq_end_io() function isn't called. - * This may be used when the target's map_rq() or clone_and_map_rq() functions fail. - */ -static void dm_kill_unmapped_request(struct request *rq, int error) -{ - rq->cmd_flags |= REQ_FAILED; - dm_complete_request(rq, error); -} - -/* - * Called with the clone's queue lock held (in the case of .request_fn) - */ -static void end_clone_request(struct request *clone, int error) -{ - struct dm_rq_target_io *tio = clone->end_io_data; - - if (!clone->q->mq_ops) { - /* - * For just cleaning up the information of the queue in which - * the clone was dispatched. - * The clone is *NOT* freed actually here because it is alloced - * from dm own mempool (REQ_ALLOCED isn't set). - */ - __blk_put_request(clone->q, clone); - } - - /* - * Actual request completion is done in a softirq context which doesn't - * hold the clone's queue lock. Otherwise, deadlock could occur because: - * - another request may be submitted by the upper level driver - * of the stacking during the completion - * - the submission which requires queue lock may be done - * against this clone's queue - */ - dm_complete_request(tio->orig, error); -} - /* * Return maximum size of I/O possible at the supplied sector up to the current * target boundary. @@ -1473,15 +905,42 @@ int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) } EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); -/* - * A target may call dm_accept_partial_bio only from the map routine. It is - * allowed for all bio types except REQ_FLUSH. - * - * dm_accept_partial_bio informs the dm that the target only wants to process - * additional n_sectors sectors of the bio and the rest of the data should be - * sent in a next bio. - * - * A diagram that explains the arithmetics: +static long dm_blk_direct_access(struct block_device *bdev, sector_t sector, + void __pmem **kaddr, pfn_t *pfn, long size) +{ + struct mapped_device *md = bdev->bd_disk->private_data; + struct dm_table *map; + struct dm_target *ti; + int srcu_idx; + long len, ret = -EIO; + + map = dm_get_live_table(md, &srcu_idx); + if (!map) + goto out; + + ti = dm_table_find_target(map, sector); + if (!dm_target_is_valid(ti)) + goto out; + + len = max_io_len(sector, ti) << SECTOR_SHIFT; + size = min(len, size); + + if (ti->type->direct_access) + ret = ti->type->direct_access(ti, sector, kaddr, pfn, size); +out: + dm_put_live_table(md, srcu_idx); + return min(ret, size); +} + +/* + * A target may call dm_accept_partial_bio only from the map routine. It is + * allowed for all bio types except REQ_PREFLUSH. + * + * dm_accept_partial_bio informs the dm that the target only wants to process + * additional n_sectors sectors of the bio and the rest of the data should be + * sent in a next bio. + * + * A diagram that explains the arithmetics: * +--------------------+---------------+-------+ * | 1 | 2 | 3 | * +--------------------+---------------+-------+ @@ -1505,7 +964,7 @@ void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) { struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; - BUG_ON(bio->bi_rw & REQ_FLUSH); + BUG_ON(bio->bi_rw & REQ_PREFLUSH); BUG_ON(bi_size > *tio->len_ptr); BUG_ON(n_sectors > bi_size); *tio->len_ptr -= bi_size - n_sectors; @@ -1672,521 +1131,175 @@ static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti, typedef unsigned (*get_num_bios_fn)(struct dm_target *ti); -static unsigned get_num_discard_bios(struct dm_target *ti) -{ - return ti->num_discard_bios; -} - -static unsigned get_num_write_same_bios(struct dm_target *ti) -{ - return ti->num_write_same_bios; -} - -typedef bool (*is_split_required_fn)(struct dm_target *ti); - -static bool is_split_required_for_discard(struct dm_target *ti) -{ - return ti->split_discard_bios; -} - -static int __send_changing_extent_only(struct clone_info *ci, - get_num_bios_fn get_num_bios, - is_split_required_fn is_split_required) -{ - struct dm_target *ti; - unsigned len; - unsigned num_bios; - - do { - ti = dm_table_find_target(ci->map, ci->sector); - if (!dm_target_is_valid(ti)) - return -EIO; - - /* - * Even though the device advertised support for this type of - * request, that does not mean every target supports it, and - * reconfiguration might also have changed that since the - * check was performed. - */ - num_bios = get_num_bios ? get_num_bios(ti) : 0; - if (!num_bios) - return -EOPNOTSUPP; - - if (is_split_required && !is_split_required(ti)) - len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti)); - else - len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti)); - - __send_duplicate_bios(ci, ti, num_bios, &len); - - ci->sector += len; - } while (ci->sector_count -= len); - - return 0; -} - -static int __send_discard(struct clone_info *ci) -{ - return __send_changing_extent_only(ci, get_num_discard_bios, - is_split_required_for_discard); -} - -static int __send_write_same(struct clone_info *ci) -{ - return __send_changing_extent_only(ci, get_num_write_same_bios, NULL); -} - -/* - * Select the correct strategy for processing a non-flush bio. - */ -static int __split_and_process_non_flush(struct clone_info *ci) -{ - struct bio *bio = ci->bio; - struct dm_target *ti; - unsigned len; - int r; - - if (unlikely(bio->bi_rw & REQ_DISCARD)) - return __send_discard(ci); - else if (unlikely(bio->bi_rw & REQ_WRITE_SAME)) - return __send_write_same(ci); - - ti = dm_table_find_target(ci->map, ci->sector); - if (!dm_target_is_valid(ti)) - return -EIO; - - len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count); - - r = __clone_and_map_data_bio(ci, ti, ci->sector, &len); - if (r < 0) - return r; - - ci->sector += len; - ci->sector_count -= len; - - return 0; -} - -/* - * Entry point to split a bio into clones and submit them to the targets. - */ -static void __split_and_process_bio(struct mapped_device *md, - struct dm_table *map, struct bio *bio) -{ - struct clone_info ci; - int error = 0; - - if (unlikely(!map)) { - bio_io_error(bio); - return; - } - - ci.map = map; - ci.md = md; - ci.io = alloc_io(md); - ci.io->error = 0; - atomic_set(&ci.io->io_count, 1); - ci.io->bio = bio; - ci.io->md = md; - spin_lock_init(&ci.io->endio_lock); - ci.sector = bio->bi_iter.bi_sector; - - start_io_acct(ci.io); - - if (bio->bi_rw & REQ_FLUSH) { - ci.bio = &ci.md->flush_bio; - ci.sector_count = 0; - error = __send_empty_flush(&ci); - /* dec_pending submits any data associated with flush */ - } else { - ci.bio = bio; - ci.sector_count = bio_sectors(bio); - while (ci.sector_count && !error) - error = __split_and_process_non_flush(&ci); - } - - /* drop the extra reference count */ - dec_pending(ci.io, error); -} -/*----------------------------------------------------------------- - * CRUD END - *---------------------------------------------------------------*/ - -/* - * The request function that just remaps the bio built up by - * dm_merge_bvec. - */ -static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio) -{ - int rw = bio_data_dir(bio); - struct mapped_device *md = q->queuedata; - int srcu_idx; - struct dm_table *map; - - map = dm_get_live_table(md, &srcu_idx); - - generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); - - /* if we're suspended, we have to queue this io for later */ - if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { - dm_put_live_table(md, srcu_idx); - - if (bio_rw(bio) != READA) - queue_io(md, bio); - else - bio_io_error(bio); - return BLK_QC_T_NONE; - } - - __split_and_process_bio(md, map, bio); - dm_put_live_table(md, srcu_idx); - return BLK_QC_T_NONE; -} - -int dm_request_based(struct mapped_device *md) -{ - return blk_queue_stackable(md->queue); -} - -static void dm_dispatch_clone_request(struct request *clone, struct request *rq) -{ - int r; - - if (blk_queue_io_stat(clone->q)) - clone->cmd_flags |= REQ_IO_STAT; - - clone->start_time = jiffies; - r = blk_insert_cloned_request(clone->q, clone); - if (r) - /* must complete clone in terms of original request */ - dm_complete_request(rq, r); -} - -static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig, - void *data) -{ - struct dm_rq_target_io *tio = data; - struct dm_rq_clone_bio_info *info = - container_of(bio, struct dm_rq_clone_bio_info, clone); - - info->orig = bio_orig; - info->tio = tio; - bio->bi_end_io = end_clone_bio; - - return 0; -} - -static int setup_clone(struct request *clone, struct request *rq, - struct dm_rq_target_io *tio, gfp_t gfp_mask) -{ - int r; - - r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask, - dm_rq_bio_constructor, tio); - if (r) - return r; - - clone->cmd = rq->cmd; - clone->cmd_len = rq->cmd_len; - clone->sense = rq->sense; - clone->end_io = end_clone_request; - clone->end_io_data = tio; - - tio->clone = clone; - - return 0; -} - -static struct request *clone_old_rq(struct request *rq, struct mapped_device *md, - struct dm_rq_target_io *tio, gfp_t gfp_mask) -{ - /* - * Create clone for use with .request_fn request_queue - */ - struct request *clone; - - clone = alloc_old_clone_request(md, gfp_mask); - if (!clone) - return NULL; - - blk_rq_init(NULL, clone); - if (setup_clone(clone, rq, tio, gfp_mask)) { - /* -ENOMEM */ - free_old_clone_request(md, clone); - return NULL; - } - - return clone; -} - -static void map_tio_request(struct kthread_work *work); - -static void init_tio(struct dm_rq_target_io *tio, struct request *rq, - struct mapped_device *md) -{ - tio->md = md; - tio->ti = NULL; - tio->clone = NULL; - tio->orig = rq; - tio->error = 0; - /* - * Avoid initializing info for blk-mq; it passes - * target-specific data through info.ptr - * (see: dm_mq_init_request) - */ - if (!md->init_tio_pdu) - memset(&tio->info, 0, sizeof(tio->info)); - if (md->kworker_task) - init_kthread_work(&tio->work, map_tio_request); -} - -static struct dm_rq_target_io *dm_old_prep_tio(struct request *rq, - struct mapped_device *md, - gfp_t gfp_mask) -{ - struct dm_rq_target_io *tio; - int srcu_idx; - struct dm_table *table; - - tio = alloc_old_rq_tio(md, gfp_mask); - if (!tio) - return NULL; - - init_tio(tio, rq, md); - - table = dm_get_live_table(md, &srcu_idx); - /* - * Must clone a request if this .request_fn DM device - * is stacked on .request_fn device(s). - */ - if (!dm_table_mq_request_based(table)) { - if (!clone_old_rq(rq, md, tio, gfp_mask)) { - dm_put_live_table(md, srcu_idx); - free_old_rq_tio(tio); - return NULL; - } - } - dm_put_live_table(md, srcu_idx); - - return tio; -} - -/* - * Called with the queue lock held. - */ -static int dm_old_prep_fn(struct request_queue *q, struct request *rq) -{ - struct mapped_device *md = q->queuedata; - struct dm_rq_target_io *tio; - - if (unlikely(rq->special)) { - DMWARN("Already has something in rq->special."); - return BLKPREP_KILL; - } - - tio = dm_old_prep_tio(rq, md, GFP_ATOMIC); - if (!tio) - return BLKPREP_DEFER; - - rq->special = tio; - rq->cmd_flags |= REQ_DONTPREP; - - return BLKPREP_OK; -} - -/* - * Returns: - * 0 : the request has been processed - * DM_MAPIO_REQUEUE : the original request needs to be requeued - * < 0 : the request was completed due to failure - */ -static int map_request(struct dm_rq_target_io *tio, struct request *rq, - struct mapped_device *md) -{ - int r; - struct dm_target *ti = tio->ti; - struct request *clone = NULL; - - if (tio->clone) { - clone = tio->clone; - r = ti->type->map_rq(ti, clone, &tio->info); - } else { - r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone); - if (r < 0) { - /* The target wants to complete the I/O */ - dm_kill_unmapped_request(rq, r); - return r; - } - if (r != DM_MAPIO_REMAPPED) - return r; - if (setup_clone(clone, rq, tio, GFP_ATOMIC)) { - /* -ENOMEM */ - ti->type->release_clone_rq(clone); - return DM_MAPIO_REQUEUE; - } - } - - switch (r) { - case DM_MAPIO_SUBMITTED: - /* The target has taken the I/O to submit by itself later */ - break; - case DM_MAPIO_REMAPPED: - /* The target has remapped the I/O so dispatch it */ - trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)), - blk_rq_pos(rq)); - dm_dispatch_clone_request(clone, rq); - break; - case DM_MAPIO_REQUEUE: - /* The target wants to requeue the I/O */ - dm_requeue_original_request(md, tio->orig); - break; - default: - if (r > 0) { - DMWARN("unimplemented target map return value: %d", r); - BUG(); - } - - /* The target wants to complete the I/O */ - dm_kill_unmapped_request(rq, r); - return r; - } - - return 0; +static unsigned get_num_discard_bios(struct dm_target *ti) +{ + return ti->num_discard_bios; } -static void map_tio_request(struct kthread_work *work) +static unsigned get_num_write_same_bios(struct dm_target *ti) { - struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work); - struct request *rq = tio->orig; - struct mapped_device *md = tio->md; + return ti->num_write_same_bios; +} + +typedef bool (*is_split_required_fn)(struct dm_target *ti); - if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) - dm_requeue_original_request(md, rq); +static bool is_split_required_for_discard(struct dm_target *ti) +{ + return ti->split_discard_bios; } -static void dm_start_request(struct mapped_device *md, struct request *orig) +static int __send_changing_extent_only(struct clone_info *ci, + get_num_bios_fn get_num_bios, + is_split_required_fn is_split_required) { - if (!orig->q->mq_ops) - blk_start_request(orig); - else - blk_mq_start_request(orig); - atomic_inc(&md->pending[rq_data_dir(orig)]); + struct dm_target *ti; + unsigned len; + unsigned num_bios; - if (md->seq_rq_merge_deadline_usecs) { - md->last_rq_pos = rq_end_sector(orig); - md->last_rq_rw = rq_data_dir(orig); - md->last_rq_start_time = ktime_get(); - } + do { + ti = dm_table_find_target(ci->map, ci->sector); + if (!dm_target_is_valid(ti)) + return -EIO; - if (unlikely(dm_stats_used(&md->stats))) { - struct dm_rq_target_io *tio = tio_from_request(orig); - tio->duration_jiffies = jiffies; - tio->n_sectors = blk_rq_sectors(orig); - dm_stats_account_io(&md->stats, orig->cmd_flags, blk_rq_pos(orig), - tio->n_sectors, false, 0, &tio->stats_aux); - } + /* + * Even though the device advertised support for this type of + * request, that does not mean every target supports it, and + * reconfiguration might also have changed that since the + * check was performed. + */ + num_bios = get_num_bios ? get_num_bios(ti) : 0; + if (!num_bios) + return -EOPNOTSUPP; - /* - * Hold the md reference here for the in-flight I/O. - * We can't rely on the reference count by device opener, - * because the device may be closed during the request completion - * when all bios are completed. - * See the comment in rq_completed() too. - */ - dm_get(md); + if (is_split_required && !is_split_required(ti)) + len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti)); + else + len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti)); + + __send_duplicate_bios(ci, ti, num_bios, &len); + + ci->sector += len; + } while (ci->sector_count -= len); + + return 0; } -#define MAX_SEQ_RQ_MERGE_DEADLINE_USECS 100000 +static int __send_discard(struct clone_info *ci) +{ + return __send_changing_extent_only(ci, get_num_discard_bios, + is_split_required_for_discard); +} -ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf) +static int __send_write_same(struct clone_info *ci) { - return sprintf(buf, "%u\n", md->seq_rq_merge_deadline_usecs); + return __send_changing_extent_only(ci, get_num_write_same_bios, NULL); } -ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md, - const char *buf, size_t count) +/* + * Select the correct strategy for processing a non-flush bio. + */ +static int __split_and_process_non_flush(struct clone_info *ci) { - unsigned deadline; + struct bio *bio = ci->bio; + struct dm_target *ti; + unsigned len; + int r; + + if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) + return __send_discard(ci); + else if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) + return __send_write_same(ci); - if (!dm_request_based(md) || md->use_blk_mq) - return count; + ti = dm_table_find_target(ci->map, ci->sector); + if (!dm_target_is_valid(ti)) + return -EIO; - if (kstrtouint(buf, 10, &deadline)) - return -EINVAL; + len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count); - if (deadline > MAX_SEQ_RQ_MERGE_DEADLINE_USECS) - deadline = MAX_SEQ_RQ_MERGE_DEADLINE_USECS; + r = __clone_and_map_data_bio(ci, ti, ci->sector, &len); + if (r < 0) + return r; - md->seq_rq_merge_deadline_usecs = deadline; + ci->sector += len; + ci->sector_count -= len; - return count; + return 0; } -static bool dm_request_peeked_before_merge_deadline(struct mapped_device *md) +/* + * Entry point to split a bio into clones and submit them to the targets. + */ +static void __split_and_process_bio(struct mapped_device *md, + struct dm_table *map, struct bio *bio) { - ktime_t kt_deadline; + struct clone_info ci; + int error = 0; + + if (unlikely(!map)) { + bio_io_error(bio); + return; + } + + ci.map = map; + ci.md = md; + ci.io = alloc_io(md); + ci.io->error = 0; + atomic_set(&ci.io->io_count, 1); + ci.io->bio = bio; + ci.io->md = md; + spin_lock_init(&ci.io->endio_lock); + ci.sector = bio->bi_iter.bi_sector; - if (!md->seq_rq_merge_deadline_usecs) - return false; + start_io_acct(ci.io); - kt_deadline = ns_to_ktime((u64)md->seq_rq_merge_deadline_usecs * NSEC_PER_USEC); - kt_deadline = ktime_add_safe(md->last_rq_start_time, kt_deadline); + if (bio->bi_rw & REQ_PREFLUSH) { + ci.bio = &ci.md->flush_bio; + ci.sector_count = 0; + error = __send_empty_flush(&ci); + /* dec_pending submits any data associated with flush */ + } else { + ci.bio = bio; + ci.sector_count = bio_sectors(bio); + while (ci.sector_count && !error) + error = __split_and_process_non_flush(&ci); + } - return !ktime_after(ktime_get(), kt_deadline); + /* drop the extra reference count */ + dec_pending(ci.io, error); } +/*----------------------------------------------------------------- + * CRUD END + *---------------------------------------------------------------*/ /* - * q->request_fn for request-based dm. - * Called with the queue lock held. + * The request function that just remaps the bio built up by + * dm_merge_bvec. */ -static void dm_request_fn(struct request_queue *q) +static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio) { + int rw = bio_data_dir(bio); struct mapped_device *md = q->queuedata; - struct dm_target *ti = md->immutable_target; - struct request *rq; - struct dm_rq_target_io *tio; - sector_t pos = 0; - - if (unlikely(!ti)) { - int srcu_idx; - struct dm_table *map = dm_get_live_table(md, &srcu_idx); - - ti = dm_table_find_target(map, pos); - dm_put_live_table(md, srcu_idx); - } - - /* - * For suspend, check blk_queue_stopped() and increment - * ->pending within a single queue_lock not to increment the - * number of in-flight I/Os after the queue is stopped in - * dm_suspend(). - */ - while (!blk_queue_stopped(q)) { - rq = blk_peek_request(q); - if (!rq) - return; + int srcu_idx; + struct dm_table *map; - /* always use block 0 to find the target for flushes for now */ - pos = 0; - if (!(rq->cmd_flags & REQ_FLUSH)) - pos = blk_rq_pos(rq); + map = dm_get_live_table(md, &srcu_idx); - if ((dm_request_peeked_before_merge_deadline(md) && - md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 && - md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq)) || - (ti->type->busy && ti->type->busy(ti))) { - blk_delay_queue(q, HZ / 100); - return; - } + generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); - dm_start_request(md, rq); + /* if we're suspended, we have to queue this io for later */ + if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { + dm_put_live_table(md, srcu_idx); - tio = tio_from_request(rq); - /* Establish tio->ti before queuing work (map_tio_request) */ - tio->ti = ti; - queue_kthread_work(&md->kworker, &tio->work); - BUG_ON(!irqs_disabled()); + if (!(bio->bi_rw & REQ_RAHEAD)) + queue_io(md, bio); + else + bio_io_error(bio); + return BLK_QC_T_NONE; } + + __split_and_process_bio(md, map, bio); + dm_put_live_table(md, srcu_idx); + return BLK_QC_T_NONE; } static int dm_any_congested(void *congested_data, int bdi_bits) @@ -2266,7 +1379,7 @@ static const struct block_device_operations dm_blk_dops; static void dm_wq_work(struct work_struct *work); -static void dm_init_md_queue(struct mapped_device *md) +void dm_init_md_queue(struct mapped_device *md) { /* * Request-based dm devices cannot be stacked on top of bio-based dm @@ -2287,7 +1400,7 @@ static void dm_init_md_queue(struct mapped_device *md) md->queue->backing_dev_info.congested_data = md; } -static void dm_init_normal_md_queue(struct mapped_device *md) +void dm_init_normal_md_queue(struct mapped_device *md) { md->use_blk_mq = false; dm_init_md_queue(md); @@ -2327,6 +1440,8 @@ static void cleanup_mapped_device(struct mapped_device *md) bdput(md->bdev); md->bdev = NULL; } + + dm_mq_cleanup_mapped_device(md); } /* @@ -2360,7 +1475,7 @@ static struct mapped_device *alloc_dev(int minor) goto bad_io_barrier; md->numa_node_id = numa_node_id; - md->use_blk_mq = use_blk_mq; + md->use_blk_mq = dm_use_blk_mq_default(); md->init_tio_pdu = false; md->type = DM_TYPE_NONE; mutex_init(&md->suspend_lock); @@ -2412,7 +1527,7 @@ static struct mapped_device *alloc_dev(int minor) bio_init(&md->flush_bio); md->flush_bio.bi_bdev = md->bdev; - md->flush_bio.bi_rw = WRITE_FLUSH; + bio_set_op_attrs(&md->flush_bio, REQ_OP_WRITE, WRITE_FLUSH); dm_stats_init(&md->stats); @@ -2445,10 +1560,6 @@ static void free_dev(struct mapped_device *md) unlock_fs(md); cleanup_mapped_device(md); - if (md->tag_set) { - blk_mq_free_tag_set(md->tag_set); - kfree(md->tag_set); - } free_table_devices(&md->table_devices); dm_stats_cleanup(&md->stats); @@ -2464,7 +1575,7 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t) if (md->bs) { /* The md already has necessary mempools. */ - if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) { + if (dm_table_bio_based(t)) { /* * Reload bioset because front_pad may have changed * because a different table was loaded. @@ -2654,176 +1765,15 @@ struct queue_limits *dm_get_queue_limits(struct mapped_device *md) } EXPORT_SYMBOL_GPL(dm_get_queue_limits); -static void dm_old_init_rq_based_worker_thread(struct mapped_device *md) -{ - /* Initialize the request-based DM worker thread */ - init_kthread_worker(&md->kworker); - md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker, - "kdmwork-%s", dm_device_name(md)); -} - -/* - * Fully initialize a .request_fn request-based queue. - */ -static int dm_old_init_request_queue(struct mapped_device *md) -{ - /* Fully initialize the queue */ - if (!blk_init_allocated_queue(md->queue, dm_request_fn, NULL)) - return -EINVAL; - - /* disable dm_request_fn's merge heuristic by default */ - md->seq_rq_merge_deadline_usecs = 0; - - dm_init_normal_md_queue(md); - blk_queue_softirq_done(md->queue, dm_softirq_done); - blk_queue_prep_rq(md->queue, dm_old_prep_fn); - - dm_old_init_rq_based_worker_thread(md); - - elv_register_queue(md->queue); - - return 0; -} - -static int dm_mq_init_request(void *data, struct request *rq, - unsigned int hctx_idx, unsigned int request_idx, - unsigned int numa_node) -{ - struct mapped_device *md = data; - struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); - - /* - * Must initialize md member of tio, otherwise it won't - * be available in dm_mq_queue_rq. - */ - tio->md = md; - - if (md->init_tio_pdu) { - /* target-specific per-io data is immediately after the tio */ - tio->info.ptr = tio + 1; - } - - return 0; -} - -static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, - const struct blk_mq_queue_data *bd) -{ - struct request *rq = bd->rq; - struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); - struct mapped_device *md = tio->md; - struct dm_target *ti = md->immutable_target; - - if (unlikely(!ti)) { - int srcu_idx; - struct dm_table *map = dm_get_live_table(md, &srcu_idx); - - ti = dm_table_find_target(map, 0); - dm_put_live_table(md, srcu_idx); - } - - if (ti->type->busy && ti->type->busy(ti)) - return BLK_MQ_RQ_QUEUE_BUSY; - - dm_start_request(md, rq); - - /* Init tio using md established in .init_request */ - init_tio(tio, rq, md); - - /* - * Establish tio->ti before queuing work (map_tio_request) - * or making direct call to map_request(). - */ - tio->ti = ti; - - /* Direct call is fine since .queue_rq allows allocations */ - if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) { - /* Undo dm_start_request() before requeuing */ - rq_end_stats(md, rq); - rq_completed(md, rq_data_dir(rq), false); - return BLK_MQ_RQ_QUEUE_BUSY; - } - - return BLK_MQ_RQ_QUEUE_OK; -} - -static struct blk_mq_ops dm_mq_ops = { - .queue_rq = dm_mq_queue_rq, - .map_queue = blk_mq_map_queue, - .complete = dm_softirq_done, - .init_request = dm_mq_init_request, -}; - -static int dm_mq_init_request_queue(struct mapped_device *md, - struct dm_target *immutable_tgt) -{ - struct request_queue *q; - int err; - - if (dm_get_md_type(md) == DM_TYPE_REQUEST_BASED) { - DMERR("request-based dm-mq may only be stacked on blk-mq device(s)"); - return -EINVAL; - } - - md->tag_set = kzalloc_node(sizeof(struct blk_mq_tag_set), GFP_KERNEL, md->numa_node_id); - if (!md->tag_set) - return -ENOMEM; - - md->tag_set->ops = &dm_mq_ops; - md->tag_set->queue_depth = dm_get_blk_mq_queue_depth(); - md->tag_set->numa_node = md->numa_node_id; - md->tag_set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; - md->tag_set->nr_hw_queues = dm_get_blk_mq_nr_hw_queues(); - md->tag_set->driver_data = md; - - md->tag_set->cmd_size = sizeof(struct dm_rq_target_io); - if (immutable_tgt && immutable_tgt->per_io_data_size) { - /* any target-specific per-io data is immediately after the tio */ - md->tag_set->cmd_size += immutable_tgt->per_io_data_size; - md->init_tio_pdu = true; - } - - err = blk_mq_alloc_tag_set(md->tag_set); - if (err) - goto out_kfree_tag_set; - - q = blk_mq_init_allocated_queue(md->tag_set, md->queue); - if (IS_ERR(q)) { - err = PTR_ERR(q); - goto out_tag_set; - } - dm_init_md_queue(md); - - /* backfill 'mq' sysfs registration normally done in blk_register_queue */ - blk_mq_register_disk(md->disk); - - return 0; - -out_tag_set: - blk_mq_free_tag_set(md->tag_set); -out_kfree_tag_set: - kfree(md->tag_set); - - return err; -} - -static unsigned filter_md_type(unsigned type, struct mapped_device *md) -{ - if (type == DM_TYPE_BIO_BASED) - return type; - - return !md->use_blk_mq ? DM_TYPE_REQUEST_BASED : DM_TYPE_MQ_REQUEST_BASED; -} - /* * Setup the DM device's queue based on md's type */ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) { int r; - unsigned md_type = filter_md_type(dm_get_md_type(md), md); + unsigned type = dm_get_md_type(md); - switch (md_type) { + switch (type) { case DM_TYPE_REQUEST_BASED: r = dm_old_init_request_queue(md); if (r) { @@ -2832,13 +1782,14 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) } break; case DM_TYPE_MQ_REQUEST_BASED: - r = dm_mq_init_request_queue(md, dm_table_get_immutable_target(t)); + r = dm_mq_init_request_queue(md, t); if (r) { DMERR("Cannot initialize queue for request-based dm-mq mapped device"); return r; } break; case DM_TYPE_BIO_BASED: + case DM_TYPE_DAX_BIO_BASED: dm_init_normal_md_queue(md); blk_queue_make_request(md->queue, dm_make_request); /* @@ -2847,6 +1798,9 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) */ bioset_free(md->queue->bio_split); md->queue->bio_split = NULL; + + if (type == DM_TYPE_DAX_BIO_BASED) + queue_flag_set_unlocked(QUEUE_FLAG_DAX, md->queue); break; } @@ -3541,10 +2495,9 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t if (!pools) return NULL; - type = filter_md_type(type, md); - switch (type) { case DM_TYPE_BIO_BASED: + case DM_TYPE_DAX_BIO_BASED: cachep = _io_cache; pool_size = dm_get_reserved_bio_based_ios(); front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); @@ -3601,26 +2554,76 @@ void dm_free_md_mempools(struct dm_md_mempools *pools) kfree(pools); } -static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, - u32 flags) +struct dm_pr { + u64 old_key; + u64 new_key; + u32 flags; + bool fail_early; +}; + +static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, + void *data) { struct mapped_device *md = bdev->bd_disk->private_data; - const struct pr_ops *ops; - fmode_t mode; - int r; + struct dm_table *table; + struct dm_target *ti; + int ret = -ENOTTY, srcu_idx; - r = dm_grab_bdev_for_ioctl(md, &bdev, &mode); - if (r < 0) - return r; + table = dm_get_live_table(md, &srcu_idx); + if (!table || !dm_table_get_size(table)) + goto out; - ops = bdev->bd_disk->fops->pr_ops; - if (ops && ops->pr_register) - r = ops->pr_register(bdev, old_key, new_key, flags); - else - r = -EOPNOTSUPP; + /* We only support devices that have a single target */ + if (dm_table_get_num_targets(table) != 1) + goto out; + ti = dm_table_get_target(table, 0); - bdput(bdev); - return r; + ret = -EINVAL; + if (!ti->type->iterate_devices) + goto out; + + ret = ti->type->iterate_devices(ti, fn, data); +out: + dm_put_live_table(md, srcu_idx); + return ret; +} + +/* + * For register / unregister we need to manually call out to every path. + */ +static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, + sector_t start, sector_t len, void *data) +{ + struct dm_pr *pr = data; + const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; + + if (!ops || !ops->pr_register) + return -EOPNOTSUPP; + return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); +} + +static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, + u32 flags) +{ + struct dm_pr pr = { + .old_key = old_key, + .new_key = new_key, + .flags = flags, + .fail_early = true, + }; + int ret; + + ret = dm_call_pr(bdev, __dm_pr_register, &pr); + if (ret && new_key) { + /* unregister all paths if we failed to register any path */ + pr.old_key = new_key; + pr.new_key = 0; + pr.flags = 0; + pr.fail_early = false; + dm_call_pr(bdev, __dm_pr_register, &pr); + } + + return ret; } static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, @@ -3721,6 +2724,7 @@ static const struct block_device_operations dm_blk_dops = { .open = dm_blk_open, .release = dm_blk_close, .ioctl = dm_blk_ioctl, + .direct_access = dm_blk_direct_access, .getgeo = dm_blk_getgeo, .pr_ops = &dm_pr_ops, .owner = THIS_MODULE @@ -3738,18 +2742,6 @@ MODULE_PARM_DESC(major, "The major number of the device mapper"); module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); -module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools"); - -module_param(use_blk_mq, bool, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(use_blk_mq, "Use block multiqueue for request-based DM devices"); - -module_param(dm_mq_nr_hw_queues, uint, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(dm_mq_nr_hw_queues, "Number of hardware queues for request-based dm-mq devices"); - -module_param(dm_mq_queue_depth, uint, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(dm_mq_queue_depth, "Queue depth for request-based dm-mq devices"); - module_param(dm_numa_node, int, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 13a758ec0f88..f0aad08b9654 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -32,14 +33,6 @@ */ #define DM_STATUS_NOFLUSH_FLAG (1 << 0) -/* - * Type of table and mapped_device's mempool - */ -#define DM_TYPE_NONE 0 -#define DM_TYPE_BIO_BASED 1 -#define DM_TYPE_REQUEST_BASED 2 -#define DM_TYPE_MQ_REQUEST_BASED 3 - /* * List of devices that a metadevice uses and should open/close. */ @@ -75,8 +68,9 @@ unsigned dm_table_get_type(struct dm_table *t); struct target_type *dm_table_get_immutable_target_type(struct dm_table *t); struct dm_target *dm_table_get_immutable_target(struct dm_table *t); struct dm_target *dm_table_get_wildcard_target(struct dm_table *t); +bool dm_table_bio_based(struct dm_table *t); bool dm_table_request_based(struct dm_table *t); -bool dm_table_mq_request_based(struct dm_table *t); +bool dm_table_all_blk_mq_devices(struct dm_table *t); void dm_table_free_md_mempools(struct dm_table *t); struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t); @@ -161,16 +155,6 @@ void dm_interface_exit(void); /* * sysfs interface */ -struct dm_kobject_holder { - struct kobject kobj; - struct completion completion; -}; - -static inline struct completion *dm_get_completion_from_kobject(struct kobject *kobj) -{ - return &container_of(kobj, struct dm_kobject_holder, kobj)->completion; -} - int dm_sysfs_init(struct mapped_device *md); void dm_sysfs_exit(struct mapped_device *md); struct kobject *dm_kobject(struct mapped_device *md); @@ -212,8 +196,6 @@ int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, void dm_internal_suspend(struct mapped_device *md); void dm_internal_resume(struct mapped_device *md); -bool dm_use_blk_mq(struct mapped_device *md); - int dm_io_init(void); void dm_io_exit(void); @@ -228,18 +210,8 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t void dm_free_md_mempools(struct dm_md_mempools *pools); /* - * Helpers that are used by DM core + * Various helpers */ unsigned dm_get_reserved_bio_based_ios(void); -unsigned dm_get_reserved_rq_based_ios(void); - -static inline bool dm_message_test_buffer_overflow(char *result, unsigned maxlen) -{ - return !maxlen || strlen(result) + 1 >= maxlen; -} - -ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf); -ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md, - const char *buf, size_t count); #endif diff --git a/drivers/md/linear.c b/drivers/md/linear.c index b7fe7e9fc777..70ff888d25d0 100644 --- a/drivers/md/linear.c +++ b/drivers/md/linear.c @@ -221,7 +221,7 @@ static void linear_make_request(struct mddev *mddev, struct bio *bio) struct bio *split; sector_t start_sector, end_sector, data_offset; - if (unlikely(bio->bi_rw & REQ_FLUSH)) { + if (unlikely(bio->bi_rw & REQ_PREFLUSH)) { md_flush_request(mddev, bio); return; } @@ -252,7 +252,7 @@ static void linear_make_request(struct mddev *mddev, struct bio *bio) split->bi_iter.bi_sector = split->bi_iter.bi_sector - start_sector + data_offset; - if (unlikely((split->bi_rw & REQ_DISCARD) && + if (unlikely((bio_op(split) == REQ_OP_DISCARD) && !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) { /* Just ignore it */ bio_endio(split); diff --git a/drivers/md/md.c b/drivers/md/md.c index 866825f10b4c..1f123f5a29da 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -394,8 +394,9 @@ static void submit_flushes(struct work_struct *ws) bi->bi_end_io = md_end_flush; bi->bi_private = rdev; bi->bi_bdev = rdev->bdev; + bio_set_op_attrs(bi, REQ_OP_WRITE, WRITE_FLUSH); atomic_inc(&mddev->flush_pending); - submit_bio(WRITE_FLUSH, bi); + submit_bio(bi); rcu_read_lock(); rdev_dec_pending(rdev, mddev); } @@ -413,7 +414,7 @@ static void md_submit_flush_data(struct work_struct *ws) /* an empty barrier - all done */ bio_endio(bio); else { - bio->bi_rw &= ~REQ_FLUSH; + bio->bi_rw &= ~REQ_PREFLUSH; mddev->pers->make_request(mddev, bio); } @@ -742,9 +743,10 @@ void md_super_write(struct mddev *mddev, struct md_rdev *rdev, bio_add_page(bio, page, size, 0); bio->bi_private = rdev; bio->bi_end_io = super_written; + bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH_FUA); atomic_inc(&mddev->pending_writes); - submit_bio(WRITE_FLUSH_FUA, bio); + submit_bio(bio); } void md_super_wait(struct mddev *mddev) @@ -754,13 +756,14 @@ void md_super_wait(struct mddev *mddev) } int sync_page_io(struct md_rdev *rdev, sector_t sector, int size, - struct page *page, int rw, bool metadata_op) + struct page *page, int op, int op_flags, bool metadata_op) { struct bio *bio = bio_alloc_mddev(GFP_NOIO, 1, rdev->mddev); int ret; bio->bi_bdev = (metadata_op && rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev; + bio_set_op_attrs(bio, op, op_flags); if (metadata_op) bio->bi_iter.bi_sector = sector + rdev->sb_start; else if (rdev->mddev->reshape_position != MaxSector && @@ -770,7 +773,8 @@ int sync_page_io(struct md_rdev *rdev, sector_t sector, int size, else bio->bi_iter.bi_sector = sector + rdev->data_offset; bio_add_page(bio, page, size, 0); - submit_bio_wait(rw, bio); + + submit_bio_wait(bio); ret = !bio->bi_error; bio_put(bio); @@ -785,7 +789,7 @@ static int read_disk_sb(struct md_rdev *rdev, int size) if (rdev->sb_loaded) return 0; - if (!sync_page_io(rdev, 0, size, rdev->sb_page, READ, true)) + if (!sync_page_io(rdev, 0, size, rdev->sb_page, REQ_OP_READ, 0, true)) goto fail; rdev->sb_loaded = 1; return 0; @@ -1471,7 +1475,7 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ return -EINVAL; bb_sector = (long long)offset; if (!sync_page_io(rdev, bb_sector, sectors << 9, - rdev->bb_page, READ, true)) + rdev->bb_page, REQ_OP_READ, 0, true)) return -EIO; bbp = (u64 *)page_address(rdev->bb_page); rdev->badblocks.shift = sb->bblog_shift; diff --git a/drivers/md/md.h b/drivers/md/md.h index b5c4be73e6e4..b4f335245bd6 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -424,7 +424,7 @@ struct mddev { /* Generic flush handling. * The last to finish preflush schedules a worker to submit - * the rest of the request (without the REQ_FLUSH flag). + * the rest of the request (without the REQ_PREFLUSH flag). */ struct bio *flush_bio; atomic_t flush_pending; @@ -618,7 +618,8 @@ extern void md_super_write(struct mddev *mddev, struct md_rdev *rdev, sector_t sector, int size, struct page *page); extern void md_super_wait(struct mddev *mddev); extern int sync_page_io(struct md_rdev *rdev, sector_t sector, int size, - struct page *page, int rw, bool metadata_op); + struct page *page, int op, int op_flags, + bool metadata_op); extern void md_do_sync(struct md_thread *thread); extern void md_new_event(struct mddev *mddev); extern int md_allow_write(struct mddev *mddev); diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c index dd483bb2e111..72ea98e89e57 100644 --- a/drivers/md/multipath.c +++ b/drivers/md/multipath.c @@ -111,7 +111,7 @@ static void multipath_make_request(struct mddev *mddev, struct bio * bio) struct multipath_bh * mp_bh; struct multipath_info *multipath; - if (unlikely(bio->bi_rw & REQ_FLUSH)) { + if (unlikely(bio->bi_rw & REQ_PREFLUSH)) { md_flush_request(mddev, bio); return; } diff --git a/drivers/md/persistent-data/dm-btree.c b/drivers/md/persistent-data/dm-btree.c index ea3d3b656fd0..2cc1877804c2 100644 --- a/drivers/md/persistent-data/dm-btree.c +++ b/drivers/md/persistent-data/dm-btree.c @@ -429,7 +429,14 @@ static int dm_btree_lookup_next_single(struct dm_btree_info *info, dm_block_t ro if (flags & INTERNAL_NODE) { i = lower_bound(n, key); - if (i < 0 || i >= nr_entries) { + if (i < 0) { + /* + * avoid early -ENODATA return when all entries are + * higher than the search @key. + */ + i = 0; + } + if (i >= nr_entries) { r = -ENODATA; goto out; } diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index 34783a3c8b3c..c3d439083212 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -458,7 +458,7 @@ static void raid0_make_request(struct mddev *mddev, struct bio *bio) struct md_rdev *tmp_dev; struct bio *split; - if (unlikely(bio->bi_rw & REQ_FLUSH)) { + if (unlikely(bio->bi_rw & REQ_PREFLUSH)) { md_flush_request(mddev, bio); return; } @@ -488,7 +488,7 @@ static void raid0_make_request(struct mddev *mddev, struct bio *bio) split->bi_iter.bi_sector = sector + zone->dev_start + tmp_dev->data_offset; - if (unlikely((split->bi_rw & REQ_DISCARD) && + if (unlikely((bio_op(split) == REQ_OP_DISCARD) && !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) { /* Just ignore it */ bio_endio(split); diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index c7c8cde0ab21..4e6da4497553 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -759,7 +759,7 @@ static void flush_pending_writes(struct r1conf *conf) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; bio->bi_next = NULL; - if (unlikely((bio->bi_rw & REQ_DISCARD) && + if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) /* Just ignore it */ bio_endio(bio); @@ -1033,7 +1033,7 @@ static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; bio->bi_next = NULL; - if (unlikely((bio->bi_rw & REQ_DISCARD) && + if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) /* Just ignore it */ bio_endio(bio); @@ -1053,12 +1053,11 @@ static void raid1_make_request(struct mddev *mddev, struct bio * bio) int i, disks; struct bitmap *bitmap; unsigned long flags; + const int op = bio_op(bio); const int rw = bio_data_dir(bio); const unsigned long do_sync = (bio->bi_rw & REQ_SYNC); - const unsigned long do_flush_fua = (bio->bi_rw & (REQ_FLUSH | REQ_FUA)); - const unsigned long do_discard = (bio->bi_rw - & (REQ_DISCARD | REQ_SECURE)); - const unsigned long do_same = (bio->bi_rw & REQ_WRITE_SAME); + const unsigned long do_flush_fua = (bio->bi_rw & + (REQ_PREFLUSH | REQ_FUA)); struct md_rdev *blocked_rdev; struct blk_plug_cb *cb; struct raid1_plug_cb *plug = NULL; @@ -1106,7 +1105,7 @@ static void raid1_make_request(struct mddev *mddev, struct bio * bio) bitmap = mddev->bitmap; /* - * make_request() can abort the operation when READA is being + * make_request() can abort the operation when read-ahead is being * used and no empty request is available. * */ @@ -1166,7 +1165,7 @@ read_again: mirror->rdev->data_offset; read_bio->bi_bdev = mirror->rdev->bdev; read_bio->bi_end_io = raid1_end_read_request; - read_bio->bi_rw = READ | do_sync; + bio_set_op_attrs(read_bio, op, do_sync); read_bio->bi_private = r1_bio; if (max_sectors < r1_bio->sectors) { @@ -1376,8 +1375,7 @@ read_again: conf->mirrors[i].rdev->data_offset); mbio->bi_bdev = conf->mirrors[i].rdev->bdev; mbio->bi_end_io = raid1_end_write_request; - mbio->bi_rw = - WRITE | do_flush_fua | do_sync | do_discard | do_same; + bio_set_op_attrs(mbio, op, do_flush_fua | do_sync); mbio->bi_private = r1_bio; atomic_inc(&r1_bio->remaining); @@ -1771,7 +1769,7 @@ static void end_sync_write(struct bio *bio) static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector, int sectors, struct page *page, int rw) { - if (sync_page_io(rdev, sector, sectors << 9, page, rw, false)) + if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false)) /* success */ return 1; if (rw == WRITE) { @@ -1825,7 +1823,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio) rdev = conf->mirrors[d].rdev; if (sync_page_io(rdev, sect, s<<9, bio->bi_io_vec[idx].bv_page, - READ, false)) { + REQ_OP_READ, 0, false)) { success = 1; break; } @@ -2030,7 +2028,7 @@ static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio) !test_bit(MD_RECOVERY_SYNC, &mddev->recovery)))) continue; - wbio->bi_rw = WRITE; + bio_set_op_attrs(wbio, REQ_OP_WRITE, 0); wbio->bi_end_io = end_sync_write; atomic_inc(&r1_bio->remaining); md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio)); @@ -2090,7 +2088,7 @@ static void fix_read_error(struct r1conf *conf, int read_disk, is_badblock(rdev, sect, s, &first_bad, &bad_sectors) == 0 && sync_page_io(rdev, sect, s<<9, - conf->tmppage, READ, false)) + conf->tmppage, REQ_OP_READ, 0, false)) success = 1; else { d++; @@ -2201,14 +2199,15 @@ static int narrow_write_error(struct r1bio *r1_bio, int i) wbio = bio_clone_mddev(r1_bio->master_bio, GFP_NOIO, mddev); } - wbio->bi_rw = WRITE; + bio_set_op_attrs(wbio, REQ_OP_WRITE, 0); wbio->bi_iter.bi_sector = r1_bio->sector; wbio->bi_iter.bi_size = r1_bio->sectors << 9; bio_trim(wbio, sector - r1_bio->sector, sectors); wbio->bi_iter.bi_sector += rdev->data_offset; wbio->bi_bdev = rdev->bdev; - if (submit_bio_wait(WRITE, wbio) < 0) + + if (submit_bio_wait(wbio) < 0) /* failure! */ ok = rdev_set_badblocks(rdev, sector, sectors, 0) @@ -2343,7 +2342,7 @@ read_more: bio->bi_iter.bi_sector = r1_bio->sector + rdev->data_offset; bio->bi_bdev = rdev->bdev; bio->bi_end_io = raid1_end_read_request; - bio->bi_rw = READ | do_sync; + bio_set_op_attrs(bio, REQ_OP_READ, do_sync); bio->bi_private = r1_bio; if (max_sectors < r1_bio->sectors) { /* Drat - have to split this up more */ @@ -2571,7 +2570,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, if (i < conf->raid_disks) still_degraded = 1; } else if (!test_bit(In_sync, &rdev->flags)) { - bio->bi_rw = WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio->bi_end_io = end_sync_write; write_targets ++; } else { @@ -2598,7 +2597,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, if (disk < 0) disk = i; } - bio->bi_rw = READ; + bio_set_op_attrs(bio, REQ_OP_READ, 0); bio->bi_end_io = end_sync_read; read_targets++; } else if (!test_bit(WriteErrorSeen, &rdev->flags) && @@ -2610,7 +2609,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, * if we are doing resync or repair. Otherwise, leave * this device alone for this sync request. */ - bio->bi_rw = WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio->bi_end_io = end_sync_write; write_targets++; } diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index c7de2a53e625..26ae74fd0d01 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -865,7 +865,7 @@ static void flush_pending_writes(struct r10conf *conf) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; bio->bi_next = NULL; - if (unlikely((bio->bi_rw & REQ_DISCARD) && + if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) /* Just ignore it */ bio_endio(bio); @@ -1041,7 +1041,7 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; bio->bi_next = NULL; - if (unlikely((bio->bi_rw & REQ_DISCARD) && + if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) /* Just ignore it */ bio_endio(bio); @@ -1058,12 +1058,10 @@ static void __make_request(struct mddev *mddev, struct bio *bio) struct r10bio *r10_bio; struct bio *read_bio; int i; + const int op = bio_op(bio); const int rw = bio_data_dir(bio); const unsigned long do_sync = (bio->bi_rw & REQ_SYNC); const unsigned long do_fua = (bio->bi_rw & REQ_FUA); - const unsigned long do_discard = (bio->bi_rw - & (REQ_DISCARD | REQ_SECURE)); - const unsigned long do_same = (bio->bi_rw & REQ_WRITE_SAME); unsigned long flags; struct md_rdev *blocked_rdev; struct blk_plug_cb *cb; @@ -1156,7 +1154,7 @@ read_again: choose_data_offset(r10_bio, rdev); read_bio->bi_bdev = rdev->bdev; read_bio->bi_end_io = raid10_end_read_request; - read_bio->bi_rw = READ | do_sync; + bio_set_op_attrs(read_bio, op, do_sync); read_bio->bi_private = r10_bio; if (max_sectors < r10_bio->sectors) { @@ -1363,8 +1361,7 @@ retry_write: rdev)); mbio->bi_bdev = rdev->bdev; mbio->bi_end_io = raid10_end_write_request; - mbio->bi_rw = - WRITE | do_sync | do_fua | do_discard | do_same; + bio_set_op_attrs(mbio, op, do_sync | do_fua); mbio->bi_private = r10_bio; atomic_inc(&r10_bio->remaining); @@ -1406,8 +1403,7 @@ retry_write: r10_bio, rdev)); mbio->bi_bdev = rdev->bdev; mbio->bi_end_io = raid10_end_write_request; - mbio->bi_rw = - WRITE | do_sync | do_fua | do_discard | do_same; + bio_set_op_attrs(mbio, op, do_sync | do_fua); mbio->bi_private = r10_bio; atomic_inc(&r10_bio->remaining); @@ -1450,7 +1446,7 @@ static void raid10_make_request(struct mddev *mddev, struct bio *bio) struct bio *split; - if (unlikely(bio->bi_rw & REQ_FLUSH)) { + if (unlikely(bio->bi_rw & REQ_PREFLUSH)) { md_flush_request(mddev, bio); return; } @@ -1992,10 +1988,10 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) tbio->bi_vcnt = vcnt; tbio->bi_iter.bi_size = fbio->bi_iter.bi_size; - tbio->bi_rw = WRITE; tbio->bi_private = r10_bio; tbio->bi_iter.bi_sector = r10_bio->devs[i].addr; tbio->bi_end_io = end_sync_write; + bio_set_op_attrs(tbio, REQ_OP_WRITE, 0); bio_copy_data(tbio, fbio); @@ -2078,7 +2074,7 @@ static void fix_recovery_read_error(struct r10bio *r10_bio) addr, s << 9, bio->bi_io_vec[idx].bv_page, - READ, false); + REQ_OP_READ, 0, false); if (ok) { rdev = conf->mirrors[dw].rdev; addr = r10_bio->devs[1].addr + sect; @@ -2086,7 +2082,7 @@ static void fix_recovery_read_error(struct r10bio *r10_bio) addr, s << 9, bio->bi_io_vec[idx].bv_page, - WRITE, false); + REQ_OP_WRITE, 0, false); if (!ok) { set_bit(WriteErrorSeen, &rdev->flags); if (!test_and_set_bit(WantReplacement, @@ -2213,7 +2209,7 @@ static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors) && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags))) return -1; - if (sync_page_io(rdev, sector, sectors << 9, page, rw, false)) + if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false)) /* success */ return 1; if (rw == WRITE) { @@ -2299,7 +2295,8 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10 r10_bio->devs[sl].addr + sect, s<<9, - conf->tmppage, READ, false); + conf->tmppage, + REQ_OP_READ, 0, false); rdev_dec_pending(rdev, mddev); rcu_read_lock(); if (success) @@ -2474,7 +2471,9 @@ static int narrow_write_error(struct r10bio *r10_bio, int i) choose_data_offset(r10_bio, rdev) + (sector - r10_bio->sector)); wbio->bi_bdev = rdev->bdev; - if (submit_bio_wait(WRITE, wbio) < 0) + bio_set_op_attrs(wbio, REQ_OP_WRITE, 0); + + if (submit_bio_wait(wbio) < 0) /* Failure! */ ok = rdev_set_badblocks(rdev, sector, sectors, 0) @@ -2548,7 +2547,7 @@ read_more: bio->bi_iter.bi_sector = r10_bio->devs[slot].addr + choose_data_offset(r10_bio, rdev); bio->bi_bdev = rdev->bdev; - bio->bi_rw = READ | do_sync; + bio_set_op_attrs(bio, REQ_OP_READ, do_sync); bio->bi_private = r10_bio; bio->bi_end_io = raid10_end_read_request; if (max_sectors < r10_bio->sectors) { @@ -3038,7 +3037,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, biolist = bio; bio->bi_private = r10_bio; bio->bi_end_io = end_sync_read; - bio->bi_rw = READ; + bio_set_op_attrs(bio, REQ_OP_READ, 0); from_addr = r10_bio->devs[j].addr; bio->bi_iter.bi_sector = from_addr + rdev->data_offset; @@ -3064,7 +3063,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, biolist = bio; bio->bi_private = r10_bio; bio->bi_end_io = end_sync_write; - bio->bi_rw = WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio->bi_iter.bi_sector = to_addr + rdev->data_offset; bio->bi_bdev = rdev->bdev; @@ -3093,7 +3092,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, biolist = bio; bio->bi_private = r10_bio; bio->bi_end_io = end_sync_write; - bio->bi_rw = WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio->bi_iter.bi_sector = to_addr + rdev->data_offset; bio->bi_bdev = rdev->bdev; @@ -3213,7 +3212,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, biolist = bio; bio->bi_private = r10_bio; bio->bi_end_io = end_sync_read; - bio->bi_rw = READ; + bio_set_op_attrs(bio, REQ_OP_READ, 0); bio->bi_iter.bi_sector = sector + conf->mirrors[d].rdev->data_offset; bio->bi_bdev = conf->mirrors[d].rdev->bdev; @@ -3235,7 +3234,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, biolist = bio; bio->bi_private = r10_bio; bio->bi_end_io = end_sync_write; - bio->bi_rw = WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio->bi_iter.bi_sector = sector + conf->mirrors[d].replacement->data_offset; bio->bi_bdev = conf->mirrors[d].replacement->bdev; @@ -4320,7 +4319,7 @@ read_more: + rdev->data_offset); read_bio->bi_private = r10_bio; read_bio->bi_end_io = end_sync_read; - read_bio->bi_rw = READ; + bio_set_op_attrs(read_bio, REQ_OP_READ, 0); read_bio->bi_flags &= (~0UL << BIO_RESET_BITS); read_bio->bi_error = 0; read_bio->bi_vcnt = 0; @@ -4354,7 +4353,7 @@ read_more: rdev2->new_data_offset; b->bi_private = r10_bio; b->bi_end_io = end_reshape_write; - b->bi_rw = WRITE; + bio_set_op_attrs(b, REQ_OP_WRITE, 0); b->bi_next = blist; blist = b; } @@ -4522,7 +4521,7 @@ static int handle_reshape_read_error(struct mddev *mddev, addr, s << 9, bvec[idx].bv_page, - READ, false); + REQ_OP_READ, 0, false); if (success) break; failed: diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c index e889e2deb7b3..5504ce2bac06 100644 --- a/drivers/md/raid5-cache.c +++ b/drivers/md/raid5-cache.c @@ -254,14 +254,14 @@ static void r5l_submit_current_io(struct r5l_log *log) __r5l_set_io_unit_state(io, IO_UNIT_IO_START); spin_unlock_irqrestore(&log->io_list_lock, flags); - submit_bio(WRITE, io->current_bio); + submit_bio(io->current_bio); } static struct bio *r5l_bio_alloc(struct r5l_log *log) { struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, log->bs); - bio->bi_rw = WRITE; + bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio->bi_bdev = log->rdev->bdev; bio->bi_iter.bi_sector = log->rdev->data_offset + log->log_start; @@ -373,7 +373,7 @@ static void r5l_append_payload_page(struct r5l_log *log, struct page *page) io->current_bio = r5l_bio_alloc(log); bio_chain(io->current_bio, prev); - submit_bio(WRITE, prev); + submit_bio(prev); } if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0)) @@ -536,7 +536,7 @@ int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio) bio_endio(bio); return 0; } - bio->bi_rw &= ~REQ_FLUSH; + bio->bi_rw &= ~REQ_PREFLUSH; return -EAGAIN; } @@ -686,7 +686,8 @@ void r5l_flush_stripe_to_raid(struct r5l_log *log) bio_reset(&log->flush_bio); log->flush_bio.bi_bdev = log->rdev->bdev; log->flush_bio.bi_end_io = r5l_log_flush_endio; - submit_bio(WRITE_FLUSH, &log->flush_bio); + bio_set_op_attrs(&log->flush_bio, REQ_OP_WRITE, WRITE_FLUSH); + submit_bio(&log->flush_bio); } static void r5l_write_super(struct r5l_log *log, sector_t cp); @@ -881,7 +882,8 @@ static int r5l_read_meta_block(struct r5l_log *log, struct r5l_meta_block *mb; u32 crc, stored_crc; - if (!sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page, READ, false)) + if (!sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page, REQ_OP_READ, 0, + false)) return -EIO; mb = page_address(page); @@ -926,7 +928,8 @@ static int r5l_recovery_flush_one_stripe(struct r5l_log *log, &disk_index, sh); sync_page_io(log->rdev, *log_offset, PAGE_SIZE, - sh->dev[disk_index].page, READ, false); + sh->dev[disk_index].page, REQ_OP_READ, 0, + false); sh->dev[disk_index].log_checksum = le32_to_cpu(payload->checksum[0]); set_bit(R5_Wantwrite, &sh->dev[disk_index].flags); @@ -934,7 +937,8 @@ static int r5l_recovery_flush_one_stripe(struct r5l_log *log, } else { disk_index = sh->pd_idx; sync_page_io(log->rdev, *log_offset, PAGE_SIZE, - sh->dev[disk_index].page, READ, false); + sh->dev[disk_index].page, REQ_OP_READ, 0, + false); sh->dev[disk_index].log_checksum = le32_to_cpu(payload->checksum[0]); set_bit(R5_Wantwrite, &sh->dev[disk_index].flags); @@ -944,7 +948,7 @@ static int r5l_recovery_flush_one_stripe(struct r5l_log *log, sync_page_io(log->rdev, r5l_ring_add(log, *log_offset, BLOCK_SECTORS), PAGE_SIZE, sh->dev[disk_index].page, - READ, false); + REQ_OP_READ, 0, false); sh->dev[disk_index].log_checksum = le32_to_cpu(payload->checksum[1]); set_bit(R5_Wantwrite, @@ -986,11 +990,13 @@ static int r5l_recovery_flush_one_stripe(struct r5l_log *log, rdev = rcu_dereference(conf->disks[disk_index].rdev); if (rdev) sync_page_io(rdev, stripe_sect, PAGE_SIZE, - sh->dev[disk_index].page, WRITE, false); + sh->dev[disk_index].page, REQ_OP_WRITE, 0, + false); rrdev = rcu_dereference(conf->disks[disk_index].replacement); if (rrdev) sync_page_io(rrdev, stripe_sect, PAGE_SIZE, - sh->dev[disk_index].page, WRITE, false); + sh->dev[disk_index].page, REQ_OP_WRITE, 0, + false); } raid5_release_stripe(sh); return 0; @@ -1062,7 +1068,8 @@ static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos, crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE); mb->checksum = cpu_to_le32(crc); - if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, WRITE_FUA, false)) { + if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE, + WRITE_FUA, false)) { __free_page(page); return -EIO; } @@ -1137,7 +1144,7 @@ static int r5l_load_log(struct r5l_log *log) if (!page) return -ENOMEM; - if (!sync_page_io(rdev, cp, PAGE_SIZE, page, READ, false)) { + if (!sync_page_io(rdev, cp, PAGE_SIZE, page, REQ_OP_READ, 0, false)) { ret = -EIO; goto ioerr; } diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 8959e6dd31dd..6953d78297b0 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -806,7 +806,8 @@ static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh dd_idx = 0; while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) dd_idx++; - if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw) + if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw || + bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite)) goto unlock_out; if (head->batch_head) { @@ -891,29 +892,28 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) if (r5l_write_stripe(conf->log, sh) == 0) return; for (i = disks; i--; ) { - int rw; + int op, op_flags = 0; int replace_only = 0; struct bio *bi, *rbi; struct md_rdev *rdev, *rrdev = NULL; sh = head_sh; if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { + op = REQ_OP_WRITE; if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) - rw = WRITE_FUA; - else - rw = WRITE; + op_flags = WRITE_FUA; if (test_bit(R5_Discard, &sh->dev[i].flags)) - rw |= REQ_DISCARD; + op = REQ_OP_DISCARD; } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) - rw = READ; + op = REQ_OP_READ; else if (test_and_clear_bit(R5_WantReplace, &sh->dev[i].flags)) { - rw = WRITE; + op = REQ_OP_WRITE; replace_only = 1; } else continue; if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) - rw |= REQ_SYNC; + op_flags |= REQ_SYNC; again: bi = &sh->dev[i].req; @@ -927,7 +927,7 @@ again: rdev = rrdev; rrdev = NULL; } - if (rw & WRITE) { + if (op_is_write(op)) { if (replace_only) rdev = NULL; if (rdev == rrdev) @@ -953,7 +953,7 @@ again: * need to check for writes. We never accept write errors * on the replacement, so we don't to check rrdev. */ - while ((rw & WRITE) && rdev && + while (op_is_write(op) && rdev && test_bit(WriteErrorSeen, &rdev->flags)) { sector_t first_bad; int bad_sectors; @@ -995,13 +995,13 @@ again: bio_reset(bi); bi->bi_bdev = rdev->bdev; - bi->bi_rw = rw; - bi->bi_end_io = (rw & WRITE) + bio_set_op_attrs(bi, op, op_flags); + bi->bi_end_io = op_is_write(op) ? raid5_end_write_request : raid5_end_read_request; bi->bi_private = sh; - pr_debug("%s: for %llu schedule op %ld on disc %d\n", + pr_debug("%s: for %llu schedule op %d on disc %d\n", __func__, (unsigned long long)sh->sector, bi->bi_rw, i); atomic_inc(&sh->count); @@ -1027,7 +1027,7 @@ again: * If this is discard request, set bi_vcnt 0. We don't * want to confuse SCSI because SCSI will replace payload */ - if (rw & REQ_DISCARD) + if (op == REQ_OP_DISCARD) bi->bi_vcnt = 0; if (rrdev) set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); @@ -1047,12 +1047,12 @@ again: bio_reset(rbi); rbi->bi_bdev = rrdev->bdev; - rbi->bi_rw = rw; - BUG_ON(!(rw & WRITE)); + bio_set_op_attrs(rbi, op, op_flags); + BUG_ON(!op_is_write(op)); rbi->bi_end_io = raid5_end_write_request; rbi->bi_private = sh; - pr_debug("%s: for %llu schedule op %ld on " + pr_debug("%s: for %llu schedule op %d on " "replacement disc %d\n", __func__, (unsigned long long)sh->sector, rbi->bi_rw, i); @@ -1076,7 +1076,7 @@ again: * If this is discard request, set bi_vcnt 0. We don't * want to confuse SCSI because SCSI will replace payload */ - if (rw & REQ_DISCARD) + if (op == REQ_OP_DISCARD) rbi->bi_vcnt = 0; if (conf->mddev->gendisk) trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), @@ -1085,9 +1085,9 @@ again: generic_make_request(rbi); } if (!rdev && !rrdev) { - if (rw & WRITE) + if (op_is_write(op)) set_bit(STRIPE_DEGRADED, &sh->state); - pr_debug("skip op %ld on disc %d for sector %llu\n", + pr_debug("skip op %d on disc %d for sector %llu\n", bi->bi_rw, i, (unsigned long long)sh->sector); clear_bit(R5_LOCKED, &sh->dev[i].flags); set_bit(STRIPE_HANDLE, &sh->state); @@ -1623,7 +1623,7 @@ again: set_bit(R5_WantFUA, &dev->flags); if (wbi->bi_rw & REQ_SYNC) set_bit(R5_SyncIO, &dev->flags); - if (wbi->bi_rw & REQ_DISCARD) + if (bio_op(wbi) == REQ_OP_DISCARD) set_bit(R5_Discard, &dev->flags); else { tx = async_copy_data(1, wbi, &dev->page, @@ -5150,7 +5150,7 @@ static void raid5_make_request(struct mddev *mddev, struct bio * bi) DEFINE_WAIT(w); bool do_prepare; - if (unlikely(bi->bi_rw & REQ_FLUSH)) { + if (unlikely(bi->bi_rw & REQ_PREFLUSH)) { int ret = r5l_handle_flush_request(conf->log, bi); if (ret == 0) @@ -5176,7 +5176,7 @@ static void raid5_make_request(struct mddev *mddev, struct bio * bi) return; } - if (unlikely(bi->bi_rw & REQ_DISCARD)) { + if (unlikely(bio_op(bi) == REQ_OP_DISCARD)) { make_discard_request(mddev, bi); return; } @@ -5233,7 +5233,7 @@ static void raid5_make_request(struct mddev *mddev, struct bio * bi) (unsigned long long)logical_sector); sh = raid5_get_active_stripe(conf, new_sector, previous, - (bi->bi_rw&RWA_MASK), 0); + (bi->bi_rw & REQ_RAHEAD), 0); if (sh) { if (unlikely(previous)) { /* expansion might have moved on while waiting for a diff --git a/drivers/media/Kconfig b/drivers/media/Kconfig index a8518fb3bca7..962f2a9a6614 100644 --- a/drivers/media/Kconfig +++ b/drivers/media/Kconfig @@ -80,6 +80,9 @@ config MEDIA_RC_SUPPORT Say Y when you have a TV or an IR device. +config MEDIA_CEC_EDID + bool + # # Media controller # Selectable only for webcam/grabbers, as other drivers don't use it diff --git a/drivers/media/Makefile b/drivers/media/Makefile index e608bbce0c35..081a7866fd44 100644 --- a/drivers/media/Makefile +++ b/drivers/media/Makefile @@ -2,6 +2,10 @@ # Makefile for the kernel multimedia device drivers. # +ifeq ($(CONFIG_MEDIA_CEC_EDID),y) + obj-$(CONFIG_MEDIA_SUPPORT) += cec-edid.o +endif + media-objs := media-device.o media-devnode.o media-entity.o # diff --git a/drivers/media/cec-edid.c b/drivers/media/cec-edid.c new file mode 100644 index 000000000000..70018247bdda --- /dev/null +++ b/drivers/media/cec-edid.c @@ -0,0 +1,168 @@ +/* + * cec-edid - HDMI Consumer Electronics Control EDID & CEC helper functions + * + * Copyright 2016 Cisco Systems, Inc. and/or its affiliates. All rights reserved. + * + * This program is free software; you may redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include + +/* + * This EDID is expected to be a CEA-861 compliant, which means that there are + * at least two blocks and one or more of the extensions blocks are CEA-861 + * blocks. + * + * The returned location is guaranteed to be < size - 1. + */ +static unsigned int cec_get_edid_spa_location(const u8 *edid, unsigned int size) +{ + unsigned int blocks = size / 128; + unsigned int block; + u8 d; + + /* Sanity check: at least 2 blocks and a multiple of the block size */ + if (blocks < 2 || size % 128) + return 0; + + /* + * If there are fewer extension blocks than the size, then update + * 'blocks'. It is allowed to have more extension blocks than the size, + * since some hardware can only read e.g. 256 bytes of the EDID, even + * though more blocks are present. The first CEA-861 extension block + * should normally be in block 1 anyway. + */ + if (edid[0x7e] + 1 < blocks) + blocks = edid[0x7e] + 1; + + for (block = 1; block < blocks; block++) { + unsigned int offset = block * 128; + + /* Skip any non-CEA-861 extension blocks */ + if (edid[offset] != 0x02 || edid[offset + 1] != 0x03) + continue; + + /* search Vendor Specific Data Block (tag 3) */ + d = edid[offset + 2] & 0x7f; + /* Check if there are Data Blocks */ + if (d <= 4) + continue; + if (d > 4) { + unsigned int i = offset + 4; + unsigned int end = offset + d; + + /* Note: 'end' is always < 'size' */ + do { + u8 tag = edid[i] >> 5; + u8 len = edid[i] & 0x1f; + + if (tag == 3 && len >= 5 && i + len <= end) + return i + 4; + i += len + 1; + } while (i < end); + } + } + return 0; +} + +u16 cec_get_edid_phys_addr(const u8 *edid, unsigned int size, + unsigned int *offset) +{ + unsigned int loc = cec_get_edid_spa_location(edid, size); + + if (offset) + *offset = loc; + if (loc == 0) + return CEC_PHYS_ADDR_INVALID; + return (edid[loc] << 8) | edid[loc + 1]; +} +EXPORT_SYMBOL_GPL(cec_get_edid_phys_addr); + +void cec_set_edid_phys_addr(u8 *edid, unsigned int size, u16 phys_addr) +{ + unsigned int loc = cec_get_edid_spa_location(edid, size); + u8 sum = 0; + unsigned int i; + + if (loc == 0) + return; + edid[loc] = phys_addr >> 8; + edid[loc + 1] = phys_addr & 0xff; + loc &= ~0x7f; + + /* update the checksum */ + for (i = loc; i < loc + 127; i++) + sum += edid[i]; + edid[i] = 256 - sum; +} +EXPORT_SYMBOL_GPL(cec_set_edid_phys_addr); + +u16 cec_phys_addr_for_input(u16 phys_addr, u8 input) +{ + /* Check if input is sane */ + if (WARN_ON(input == 0 || input > 0xf)) + return CEC_PHYS_ADDR_INVALID; + + if (phys_addr == 0) + return input << 12; + + if ((phys_addr & 0x0fff) == 0) + return phys_addr | (input << 8); + + if ((phys_addr & 0x00ff) == 0) + return phys_addr | (input << 4); + + if ((phys_addr & 0x000f) == 0) + return phys_addr | input; + + /* + * All nibbles are used so no valid physical addresses can be assigned + * to the input. + */ + return CEC_PHYS_ADDR_INVALID; +} +EXPORT_SYMBOL_GPL(cec_phys_addr_for_input); + +int cec_phys_addr_validate(u16 phys_addr, u16 *parent, u16 *port) +{ + int i; + + if (parent) + *parent = phys_addr; + if (port) + *port = 0; + if (phys_addr == CEC_PHYS_ADDR_INVALID) + return 0; + for (i = 0; i < 16; i += 4) + if (phys_addr & (0xf << i)) + break; + if (i == 16) + return 0; + if (parent) + *parent = phys_addr & (0xfff0 << i); + if (port) + *port = (phys_addr >> i) & 0xf; + for (i += 4; i < 16; i += 4) + if ((phys_addr & (0xf << i)) == 0) + return -EINVAL; + return 0; +} +EXPORT_SYMBOL_GPL(cec_phys_addr_validate); + +MODULE_AUTHOR("Hans Verkuil "); +MODULE_DESCRIPTION("CEC EDID helper functions"); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c index cf1dadd0be9e..3ec3cebe62b9 100644 --- a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c +++ b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c @@ -777,7 +777,7 @@ static void precalculate_color(struct tpg_data *tpg, int k) * Remember that r, g and b are still in the 0 - 0xff0 range. */ if (tpg->real_rgb_range == V4L2_DV_RGB_RANGE_LIMITED && - tpg->rgb_range == V4L2_DV_RGB_RANGE_FULL) { + tpg->rgb_range == V4L2_DV_RGB_RANGE_FULL && !tpg->is_yuv) { /* * Convert from full range (which is what r, g and b are) * to limited range (which is the 'real' RGB range), which @@ -787,7 +787,7 @@ static void precalculate_color(struct tpg_data *tpg, int k) g = (g * 219) / 255 + (16 << 4); b = (b * 219) / 255 + (16 << 4); } else if (tpg->real_rgb_range != V4L2_DV_RGB_RANGE_LIMITED && - tpg->rgb_range == V4L2_DV_RGB_RANGE_LIMITED) { + tpg->rgb_range == V4L2_DV_RGB_RANGE_LIMITED && !tpg->is_yuv) { /* * Clamp r, g and b to the limited range and convert to full * range since that's what we deliver. diff --git a/drivers/media/dvb-core/demux.h b/drivers/media/dvb-core/demux.h index 6d3b95b8939d..7f1dffef4353 100644 --- a/drivers/media/dvb-core/demux.h +++ b/drivers/media/dvb-core/demux.h @@ -143,7 +143,7 @@ struct dmx_ts_feed { int type, enum dmx_ts_pes pes_type, size_t circular_buffer_size, - struct timespec timeout); + ktime_t timeout); int (*start_filtering)(struct dmx_ts_feed *feed); int (*stop_filtering)(struct dmx_ts_feed *feed); }; diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c index a168cbe1c998..7b67e1dd97fd 100644 --- a/drivers/media/dvb-core/dmxdev.c +++ b/drivers/media/dvb-core/dmxdev.c @@ -556,7 +556,7 @@ static int dvb_dmxdev_start_feed(struct dmxdev *dmxdev, struct dmxdev_filter *filter, struct dmxdev_feed *feed) { - struct timespec timeout = { 0 }; + ktime_t timeout = ktime_set(0, 0); struct dmx_pes_filter_params *para = &filter->params.pes; dmx_output_t otype; int ret; diff --git a/drivers/media/dvb-core/dvb_ca_en50221.c b/drivers/media/dvb-core/dvb_ca_en50221.c index f82cd1ff4f3a..b5b5b195ea7f 100644 --- a/drivers/media/dvb-core/dvb_ca_en50221.c +++ b/drivers/media/dvb-core/dvb_ca_en50221.c @@ -123,6 +123,7 @@ struct dvb_ca_slot { /* Private CA-interface information */ struct dvb_ca_private { + struct kref refcount; /* pointer back to the public data structure */ struct dvb_ca_en50221 *pub; @@ -161,6 +162,34 @@ struct dvb_ca_private { struct mutex ioctl_mutex; }; +static void dvb_ca_private_free(struct dvb_ca_private *ca) +{ + unsigned int i; + + dvb_unregister_device(ca->dvbdev); + for (i = 0; i < ca->slot_count; i++) + vfree(ca->slot_info[i].rx_buffer.data); + + kfree(ca->slot_info); + kfree(ca); +} + +static void dvb_ca_private_release(struct kref *ref) +{ + struct dvb_ca_private *ca = container_of(ref, struct dvb_ca_private, refcount); + dvb_ca_private_free(ca); +} + +static void dvb_ca_private_get(struct dvb_ca_private *ca) +{ + kref_get(&ca->refcount); +} + +static void dvb_ca_private_put(struct dvb_ca_private *ca) +{ + kref_put(&ca->refcount, dvb_ca_private_release); +} + static void dvb_ca_en50221_thread_wakeup(struct dvb_ca_private *ca); static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot, u8 * ebuf, int ecount); static int dvb_ca_en50221_write_data(struct dvb_ca_private *ca, int slot, u8 * ebuf, int ecount); @@ -1558,6 +1587,8 @@ static int dvb_ca_en50221_io_open(struct inode *inode, struct file *file) dvb_ca_en50221_thread_update_delay(ca); dvb_ca_en50221_thread_wakeup(ca); + dvb_ca_private_get(ca); + return 0; } @@ -1586,6 +1617,8 @@ static int dvb_ca_en50221_io_release(struct inode *inode, struct file *file) module_put(ca->pub->owner); + dvb_ca_private_put(ca); + return err; } @@ -1681,6 +1714,7 @@ int dvb_ca_en50221_init(struct dvb_adapter *dvb_adapter, ret = -ENOMEM; goto exit; } + kref_init(&ca->refcount); ca->pub = pubca; ca->flags = flags; ca->slot_count = slot_count; @@ -1759,10 +1793,7 @@ void dvb_ca_en50221_release(struct dvb_ca_en50221 *pubca) for (i = 0; i < ca->slot_count; i++) { dvb_ca_en50221_slot_shutdown(ca, i); - vfree(ca->slot_info[i].rx_buffer.data); } - kfree(ca->slot_info); - dvb_unregister_device(ca->dvbdev); - kfree(ca); + dvb_ca_private_put(ca); pubca->private = NULL; } diff --git a/drivers/media/dvb-core/dvb_demux.c b/drivers/media/dvb-core/dvb_demux.c index 0cc5e935166c..a0cf7b0d03e8 100644 --- a/drivers/media/dvb-core/dvb_demux.c +++ b/drivers/media/dvb-core/dvb_demux.c @@ -398,28 +398,23 @@ static void dvb_dmx_swfilter_packet(struct dvb_demux *demux, const u8 *buf) int dvr_done = 0; if (dvb_demux_speedcheck) { - struct timespec cur_time, delta_time; + ktime_t cur_time; u64 speed_bytes, speed_timedelta; demux->speed_pkts_cnt++; /* show speed every SPEED_PKTS_INTERVAL packets */ if (!(demux->speed_pkts_cnt % SPEED_PKTS_INTERVAL)) { - cur_time = current_kernel_time(); + cur_time = ktime_get(); - if (demux->speed_last_time.tv_sec != 0 && - demux->speed_last_time.tv_nsec != 0) { - delta_time = timespec_sub(cur_time, - demux->speed_last_time); + if (ktime_to_ns(demux->speed_last_time) != 0) { speed_bytes = (u64)demux->speed_pkts_cnt * 188 * 8; /* convert to 1024 basis */ speed_bytes = 1000 * div64_u64(speed_bytes, 1024); - speed_timedelta = - (u64)timespec_to_ns(&delta_time); - speed_timedelta = div64_u64(speed_timedelta, - 1000000); /* nsec -> usec */ + speed_timedelta = ktime_ms_delta(cur_time, + demux->speed_last_time); printk(KERN_INFO "TS speed %llu Kbits/sec \n", div64_u64(speed_bytes, speed_timedelta)); @@ -666,7 +661,7 @@ out: static int dmx_ts_feed_set(struct dmx_ts_feed *ts_feed, u16 pid, int ts_type, enum dmx_ts_pes pes_type, - size_t circular_buffer_size, struct timespec timeout) + size_t circular_buffer_size, ktime_t timeout) { struct dvb_demux_feed *feed = (struct dvb_demux_feed *)ts_feed; struct dvb_demux *demux = feed->demux; diff --git a/drivers/media/dvb-core/dvb_demux.h b/drivers/media/dvb-core/dvb_demux.h index ae7fc33c3231..5ed3cab4ad28 100644 --- a/drivers/media/dvb-core/dvb_demux.h +++ b/drivers/media/dvb-core/dvb_demux.h @@ -83,7 +83,7 @@ struct dvb_demux_feed { u8 *buffer; int buffer_size; - struct timespec timeout; + ktime_t timeout; struct dvb_demux_filter *filter; int ts_type; @@ -134,7 +134,7 @@ struct dvb_demux { uint8_t *cnt_storage; /* for TS continuity check */ - struct timespec speed_last_time; /* for TS speed check */ + ktime_t speed_last_time; /* for TS speed check */ uint32_t speed_pkts_cnt; /* for TS speed check */ }; diff --git a/drivers/media/dvb-core/dvb_frontend.c b/drivers/media/dvb-core/dvb_frontend.c index c0142614c408..be99c8dbc5f8 100644 --- a/drivers/media/dvb-core/dvb_frontend.c +++ b/drivers/media/dvb-core/dvb_frontend.c @@ -99,6 +99,7 @@ MODULE_PARM_DESC(dvb_mfe_wait_time, "Wait up to seconds on open( static DEFINE_MUTEX(frontend_mutex); struct dvb_frontend_private { + struct kref refcount; /* thread/frontend values */ struct dvb_device *dvbdev; @@ -137,6 +138,23 @@ struct dvb_frontend_private { #endif }; +static void dvb_frontend_private_free(struct kref *ref) +{ + struct dvb_frontend_private *fepriv = + container_of(ref, struct dvb_frontend_private, refcount); + kfree(fepriv); +} + +static void dvb_frontend_private_put(struct dvb_frontend_private *fepriv) +{ + kref_put(&fepriv->refcount, dvb_frontend_private_free); +} + +static void dvb_frontend_private_get(struct dvb_frontend_private *fepriv) +{ + kref_get(&fepriv->refcount); +} + static void dvb_frontend_wakeup(struct dvb_frontend *fe); static int dtv_get_frontend(struct dvb_frontend *fe, struct dtv_frontend_properties *c, @@ -2543,6 +2561,8 @@ static int dvb_frontend_open(struct inode *inode, struct file *file) fepriv->events.eventr = fepriv->events.eventw = 0; } + dvb_frontend_private_get(fepriv); + if (adapter->mfe_shared) mutex_unlock (&adapter->mfe_lock); return ret; @@ -2591,6 +2611,8 @@ static int dvb_frontend_release(struct inode *inode, struct file *file) fe->ops.ts_bus_ctrl(fe, 0); } + dvb_frontend_private_put(fepriv); + return ret; } @@ -2679,6 +2701,8 @@ int dvb_register_frontend(struct dvb_adapter* dvb, } fepriv = fe->frontend_priv; + kref_init(&fepriv->refcount); + sema_init(&fepriv->sem, 1); init_waitqueue_head (&fepriv->wait_queue); init_waitqueue_head (&fepriv->events.wait_queue); @@ -2713,18 +2737,11 @@ int dvb_unregister_frontend(struct dvb_frontend* fe) mutex_lock(&frontend_mutex); dvb_frontend_stop (fe); - mutex_unlock(&frontend_mutex); - - if (fepriv->dvbdev->users < -1) - wait_event(fepriv->dvbdev->wait_queue, - fepriv->dvbdev->users==-1); - - mutex_lock(&frontend_mutex); dvb_unregister_device (fepriv->dvbdev); /* fe is invalid now */ - kfree(fepriv); mutex_unlock(&frontend_mutex); + dvb_frontend_private_put(fepriv); return 0; } EXPORT_SYMBOL(dvb_unregister_frontend); diff --git a/drivers/media/dvb-core/dvb_net.c b/drivers/media/dvb-core/dvb_net.c index ce6a711b42d4..9914f69a4a02 100644 --- a/drivers/media/dvb-core/dvb_net.c +++ b/drivers/media/dvb-core/dvb_net.c @@ -997,7 +997,7 @@ static int dvb_net_feed_start(struct net_device *dev) netdev_dbg(dev, "start filtering\n"); priv->secfeed->start_filtering(priv->secfeed); } else if (priv->feedtype == DVB_NET_FEEDTYPE_ULE) { - struct timespec timeout = { 0, 10000000 }; // 10 msec + ktime_t timeout = ns_to_ktime(10 * NSEC_PER_MSEC); /* we have payloads encapsulated in TS */ netdev_dbg(dev, "alloc tsfeed\n"); diff --git a/drivers/media/dvb-core/dvb_ringbuffer.c b/drivers/media/dvb-core/dvb_ringbuffer.c index 1100e98a7b1d..7df7fb3738a0 100644 --- a/drivers/media/dvb-core/dvb_ringbuffer.c +++ b/drivers/media/dvb-core/dvb_ringbuffer.c @@ -55,7 +55,13 @@ void dvb_ringbuffer_init(struct dvb_ringbuffer *rbuf, void *data, size_t len) int dvb_ringbuffer_empty(struct dvb_ringbuffer *rbuf) { - return (rbuf->pread==rbuf->pwrite); + /* smp_load_acquire() to load write pointer on reader side + * this pairs with smp_store_release() in dvb_ringbuffer_write(), + * dvb_ringbuffer_write_user(), or dvb_ringbuffer_reset() + * + * for memory barriers also see Documentation/circular-buffers.txt + */ + return (rbuf->pread == smp_load_acquire(&rbuf->pwrite)); } @@ -64,7 +70,12 @@ ssize_t dvb_ringbuffer_free(struct dvb_ringbuffer *rbuf) { ssize_t free; - free = rbuf->pread - rbuf->pwrite; + /* ACCESS_ONCE() to load read pointer on writer side + * this pairs with smp_store_release() in dvb_ringbuffer_read(), + * dvb_ringbuffer_read_user(), dvb_ringbuffer_flush(), + * or dvb_ringbuffer_reset() + */ + free = ACCESS_ONCE(rbuf->pread) - rbuf->pwrite; if (free <= 0) free += rbuf->size; return free-1; @@ -76,7 +87,11 @@ ssize_t dvb_ringbuffer_avail(struct dvb_ringbuffer *rbuf) { ssize_t avail; - avail = rbuf->pwrite - rbuf->pread; + /* smp_load_acquire() to load write pointer on reader side + * this pairs with smp_store_release() in dvb_ringbuffer_write(), + * dvb_ringbuffer_write_user(), or dvb_ringbuffer_reset() + */ + avail = smp_load_acquire(&rbuf->pwrite) - rbuf->pread; if (avail < 0) avail += rbuf->size; return avail; @@ -86,14 +101,25 @@ ssize_t dvb_ringbuffer_avail(struct dvb_ringbuffer *rbuf) void dvb_ringbuffer_flush(struct dvb_ringbuffer *rbuf) { - rbuf->pread = rbuf->pwrite; + /* dvb_ringbuffer_flush() counts as read operation + * smp_load_acquire() to load write pointer + * smp_store_release() to update read pointer, this ensures that the + * correct pointer is visible for subsequent dvb_ringbuffer_free() + * calls on other cpu cores + */ + smp_store_release(&rbuf->pread, smp_load_acquire(&rbuf->pwrite)); rbuf->error = 0; } EXPORT_SYMBOL(dvb_ringbuffer_flush); void dvb_ringbuffer_reset(struct dvb_ringbuffer *rbuf) { - rbuf->pread = rbuf->pwrite = 0; + /* dvb_ringbuffer_reset() counts as read and write operation + * smp_store_release() to update read pointer + */ + smp_store_release(&rbuf->pread, 0); + /* smp_store_release() to update write pointer */ + smp_store_release(&rbuf->pwrite, 0); rbuf->error = 0; } @@ -119,12 +145,17 @@ ssize_t dvb_ringbuffer_read_user(struct dvb_ringbuffer *rbuf, u8 __user *buf, si return -EFAULT; buf += split; todo -= split; - rbuf->pread = 0; + /* smp_store_release() for read pointer update to ensure + * that buf is not overwritten until read is complete, + * this pairs with ACCESS_ONCE() in dvb_ringbuffer_free() + */ + smp_store_release(&rbuf->pread, 0); } if (copy_to_user(buf, rbuf->data+rbuf->pread, todo)) return -EFAULT; - rbuf->pread = (rbuf->pread + todo) % rbuf->size; + /* smp_store_release() to update read pointer, see above */ + smp_store_release(&rbuf->pread, (rbuf->pread + todo) % rbuf->size); return len; } @@ -139,11 +170,16 @@ void dvb_ringbuffer_read(struct dvb_ringbuffer *rbuf, u8 *buf, size_t len) memcpy(buf, rbuf->data+rbuf->pread, split); buf += split; todo -= split; - rbuf->pread = 0; + /* smp_store_release() for read pointer update to ensure + * that buf is not overwritten until read is complete, + * this pairs with ACCESS_ONCE() in dvb_ringbuffer_free() + */ + smp_store_release(&rbuf->pread, 0); } memcpy(buf, rbuf->data+rbuf->pread, todo); - rbuf->pread = (rbuf->pread + todo) % rbuf->size; + /* smp_store_release() to update read pointer, see above */ + smp_store_release(&rbuf->pread, (rbuf->pread + todo) % rbuf->size); } @@ -158,10 +194,16 @@ ssize_t dvb_ringbuffer_write(struct dvb_ringbuffer *rbuf, const u8 *buf, size_t memcpy(rbuf->data+rbuf->pwrite, buf, split); buf += split; todo -= split; - rbuf->pwrite = 0; + /* smp_store_release() for write pointer update to ensure that + * written data is visible on other cpu cores before the pointer + * update, this pairs with smp_load_acquire() in + * dvb_ringbuffer_empty() or dvb_ringbuffer_avail() + */ + smp_store_release(&rbuf->pwrite, 0); } memcpy(rbuf->data+rbuf->pwrite, buf, todo); - rbuf->pwrite = (rbuf->pwrite + todo) % rbuf->size; + /* smp_store_release() for write pointer update, see above */ + smp_store_release(&rbuf->pwrite, (rbuf->pwrite + todo) % rbuf->size); return len; } @@ -181,12 +223,18 @@ ssize_t dvb_ringbuffer_write_user(struct dvb_ringbuffer *rbuf, return len - todo; buf += split; todo -= split; - rbuf->pwrite = 0; + /* smp_store_release() for write pointer update to ensure that + * written data is visible on other cpu cores before the pointer + * update, this pairs with smp_load_acquire() in + * dvb_ringbuffer_empty() or dvb_ringbuffer_avail() + */ + smp_store_release(&rbuf->pwrite, 0); } status = copy_from_user(rbuf->data+rbuf->pwrite, buf, todo); if (status) return len - todo; - rbuf->pwrite = (rbuf->pwrite + todo) % rbuf->size; + /* smp_store_release() for write pointer update, see above */ + smp_store_release(&rbuf->pwrite, (rbuf->pwrite + todo) % rbuf->size); return len; } diff --git a/drivers/media/dvb-frontends/Kconfig b/drivers/media/dvb-frontends/Kconfig index a82f77c49bd5..c645aa81f423 100644 --- a/drivers/media/dvb-frontends/Kconfig +++ b/drivers/media/dvb-frontends/Kconfig @@ -73,6 +73,14 @@ config DVB_SI2165 Say Y when you want to support this frontend. +config DVB_MN88472 + tristate "Panasonic MN88472" + depends on DVB_CORE && I2C + select REGMAP_I2C + default m if !MEDIA_SUBDRV_AUTOSELECT + help + Say Y when you want to support this frontend. + config DVB_MN88473 tristate "Panasonic MN88473" depends on DVB_CORE && I2C @@ -853,6 +861,13 @@ config DVB_ASCOT2E help Say Y when you want to support this frontend. +config DVB_HELENE + tristate "Sony HELENE Sat/Ter tuner (CXD2858ER)" + depends on DVB_CORE && I2C + default m if !MEDIA_SUBDRV_AUTOSELECT + help + Say Y when you want to support this frontend. + comment "Tools to develop new frontends" config DVB_DUMMY_FE diff --git a/drivers/media/dvb-frontends/Makefile b/drivers/media/dvb-frontends/Makefile index eb7191f4219d..e90165ad361b 100644 --- a/drivers/media/dvb-frontends/Makefile +++ b/drivers/media/dvb-frontends/Makefile @@ -95,6 +95,7 @@ obj-$(CONFIG_DVB_STV0900) += stv0900.o obj-$(CONFIG_DVB_STV090x) += stv090x.o obj-$(CONFIG_DVB_STV6110x) += stv6110x.o obj-$(CONFIG_DVB_M88DS3103) += m88ds3103.o +obj-$(CONFIG_DVB_MN88472) += mn88472.o obj-$(CONFIG_DVB_MN88473) += mn88473.o obj-$(CONFIG_DVB_ISL6423) += isl6423.o obj-$(CONFIG_DVB_EC100) += ec100.o @@ -123,3 +124,4 @@ obj-$(CONFIG_DVB_AS102_FE) += as102_fe.o obj-$(CONFIG_DVB_TC90522) += tc90522.o obj-$(CONFIG_DVB_HORUS3A) += horus3a.o obj-$(CONFIG_DVB_ASCOT2E) += ascot2e.o +obj-$(CONFIG_DVB_HELENE) += helene.o diff --git a/drivers/media/dvb-frontends/af9033.c b/drivers/media/dvb-frontends/af9033.c index efebe5ce2429..9a8157a5f49d 100644 --- a/drivers/media/dvb-frontends/af9033.c +++ b/drivers/media/dvb-frontends/af9033.c @@ -41,7 +41,6 @@ struct af9033_dev { u64 post_bit_count; u64 error_block_count; u64 total_block_count; - struct delayed_work stat_work; }; /* write multiple registers */ @@ -468,8 +467,6 @@ static int af9033_init(struct dvb_frontend *fe) c->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; c->post_bit_error.len = 1; c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - /* start statistics polling */ - schedule_delayed_work(&dev->stat_work, msecs_to_jiffies(2000)); return 0; @@ -485,9 +482,6 @@ static int af9033_sleep(struct dvb_frontend *fe) int ret, i; u8 tmp; - /* stop statistics polling */ - cancel_delayed_work_sync(&dev->stat_work); - ret = af9033_wr_reg(dev, 0x80004c, 1); if (ret < 0) goto err; @@ -821,36 +815,39 @@ err: static int af9033_read_status(struct dvb_frontend *fe, enum fe_status *status) { struct af9033_dev *dev = fe->demodulator_priv; - int ret; - u8 tmp; + struct dtv_frontend_properties *c = &fe->dtv_property_cache; + int ret, i, tmp = 0; + u8 u8tmp, buf[7]; + + dev_dbg(&dev->client->dev, "\n"); *status = 0; /* radio channel status, 0=no result, 1=has signal, 2=no signal */ - ret = af9033_rd_reg(dev, 0x800047, &tmp); + ret = af9033_rd_reg(dev, 0x800047, &u8tmp); if (ret < 0) goto err; /* has signal */ - if (tmp == 0x01) + if (u8tmp == 0x01) *status |= FE_HAS_SIGNAL; - if (tmp != 0x02) { + if (u8tmp != 0x02) { /* TPS lock */ - ret = af9033_rd_reg_mask(dev, 0x80f5a9, &tmp, 0x01); + ret = af9033_rd_reg_mask(dev, 0x80f5a9, &u8tmp, 0x01); if (ret < 0) goto err; - if (tmp) + if (u8tmp) *status |= FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_VITERBI; /* full lock */ - ret = af9033_rd_reg_mask(dev, 0x80f999, &tmp, 0x01); + ret = af9033_rd_reg_mask(dev, 0x80f999, &u8tmp, 0x01); if (ret < 0) goto err; - if (tmp) + if (u8tmp) *status |= FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK; @@ -858,6 +855,148 @@ static int af9033_read_status(struct dvb_frontend *fe, enum fe_status *status) dev->fe_status = *status; + /* signal strength */ + if (dev->fe_status & FE_HAS_SIGNAL) { + if (dev->is_af9035) { + ret = af9033_rd_reg(dev, 0x80004a, &u8tmp); + if (ret) + goto err; + tmp = -u8tmp * 1000; + } else { + ret = af9033_rd_reg(dev, 0x8000f7, &u8tmp); + if (ret) + goto err; + tmp = (u8tmp - 100) * 1000; + } + + c->strength.len = 1; + c->strength.stat[0].scale = FE_SCALE_DECIBEL; + c->strength.stat[0].svalue = tmp; + } else { + c->strength.len = 1; + c->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + } + + /* CNR */ + if (dev->fe_status & FE_HAS_VITERBI) { + u32 snr_val, snr_lut_size; + const struct val_snr *snr_lut = NULL; + + /* read value */ + ret = af9033_rd_regs(dev, 0x80002c, buf, 3); + if (ret) + goto err; + + snr_val = (buf[2] << 16) | (buf[1] << 8) | (buf[0] << 0); + + /* read superframe number */ + ret = af9033_rd_reg(dev, 0x80f78b, &u8tmp); + if (ret) + goto err; + + if (u8tmp) + snr_val /= u8tmp; + + /* read current transmission mode */ + ret = af9033_rd_reg(dev, 0x80f900, &u8tmp); + if (ret) + goto err; + + switch ((u8tmp >> 0) & 3) { + case 0: + snr_val *= 4; + break; + case 1: + snr_val *= 1; + break; + case 2: + snr_val *= 2; + break; + default: + snr_val *= 0; + break; + } + + /* read current modulation */ + ret = af9033_rd_reg(dev, 0x80f903, &u8tmp); + if (ret) + goto err; + + switch ((u8tmp >> 0) & 3) { + case 0: + snr_lut_size = ARRAY_SIZE(qpsk_snr_lut); + snr_lut = qpsk_snr_lut; + break; + case 1: + snr_lut_size = ARRAY_SIZE(qam16_snr_lut); + snr_lut = qam16_snr_lut; + break; + case 2: + snr_lut_size = ARRAY_SIZE(qam64_snr_lut); + snr_lut = qam64_snr_lut; + break; + default: + snr_lut_size = 0; + tmp = 0; + break; + } + + for (i = 0; i < snr_lut_size; i++) { + tmp = snr_lut[i].snr * 1000; + if (snr_val < snr_lut[i].val) + break; + } + + c->cnr.len = 1; + c->cnr.stat[0].scale = FE_SCALE_DECIBEL; + c->cnr.stat[0].svalue = tmp; + } else { + c->cnr.len = 1; + c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + } + + /* UCB/PER/BER */ + if (dev->fe_status & FE_HAS_LOCK) { + /* outer FEC, 204 byte packets */ + u16 abort_packet_count, rsd_packet_count; + /* inner FEC, bits */ + u32 rsd_bit_err_count; + + /* + * Packet count used for measurement is 10000 + * (rsd_packet_count). Maybe it should be increased? + */ + + ret = af9033_rd_regs(dev, 0x800032, buf, 7); + if (ret) + goto err; + + abort_packet_count = (buf[1] << 8) | (buf[0] << 0); + rsd_bit_err_count = (buf[4] << 16) | (buf[3] << 8) | buf[2]; + rsd_packet_count = (buf[6] << 8) | (buf[5] << 0); + + dev->error_block_count += abort_packet_count; + dev->total_block_count += rsd_packet_count; + dev->post_bit_error += rsd_bit_err_count; + dev->post_bit_count += rsd_packet_count * 204 * 8; + + c->block_count.len = 1; + c->block_count.stat[0].scale = FE_SCALE_COUNTER; + c->block_count.stat[0].uvalue = dev->total_block_count; + + c->block_error.len = 1; + c->block_error.stat[0].scale = FE_SCALE_COUNTER; + c->block_error.stat[0].uvalue = dev->error_block_count; + + c->post_bit_count.len = 1; + c->post_bit_count.stat[0].scale = FE_SCALE_COUNTER; + c->post_bit_count.stat[0].uvalue = dev->post_bit_count; + + c->post_bit_error.len = 1; + c->post_bit_error.stat[0].scale = FE_SCALE_COUNTER; + c->post_bit_error.stat[0].uvalue = dev->post_bit_error; + } + return 0; err: @@ -1059,159 +1198,6 @@ err: return ret; } -static void af9033_stat_work(struct work_struct *work) -{ - struct af9033_dev *dev = container_of(work, struct af9033_dev, stat_work.work); - struct dtv_frontend_properties *c = &dev->fe.dtv_property_cache; - int ret, tmp, i, len; - u8 u8tmp, buf[7]; - - dev_dbg(&dev->client->dev, "\n"); - - /* signal strength */ - if (dev->fe_status & FE_HAS_SIGNAL) { - if (dev->is_af9035) { - ret = af9033_rd_reg(dev, 0x80004a, &u8tmp); - tmp = -u8tmp * 1000; - } else { - ret = af9033_rd_reg(dev, 0x8000f7, &u8tmp); - tmp = (u8tmp - 100) * 1000; - } - if (ret) - goto err; - - c->strength.len = 1; - c->strength.stat[0].scale = FE_SCALE_DECIBEL; - c->strength.stat[0].svalue = tmp; - } else { - c->strength.len = 1; - c->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - } - - /* CNR */ - if (dev->fe_status & FE_HAS_VITERBI) { - u32 snr_val; - const struct val_snr *snr_lut; - - /* read value */ - ret = af9033_rd_regs(dev, 0x80002c, buf, 3); - if (ret) - goto err; - - snr_val = (buf[2] << 16) | (buf[1] << 8) | (buf[0] << 0); - - /* read superframe number */ - ret = af9033_rd_reg(dev, 0x80f78b, &u8tmp); - if (ret) - goto err; - - if (u8tmp) - snr_val /= u8tmp; - - /* read current transmission mode */ - ret = af9033_rd_reg(dev, 0x80f900, &u8tmp); - if (ret) - goto err; - - switch ((u8tmp >> 0) & 3) { - case 0: - snr_val *= 4; - break; - case 1: - snr_val *= 1; - break; - case 2: - snr_val *= 2; - break; - default: - goto err_schedule_delayed_work; - } - - /* read current modulation */ - ret = af9033_rd_reg(dev, 0x80f903, &u8tmp); - if (ret) - goto err; - - switch ((u8tmp >> 0) & 3) { - case 0: - len = ARRAY_SIZE(qpsk_snr_lut); - snr_lut = qpsk_snr_lut; - break; - case 1: - len = ARRAY_SIZE(qam16_snr_lut); - snr_lut = qam16_snr_lut; - break; - case 2: - len = ARRAY_SIZE(qam64_snr_lut); - snr_lut = qam64_snr_lut; - break; - default: - goto err_schedule_delayed_work; - } - - for (i = 0; i < len; i++) { - tmp = snr_lut[i].snr * 1000; - if (snr_val < snr_lut[i].val) - break; - } - - c->cnr.len = 1; - c->cnr.stat[0].scale = FE_SCALE_DECIBEL; - c->cnr.stat[0].svalue = tmp; - } else { - c->cnr.len = 1; - c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - } - - /* UCB/PER/BER */ - if (dev->fe_status & FE_HAS_LOCK) { - /* outer FEC, 204 byte packets */ - u16 abort_packet_count, rsd_packet_count; - /* inner FEC, bits */ - u32 rsd_bit_err_count; - - /* - * Packet count used for measurement is 10000 - * (rsd_packet_count). Maybe it should be increased? - */ - - ret = af9033_rd_regs(dev, 0x800032, buf, 7); - if (ret) - goto err; - - abort_packet_count = (buf[1] << 8) | (buf[0] << 0); - rsd_bit_err_count = (buf[4] << 16) | (buf[3] << 8) | buf[2]; - rsd_packet_count = (buf[6] << 8) | (buf[5] << 0); - - dev->error_block_count += abort_packet_count; - dev->total_block_count += rsd_packet_count; - dev->post_bit_error += rsd_bit_err_count; - dev->post_bit_count += rsd_packet_count * 204 * 8; - - c->block_count.len = 1; - c->block_count.stat[0].scale = FE_SCALE_COUNTER; - c->block_count.stat[0].uvalue = dev->total_block_count; - - c->block_error.len = 1; - c->block_error.stat[0].scale = FE_SCALE_COUNTER; - c->block_error.stat[0].uvalue = dev->error_block_count; - - c->post_bit_count.len = 1; - c->post_bit_count.stat[0].scale = FE_SCALE_COUNTER; - c->post_bit_count.stat[0].uvalue = dev->post_bit_count; - - c->post_bit_error.len = 1; - c->post_bit_error.stat[0].scale = FE_SCALE_COUNTER; - c->post_bit_error.stat[0].uvalue = dev->post_bit_error; - } - -err_schedule_delayed_work: - schedule_delayed_work(&dev->stat_work, msecs_to_jiffies(2000)); - return; -err: - dev_dbg(&dev->client->dev, "failed=%d\n", ret); -} - static struct dvb_frontend_ops af9033_ops = { .delsys = { SYS_DVBT }, .info = { @@ -1272,7 +1258,6 @@ static int af9033_probe(struct i2c_client *client, /* setup the state */ dev->client = client; - INIT_DELAYED_WORK(&dev->stat_work, af9033_stat_work); memcpy(&dev->cfg, cfg, sizeof(struct af9033_config)); if (dev->cfg.clock != 12000000) { @@ -1372,9 +1357,6 @@ static int af9033_remove(struct i2c_client *client) dev_dbg(&dev->client->dev, "\n"); - /* stop statistics polling */ - cancel_delayed_work_sync(&dev->stat_work); - dev->fe.ops.release = NULL; dev->fe.demodulator_priv = NULL; kfree(dev); @@ -1391,6 +1373,7 @@ MODULE_DEVICE_TABLE(i2c, af9033_id_table); static struct i2c_driver af9033_driver = { .driver = { .name = "af9033", + .suppress_bind_attrs = true, }, .probe = af9033_probe, .remove = af9033_remove, diff --git a/drivers/media/dvb-frontends/ascot2e.c b/drivers/media/dvb-frontends/ascot2e.c index f770f6a2c987..8cc8c4597b6a 100644 --- a/drivers/media/dvb-frontends/ascot2e.c +++ b/drivers/media/dvb-frontends/ascot2e.c @@ -132,7 +132,7 @@ static int ascot2e_write_regs(struct ascot2e_priv *priv, } }; - if (len + 1 >= sizeof(buf)) { + if (len + 1 > sizeof(buf)) { dev_warn(&priv->i2c->dev,"wr reg=%04x: len=%d is too big!\n", reg, len + 1); return -E2BIG; diff --git a/drivers/media/dvb-frontends/cxd2841er.c b/drivers/media/dvb-frontends/cxd2841er.c index 900186ba8e62..09c39346167f 100644 --- a/drivers/media/dvb-frontends/cxd2841er.c +++ b/drivers/media/dvb-frontends/cxd2841er.c @@ -1,7 +1,9 @@ /* * cxd2841er.c * - * Sony CXD2441ER digital demodulator driver + * Sony digital demodulator driver for + * CXD2841ER - DVB-S/S2/T/T2/C/C2 + * CXD2854ER - DVB-S/S2/T/T2/C/C2, ISDB-T/S * * Copyright 2012 Sony Corporation * Copyright (C) 2014 NetUP Inc. @@ -34,6 +36,16 @@ #include "cxd2841er_priv.h" #define MAX_WRITE_REGSIZE 16 +#define LOG2_E_100X 144 + +/* DVB-C constellation */ +enum sony_dvbc_constellation_t { + SONY_DVBC_CONSTELLATION_16QAM, + SONY_DVBC_CONSTELLATION_32QAM, + SONY_DVBC_CONSTELLATION_64QAM, + SONY_DVBC_CONSTELLATION_128QAM, + SONY_DVBC_CONSTELLATION_256QAM +}; enum cxd2841er_state { STATE_SHUTDOWN = 0, @@ -51,6 +63,8 @@ struct cxd2841er_priv { const struct cxd2841er_config *config; enum cxd2841er_state state; u8 system; + enum cxd2841er_xtal xtal; + enum fe_caps caps; }; static const struct cxd2841er_cnr_data s_cn_data[] = { @@ -188,6 +202,9 @@ static const struct cxd2841er_cnr_data s2_cn_data[] = { }; #define MAKE_IFFREQ_CONFIG(iffreq) ((u32)(((iffreq)/41.0)*16777216.0 + 0.5)) +#define MAKE_IFFREQ_CONFIG_XTAL(xtal, iffreq) ((xtal == SONY_XTAL_24000) ? \ + (u32)(((iffreq)/48.0)*16777216.0 + 0.5) : \ + (u32)(((iffreq)/41.0)*16777216.0 + 0.5)) static void cxd2841er_i2c_debug(struct cxd2841er_priv *priv, u8 addr, u8 reg, u8 write, @@ -217,7 +234,7 @@ static int cxd2841er_write_regs(struct cxd2841er_priv *priv, }; if (len + 1 >= sizeof(buf)) { - dev_warn(&priv->i2c->dev,"wr reg=%04x: len=%d is too big!\n", + dev_warn(&priv->i2c->dev, "wr reg=%04x: len=%d is too big!\n", reg, len + 1); return -E2BIG; } @@ -282,6 +299,7 @@ static int cxd2841er_read_regs(struct cxd2841er_priv *priv, KBUILD_MODNAME, ret, i2c_addr, reg); return ret; } + cxd2841er_i2c_debug(priv, i2c_addr, reg, 0, val, len); return 0; } @@ -427,6 +445,15 @@ static int cxd2841er_sleep_tc_to_active_t2_band(struct cxd2841er_priv *priv, static int cxd2841er_sleep_tc_to_active_c_band(struct cxd2841er_priv *priv, u32 bandwidth); +static int cxd2841er_sleep_tc_to_active_i(struct cxd2841er_priv *priv, + u32 bandwidth); + +static int cxd2841er_active_i_to_sleep_tc(struct cxd2841er_priv *priv); + +static int cxd2841er_sleep_tc_to_shutdown(struct cxd2841er_priv *priv); + +static int cxd2841er_shutdown_to_sleep_tc(struct cxd2841er_priv *priv); + static int cxd2841er_retune_active(struct cxd2841er_priv *priv, struct dtv_frontend_properties *p) { @@ -454,7 +481,13 @@ static int cxd2841er_retune_active(struct cxd2841er_priv *priv, priv, p->bandwidth_hz); case SYS_DVBC_ANNEX_A: return cxd2841er_sleep_tc_to_active_c_band( - priv, 8000000); + priv, p->bandwidth_hz); + case SYS_ISDBT: + cxd2841er_active_i_to_sleep_tc(priv); + cxd2841er_sleep_tc_to_shutdown(priv); + cxd2841er_shutdown_to_sleep_tc(priv); + return cxd2841er_sleep_tc_to_active_i( + priv, p->bandwidth_hz); } } dev_dbg(&priv->i2c->dev, "%s(): invalid delivery system %d\n", @@ -669,6 +702,45 @@ static int cxd2841er_active_c_to_sleep_tc(struct cxd2841er_priv *priv) return 0; } +static int cxd2841er_active_i_to_sleep_tc(struct cxd2841er_priv *priv) +{ + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + if (priv->state != STATE_ACTIVE_TC) { + dev_err(&priv->i2c->dev, "%s(): invalid state %d\n", + __func__, priv->state); + return -EINVAL; + } + /* Set SLV-T Bank : 0x00 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); + /* disable TS output */ + cxd2841er_write_reg(priv, I2C_SLVT, 0xc3, 0x01); + /* enable Hi-Z setting 1 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x80, 0x3f); + /* enable Hi-Z setting 2 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x81, 0xff); + + /* TODO: Cancel demod parameter */ + + /* Set SLV-X Bank : 0x00 */ + cxd2841er_write_reg(priv, I2C_SLVX, 0x00, 0x00); + /* disable ADC 1 */ + cxd2841er_write_reg(priv, I2C_SLVX, 0x18, 0x01); + /* Set SLV-T Bank : 0x00 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); + /* Disable ADC 2 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x43, 0x0a); + /* Disable ADC 3 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x41, 0x0a); + /* Disable ADC clock */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x30, 0x00); + /* Disable RF level monitor */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x2f, 0x00); + /* Disable demod clock */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x2c, 0x00); + priv->state = STATE_SLEEP_TC; + return 0; +} + static int cxd2841er_shutdown_to_sleep_s(struct cxd2841er_priv *priv) { dev_dbg(&priv->i2c->dev, "%s()\n", __func__); @@ -686,8 +758,25 @@ static int cxd2841er_shutdown_to_sleep_s(struct cxd2841er_priv *priv) cxd2841er_write_reg(priv, I2C_SLVX, 0x00, 0x00); /* Set demod SW reset */ cxd2841er_write_reg(priv, I2C_SLVX, 0x10, 0x01); - /* Set X'tal clock to 20.5Mhz */ - cxd2841er_write_reg(priv, I2C_SLVX, 0x14, 0x00); + + switch (priv->xtal) { + case SONY_XTAL_20500: + cxd2841er_write_reg(priv, I2C_SLVX, 0x14, 0x00); + break; + case SONY_XTAL_24000: + /* Select demod frequency */ + cxd2841er_write_reg(priv, I2C_SLVX, 0x12, 0x00); + cxd2841er_write_reg(priv, I2C_SLVX, 0x14, 0x03); + break; + case SONY_XTAL_41000: + cxd2841er_write_reg(priv, I2C_SLVX, 0x14, 0x01); + break; + default: + dev_dbg(&priv->i2c->dev, "%s(): invalid demod xtal %d\n", + __func__, priv->xtal); + return -EINVAL; + } + /* Set demod mode */ cxd2841er_write_reg(priv, I2C_SLVX, 0x17, 0x0a); /* Clear demod SW reset */ @@ -712,6 +801,8 @@ static int cxd2841er_shutdown_to_sleep_s(struct cxd2841er_priv *priv) static int cxd2841er_shutdown_to_sleep_tc(struct cxd2841er_priv *priv) { + u8 data = 0; + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); if (priv->state != STATE_SHUTDOWN) { dev_dbg(&priv->i2c->dev, "%s(): invalid demod state %d\n", @@ -727,9 +818,24 @@ static int cxd2841er_shutdown_to_sleep_tc(struct cxd2841er_priv *priv) cxd2841er_write_reg(priv, I2C_SLVX, 0x00, 0x00); /* Set demod SW reset */ cxd2841er_write_reg(priv, I2C_SLVX, 0x10, 0x01); - /* Set X'tal clock to 20.5Mhz */ + /* Select ADC clock mode */ cxd2841er_write_reg(priv, I2C_SLVX, 0x13, 0x00); - cxd2841er_write_reg(priv, I2C_SLVX, 0x14, 0x00); + + switch (priv->xtal) { + case SONY_XTAL_20500: + data = 0x0; + break; + case SONY_XTAL_24000: + /* Select demod frequency */ + cxd2841er_write_reg(priv, I2C_SLVX, 0x12, 0x00); + data = 0x3; + break; + case SONY_XTAL_41000: + cxd2841er_write_reg(priv, I2C_SLVX, 0x12, 0x00); + data = 0x1; + break; + } + cxd2841er_write_reg(priv, I2C_SLVX, 0x14, data); /* Clear demod SW reset */ cxd2841er_write_reg(priv, I2C_SLVX, 0x10, 0x00); usleep_range(1000, 2000); @@ -809,11 +915,14 @@ static void cxd2841er_set_ts_clock_mode(struct cxd2841er_priv *priv, static u8 cxd2841er_chip_id(struct cxd2841er_priv *priv) { - u8 chip_id; + u8 chip_id = 0; dev_dbg(&priv->i2c->dev, "%s()\n", __func__); - cxd2841er_write_reg(priv, I2C_SLVT, 0, 0); - cxd2841er_read_reg(priv, I2C_SLVT, 0xfd, &chip_id); + if (cxd2841er_write_reg(priv, I2C_SLVT, 0, 0) == 0) + cxd2841er_read_reg(priv, I2C_SLVT, 0xfd, &chip_id); + else if (cxd2841er_write_reg(priv, I2C_SLVX, 0, 0) == 0) + cxd2841er_read_reg(priv, I2C_SLVX, 0xfd, &chip_id); + return chip_id; } @@ -896,6 +1005,25 @@ static int cxd2841er_read_status_c(struct cxd2841er_priv *priv, u8 *tslock) return 0; } +static int cxd2841er_read_status_i(struct cxd2841er_priv *priv, + u8 *sync, u8 *tslock, u8 *unlock) +{ + u8 data = 0; + + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + if (priv->state != STATE_ACTIVE_TC) + return -EINVAL; + /* Set SLV-T Bank : 0x60 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x60); + cxd2841er_read_reg(priv, I2C_SLVT, 0x10, &data); + dev_dbg(&priv->i2c->dev, + "%s(): lock=0x%x\n", __func__, data); + *sync = ((data & 0x02) ? 1 : 0); + *tslock = ((data & 0x01) ? 1 : 0); + *unlock = ((data & 0x10) ? 1 : 0); + return 0; +} + static int cxd2841er_read_status_tc(struct dvb_frontend *fe, enum fe_status *status) { @@ -921,6 +1049,20 @@ static int cxd2841er_read_status_tc(struct dvb_frontend *fe, FE_HAS_SYNC; if (tslock) *status |= FE_HAS_LOCK; + } else if (priv->system == SYS_ISDBT) { + ret = cxd2841er_read_status_i( + priv, &sync, &tslock, &unlock); + if (ret) + goto done; + if (unlock) + goto done; + if (sync) + *status = FE_HAS_SIGNAL | + FE_HAS_CARRIER | + FE_HAS_VITERBI | + FE_HAS_SYNC; + if (tslock) + *status |= FE_HAS_LOCK; } else if (priv->system == SYS_DVBC_ANNEX_A) { ret = cxd2841er_read_status_c(priv, &tslock); if (ret) @@ -997,6 +1139,76 @@ static int cxd2841er_get_carrier_offset_s_s2(struct cxd2841er_priv *priv, return 0; } +static int cxd2841er_get_carrier_offset_i(struct cxd2841er_priv *priv, + u32 bandwidth, int *offset) +{ + u8 data[4]; + + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + if (priv->state != STATE_ACTIVE_TC) { + dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n", + __func__, priv->state); + return -EINVAL; + } + if (priv->system != SYS_ISDBT) { + dev_dbg(&priv->i2c->dev, "%s(): invalid delivery system %d\n", + __func__, priv->system); + return -EINVAL; + } + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x60); + cxd2841er_read_regs(priv, I2C_SLVT, 0x4c, data, sizeof(data)); + *offset = -1 * sign_extend32( + ((u32)(data[0] & 0x1F) << 24) | ((u32)data[1] << 16) | + ((u32)data[2] << 8) | (u32)data[3], 29); + + switch (bandwidth) { + case 6000000: + *offset = -1 * ((*offset) * 8/264); + break; + case 7000000: + *offset = -1 * ((*offset) * 8/231); + break; + case 8000000: + *offset = -1 * ((*offset) * 8/198); + break; + default: + dev_dbg(&priv->i2c->dev, "%s(): invalid bandwidth %d\n", + __func__, bandwidth); + return -EINVAL; + } + + dev_dbg(&priv->i2c->dev, "%s(): bandwidth %d offset %d\n", + __func__, bandwidth, *offset); + + return 0; +} + +static int cxd2841er_get_carrier_offset_t(struct cxd2841er_priv *priv, + u32 bandwidth, int *offset) +{ + u8 data[4]; + + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + if (priv->state != STATE_ACTIVE_TC) { + dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n", + __func__, priv->state); + return -EINVAL; + } + if (priv->system != SYS_DVBT) { + dev_dbg(&priv->i2c->dev, "%s(): invalid delivery system %d\n", + __func__, priv->system); + return -EINVAL; + } + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + cxd2841er_read_regs(priv, I2C_SLVT, 0x4c, data, sizeof(data)); + *offset = -1 * sign_extend32( + ((u32)(data[0] & 0x1F) << 24) | ((u32)data[1] << 16) | + ((u32)data[2] << 8) | (u32)data[3], 29); + *offset *= (bandwidth / 1000000); + *offset /= 235; + return 0; +} + static int cxd2841er_get_carrier_offset_t2(struct cxd2841er_priv *priv, u32 bandwidth, int *offset) { @@ -1060,6 +1272,24 @@ static int cxd2841er_get_carrier_offset_c(struct cxd2841er_priv *priv, return 0; } +static int cxd2841er_read_packet_errors_c( + struct cxd2841er_priv *priv, u32 *penum) +{ + u8 data[3]; + + *penum = 0; + if (priv->state != STATE_ACTIVE_TC) { + dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n", + __func__, priv->state); + return -EINVAL; + } + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x40); + cxd2841er_read_regs(priv, I2C_SLVT, 0xea, data, sizeof(data)); + if (data[2] & 0x01) + *penum = ((u32)data[0] << 8) | (u32)data[1]; + return 0; +} + static int cxd2841er_read_packet_errors_t( struct cxd2841er_priv *priv, u32 *penum) { @@ -1096,11 +1326,85 @@ static int cxd2841er_read_packet_errors_t2( return 0; } -static u32 cxd2841er_mon_read_ber_s(struct cxd2841er_priv *priv) +static int cxd2841er_read_packet_errors_i( + struct cxd2841er_priv *priv, u32 *penum) +{ + u8 data[2]; + + *penum = 0; + if (priv->state != STATE_ACTIVE_TC) { + dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n", + __func__, priv->state); + return -EINVAL; + } + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x60); + cxd2841er_read_regs(priv, I2C_SLVT, 0xA1, data, 1); + + if (!(data[0] & 0x01)) + return 0; + + /* Layer A */ + cxd2841er_read_regs(priv, I2C_SLVT, 0xA2, data, sizeof(data)); + *penum = ((u32)data[0] << 8) | (u32)data[1]; + + /* Layer B */ + cxd2841er_read_regs(priv, I2C_SLVT, 0xA4, data, sizeof(data)); + *penum += ((u32)data[0] << 8) | (u32)data[1]; + + /* Layer C */ + cxd2841er_read_regs(priv, I2C_SLVT, 0xA6, data, sizeof(data)); + *penum += ((u32)data[0] << 8) | (u32)data[1]; + + return 0; +} + +static int cxd2841er_read_ber_c(struct cxd2841er_priv *priv, + u32 *bit_error, u32 *bit_count) +{ + u8 data[3]; + u32 bit_err, period_exp; + + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + if (priv->state != STATE_ACTIVE_TC) { + dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n", + __func__, priv->state); + return -EINVAL; + } + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x40); + cxd2841er_read_regs(priv, I2C_SLVT, 0x62, data, sizeof(data)); + if (!(data[0] & 0x80)) { + dev_dbg(&priv->i2c->dev, + "%s(): no valid BER data\n", __func__); + return -EINVAL; + } + bit_err = ((u32)(data[0] & 0x3f) << 16) | + ((u32)data[1] << 8) | + (u32)data[2]; + cxd2841er_read_reg(priv, I2C_SLVT, 0x60, data); + period_exp = data[0] & 0x1f; + + if ((period_exp <= 11) && (bit_err > (1 << period_exp) * 204 * 8)) { + dev_dbg(&priv->i2c->dev, + "%s(): period_exp(%u) or bit_err(%u) not in range. no valid BER data\n", + __func__, period_exp, bit_err); + return -EINVAL; + } + + dev_dbg(&priv->i2c->dev, + "%s(): period_exp(%u) or bit_err(%u) count=%d\n", + __func__, period_exp, bit_err, + ((1 << period_exp) * 204 * 8)); + + *bit_error = bit_err; + *bit_count = ((1 << period_exp) * 204 * 8); + + return 0; +} + +static int cxd2841er_mon_read_ber_s(struct cxd2841er_priv *priv, + u32 *bit_error, u32 *bit_count) { u8 data[11]; - u32 bit_error, bit_count; - u32 temp_q, temp_r; /* Set SLV-T Bank : 0xA0 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0xa0); @@ -1116,40 +1420,30 @@ static u32 cxd2841er_mon_read_ber_s(struct cxd2841er_priv *priv) */ cxd2841er_read_regs(priv, I2C_SLVT, 0x35, data, 11); if (data[0] & 0x01) { - bit_error = ((u32)(data[1] & 0x3F) << 16) | - ((u32)(data[2] & 0xFF) << 8) | - (u32)(data[3] & 0xFF); - bit_count = ((u32)(data[8] & 0x3F) << 16) | - ((u32)(data[9] & 0xFF) << 8) | - (u32)(data[10] & 0xFF); - /* - * BER = bitError / bitCount - * = (bitError * 10^7) / bitCount - * = ((bitError * 625 * 125 * 128) / bitCount - */ - if ((bit_count == 0) || (bit_error > bit_count)) { + *bit_error = ((u32)(data[1] & 0x3F) << 16) | + ((u32)(data[2] & 0xFF) << 8) | + (u32)(data[3] & 0xFF); + *bit_count = ((u32)(data[8] & 0x3F) << 16) | + ((u32)(data[9] & 0xFF) << 8) | + (u32)(data[10] & 0xFF); + if ((*bit_count == 0) || (*bit_error > *bit_count)) { dev_dbg(&priv->i2c->dev, "%s(): invalid bit_error %d, bit_count %d\n", - __func__, bit_error, bit_count); - return 0; + __func__, *bit_error, *bit_count); + return -EINVAL; } - temp_q = div_u64_rem(10000000ULL * bit_error, - bit_count, &temp_r); - if (bit_count != 1 && temp_r >= bit_count / 2) - temp_q++; - return temp_q; + return 0; } dev_dbg(&priv->i2c->dev, "%s(): no data available\n", __func__); - return 0; + return -EINVAL; } -static u32 cxd2841er_mon_read_ber_s2(struct cxd2841er_priv *priv) +static int cxd2841er_mon_read_ber_s2(struct cxd2841er_priv *priv, + u32 *bit_error, u32 *bit_count) { u8 data[5]; - u32 bit_error, period; - u32 temp_q, temp_r; - u32 result = 0; + u32 period; /* Set SLV-T Bank : 0xB2 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0xb2); @@ -1164,10 +1458,10 @@ static u32 cxd2841er_mon_read_ber_s2(struct cxd2841er_priv *priv) cxd2841er_read_regs(priv, I2C_SLVT, 0x30, data, 5); if (data[0] & 0x01) { /* Bit error count */ - bit_error = ((u32)(data[1] & 0x0F) << 24) | - ((u32)(data[2] & 0xFF) << 16) | - ((u32)(data[3] & 0xFF) << 8) | - (u32)(data[4] & 0xFF); + *bit_error = ((u32)(data[1] & 0x0F) << 24) | + ((u32)(data[2] & 0xFF) << 16) | + ((u32)(data[3] & 0xFF) << 8) | + (u32)(data[4] & 0xFF); /* Set SLV-T Bank : 0xA0 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0xa0); @@ -1177,40 +1471,30 @@ static u32 cxd2841er_mon_read_ber_s2(struct cxd2841er_priv *priv) if (period == 0) { dev_dbg(&priv->i2c->dev, "%s(): period is 0\n", __func__); - return 0; + return -EINVAL; } - if (bit_error > (period * 64800)) { + if (*bit_error > (period * 64800)) { dev_dbg(&priv->i2c->dev, "%s(): invalid bit_err 0x%x period 0x%x\n", - __func__, bit_error, period); - return 0; + __func__, *bit_error, period); + return -EINVAL; } - /* - * BER = bitError / (period * 64800) - * = (bitError * 10^7) / (period * 64800) - * = (bitError * 10^5) / (period * 648) - * = (bitError * 12500) / (period * 81) - * = (bitError * 10) * 1250 / (period * 81) - */ - temp_q = div_u64_rem(12500ULL * bit_error, - period * 81, &temp_r); - if (temp_r >= period * 40) - temp_q++; - result = temp_q; + *bit_count = period * 64800; + + return 0; } else { dev_dbg(&priv->i2c->dev, "%s(): no data available\n", __func__); } - return result; + return -EINVAL; } -static int cxd2841er_read_ber_t2(struct cxd2841er_priv *priv, u32 *ber) +static int cxd2841er_read_ber_t2(struct cxd2841er_priv *priv, + u32 *bit_error, u32 *bit_count) { u8 data[4]; - u32 div, q, r; - u32 bit_err, period_exp, n_ldpc; + u32 period_exp, n_ldpc; - *ber = 0; if (priv->state != STATE_ACTIVE_TC) { dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n", __func__, priv->state); @@ -1221,40 +1505,44 @@ static int cxd2841er_read_ber_t2(struct cxd2841er_priv *priv, u32 *ber) if (!(data[0] & 0x10)) { dev_dbg(&priv->i2c->dev, "%s(): no valid BER data\n", __func__); - return 0; + return -EINVAL; } - bit_err = ((u32)(data[0] & 0x0f) << 24) | - ((u32)data[1] << 16) | - ((u32)data[2] << 8) | - (u32)data[3]; + *bit_error = ((u32)(data[0] & 0x0f) << 24) | + ((u32)data[1] << 16) | + ((u32)data[2] << 8) | + (u32)data[3]; cxd2841er_read_reg(priv, I2C_SLVT, 0x6f, data); period_exp = data[0] & 0x0f; cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x22); cxd2841er_read_reg(priv, I2C_SLVT, 0x5e, data); n_ldpc = ((data[0] & 0x03) == 0 ? 16200 : 64800); - if (bit_err > ((1U << period_exp) * n_ldpc)) { + if (*bit_error > ((1U << period_exp) * n_ldpc)) { dev_dbg(&priv->i2c->dev, "%s(): invalid BER value\n", __func__); return -EINVAL; } + + /* + * FIXME: the right thing would be to return bit_error untouched, + * but, as we don't know the scale returned by the counters, let's + * at least preserver BER = bit_error/bit_count. + */ if (period_exp >= 4) { - div = (1U << (period_exp - 4)) * (n_ldpc / 200); - q = div_u64_rem(3125ULL * bit_err, div, &r); + *bit_count = (1U << (period_exp - 4)) * (n_ldpc / 200); + *bit_error *= 3125ULL; } else { - div = (1U << period_exp) * (n_ldpc / 200); - q = div_u64_rem(50000ULL * bit_err, div, &r); + *bit_count = (1U << period_exp) * (n_ldpc / 200); + *bit_error *= 50000ULL; } - *ber = (r >= div / 2) ? q + 1 : q; return 0; } -static int cxd2841er_read_ber_t(struct cxd2841er_priv *priv, u32 *ber) +static int cxd2841er_read_ber_t(struct cxd2841er_priv *priv, + u32 *bit_error, u32 *bit_count) { u8 data[2]; - u32 div, q, r; - u32 bit_err, period; + u32 period; - *ber = 0; if (priv->state != STATE_ACTIVE_TC) { dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n", __func__, priv->state); @@ -1268,16 +1556,22 @@ static int cxd2841er_read_ber_t(struct cxd2841er_priv *priv, u32 *ber) return 0; } cxd2841er_read_regs(priv, I2C_SLVT, 0x22, data, sizeof(data)); - bit_err = ((u32)data[0] << 8) | (u32)data[1]; + *bit_error = ((u32)data[0] << 8) | (u32)data[1]; cxd2841er_read_reg(priv, I2C_SLVT, 0x6f, data); period = ((data[0] & 0x07) == 0) ? 256 : (4096 << (data[0] & 0x07)); - div = period / 128; - q = div_u64_rem(78125ULL * bit_err, div, &r); - *ber = (r >= div / 2) ? q + 1 : q; + + /* + * FIXME: the right thing would be to return bit_error untouched, + * but, as we don't know the scale returned by the counters, let's + * at least preserver BER = bit_error/bit_count. + */ + *bit_count = period / 128; + *bit_error *= 78125ULL; return 0; } -static u32 cxd2841er_dvbs_read_snr(struct cxd2841er_priv *priv, u8 delsys) +static u32 cxd2841er_dvbs_read_snr(struct cxd2841er_priv *priv, + u8 delsys, u32 *snr) { u8 data[3]; u32 res = 0, value; @@ -1335,9 +1629,71 @@ static u32 cxd2841er_dvbs_read_snr(struct cxd2841er_priv *priv, u8 delsys) } else { dev_dbg(&priv->i2c->dev, "%s(): no data available\n", __func__); + return -EINVAL; } done: - return res; + *snr = res; + return 0; +} + +static uint32_t sony_log(uint32_t x) +{ + return (((10000>>8)*(intlog2(x)>>16) + LOG2_E_100X/2)/LOG2_E_100X); +} + +static int cxd2841er_read_snr_c(struct cxd2841er_priv *priv, u32 *snr) +{ + u32 reg; + u8 data[2]; + enum sony_dvbc_constellation_t qam = SONY_DVBC_CONSTELLATION_16QAM; + + *snr = 0; + if (priv->state != STATE_ACTIVE_TC) { + dev_dbg(&priv->i2c->dev, + "%s(): invalid state %d\n", + __func__, priv->state); + return -EINVAL; + } + + /* + * Freeze registers: ensure multiple separate register reads + * are from the same snapshot + */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x01, 0x01); + + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x40); + cxd2841er_read_regs(priv, I2C_SLVT, 0x19, data, 1); + qam = (enum sony_dvbc_constellation_t) (data[0] & 0x07); + cxd2841er_read_regs(priv, I2C_SLVT, 0x4C, data, 2); + + reg = ((u32)(data[0]&0x1f) << 8) | (u32)data[1]; + if (reg == 0) { + dev_dbg(&priv->i2c->dev, + "%s(): reg value out of range\n", __func__); + return 0; + } + + switch (qam) { + case SONY_DVBC_CONSTELLATION_16QAM: + case SONY_DVBC_CONSTELLATION_64QAM: + case SONY_DVBC_CONSTELLATION_256QAM: + /* SNR(dB) = -9.50 * ln(IREG_SNR_ESTIMATE / (24320)) */ + if (reg < 126) + reg = 126; + *snr = -95 * (int32_t)sony_log(reg) + 95941; + break; + case SONY_DVBC_CONSTELLATION_32QAM: + case SONY_DVBC_CONSTELLATION_128QAM: + /* SNR(dB) = -8.75 * ln(IREG_SNR_ESTIMATE / (20800)) */ + if (reg < 69) + reg = 69; + *snr = -88 * (int32_t)sony_log(reg) + 86999; + break; + default: + return -EINVAL; + } + + return 0; } static int cxd2841er_read_snr_t(struct cxd2841er_priv *priv, u32 *snr) @@ -1391,14 +1747,80 @@ static int cxd2841er_read_snr_t2(struct cxd2841er_priv *priv, u32 *snr) return 0; } -static u16 cxd2841er_read_agc_gain_t_t2(struct cxd2841er_priv *priv, - u8 delsys) +static int cxd2841er_read_snr_i(struct cxd2841er_priv *priv, u32 *snr) +{ + u32 reg; + u8 data[2]; + + *snr = 0; + if (priv->state != STATE_ACTIVE_TC) { + dev_dbg(&priv->i2c->dev, + "%s(): invalid state %d\n", __func__, + priv->state); + return -EINVAL; + } + + /* Freeze all registers */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x01, 0x01); + + + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x60); + cxd2841er_read_regs(priv, I2C_SLVT, 0x28, data, sizeof(data)); + reg = ((u32)data[0] << 8) | (u32)data[1]; + if (reg == 0) { + dev_dbg(&priv->i2c->dev, + "%s(): reg value out of range\n", __func__); + return 0; + } + if (reg > 4996) + reg = 4996; + *snr = 100 * intlog10(reg) - 9031; + return 0; +} + +static u16 cxd2841er_read_agc_gain_c(struct cxd2841er_priv *priv, + u8 delsys) +{ + u8 data[2]; + + cxd2841er_write_reg( + priv, I2C_SLVT, 0x00, 0x40); + cxd2841er_read_regs(priv, I2C_SLVT, 0x49, data, 2); + dev_dbg(&priv->i2c->dev, + "%s(): AGC value=%u\n", + __func__, (((u16)data[0] & 0x0F) << 8) | + (u16)(data[1] & 0xFF)); + return ((((u16)data[0] & 0x0F) << 8) | (u16)(data[1] & 0xFF)) << 4; +} + +static u16 cxd2841er_read_agc_gain_t_t2(struct cxd2841er_priv *priv, + u8 delsys) { u8 data[2]; cxd2841er_write_reg( priv, I2C_SLVT, 0x00, (delsys == SYS_DVBT ? 0x10 : 0x20)); cxd2841er_read_regs(priv, I2C_SLVT, 0x26, data, 2); + dev_dbg(&priv->i2c->dev, + "%s(): AGC value=%u\n", + __func__, (((u16)data[0] & 0x0F) << 8) | + (u16)(data[1] & 0xFF)); + return ((((u16)data[0] & 0x0F) << 8) | (u16)(data[1] & 0xFF)) << 4; +} + +static u16 cxd2841er_read_agc_gain_i(struct cxd2841er_priv *priv, + u8 delsys) +{ + u8 data[2]; + + cxd2841er_write_reg( + priv, I2C_SLVT, 0x00, 0x60); + cxd2841er_read_regs(priv, I2C_SLVT, 0x26, data, 2); + + dev_dbg(&priv->i2c->dev, + "%s(): AGC value=%u\n", + __func__, (((u16)data[0] & 0x0F) << 8) | + (u16)(data[1] & 0xFF)); return ((((u16)data[0] & 0x0F) << 8) | (u16)(data[1] & 0xFF)) << 4; } @@ -1417,101 +1839,170 @@ static u16 cxd2841er_read_agc_gain_s(struct cxd2841er_priv *priv) return ((((u16)data[0] & 0x1F) << 8) | (u16)(data[1] & 0xFF)) << 3; } -static int cxd2841er_read_ber(struct dvb_frontend *fe, u32 *ber) +static void cxd2841er_read_ber(struct dvb_frontend *fe) { struct dtv_frontend_properties *p = &fe->dtv_property_cache; struct cxd2841er_priv *priv = fe->demodulator_priv; + u32 ret, bit_error = 0, bit_count = 0; dev_dbg(&priv->i2c->dev, "%s()\n", __func__); - *ber = 0; switch (p->delivery_system) { + case SYS_DVBC_ANNEX_A: + case SYS_DVBC_ANNEX_B: + case SYS_DVBC_ANNEX_C: + ret = cxd2841er_read_ber_c(priv, &bit_error, &bit_count); + break; case SYS_DVBS: - *ber = cxd2841er_mon_read_ber_s(priv); + ret = cxd2841er_mon_read_ber_s(priv, &bit_error, &bit_count); break; case SYS_DVBS2: - *ber = cxd2841er_mon_read_ber_s2(priv); + ret = cxd2841er_mon_read_ber_s2(priv, &bit_error, &bit_count); break; case SYS_DVBT: - return cxd2841er_read_ber_t(priv, ber); + ret = cxd2841er_read_ber_t(priv, &bit_error, &bit_count); + break; case SYS_DVBT2: - return cxd2841er_read_ber_t2(priv, ber); - default: - *ber = 0; + ret = cxd2841er_read_ber_t2(priv, &bit_error, &bit_count); break; + default: + p->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + return; + } + + if (!ret) { + p->post_bit_error.stat[0].scale = FE_SCALE_COUNTER; + p->post_bit_error.stat[0].uvalue += bit_error; + p->post_bit_count.stat[0].scale = FE_SCALE_COUNTER; + p->post_bit_count.stat[0].uvalue += bit_count; + } else { + p->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; } - return 0; } -static int cxd2841er_read_signal_strength(struct dvb_frontend *fe, - u16 *strength) +static void cxd2841er_read_signal_strength(struct dvb_frontend *fe) { struct dtv_frontend_properties *p = &fe->dtv_property_cache; struct cxd2841er_priv *priv = fe->demodulator_priv; + s32 strength; dev_dbg(&priv->i2c->dev, "%s()\n", __func__); switch (p->delivery_system) { case SYS_DVBT: case SYS_DVBT2: - *strength = 65535 - cxd2841er_read_agc_gain_t_t2( - priv, p->delivery_system); + strength = cxd2841er_read_agc_gain_t_t2(priv, + p->delivery_system); + p->strength.stat[0].scale = FE_SCALE_DECIBEL; + /* Formula was empirically determinated @ 410 MHz */ + p->strength.stat[0].uvalue = strength * 366 / 100 - 89520; + break; /* Code moved out of the function */ + case SYS_DVBC_ANNEX_A: + case SYS_DVBC_ANNEX_B: + case SYS_DVBC_ANNEX_C: + strength = cxd2841er_read_agc_gain_c(priv, + p->delivery_system); + p->strength.stat[0].scale = FE_SCALE_DECIBEL; + /* + * Formula was empirically determinated via linear regression, + * using frequencies: 175 MHz, 410 MHz and 800 MHz, and a + * stream modulated with QAM64 + */ + p->strength.stat[0].uvalue = strength * 4045 / 1000 - 85224; + break; + case SYS_ISDBT: + strength = cxd2841er_read_agc_gain_i(priv, p->delivery_system); + p->strength.stat[0].scale = FE_SCALE_DECIBEL; + /* + * Formula was empirically determinated via linear regression, + * using frequencies: 175 MHz, 410 MHz and 800 MHz. + */ + p->strength.stat[0].uvalue = strength * 3775 / 1000 - 90185; break; case SYS_DVBS: case SYS_DVBS2: - *strength = 65535 - cxd2841er_read_agc_gain_s(priv); + strength = 65535 - cxd2841er_read_agc_gain_s(priv); + p->strength.stat[0].scale = FE_SCALE_RELATIVE; + p->strength.stat[0].uvalue = strength; break; default: - *strength = 0; + p->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE; break; } - return 0; } -static int cxd2841er_read_snr(struct dvb_frontend *fe, u16 *snr) +static void cxd2841er_read_snr(struct dvb_frontend *fe) { u32 tmp = 0; + int ret = 0; struct dtv_frontend_properties *p = &fe->dtv_property_cache; struct cxd2841er_priv *priv = fe->demodulator_priv; dev_dbg(&priv->i2c->dev, "%s()\n", __func__); switch (p->delivery_system) { + case SYS_DVBC_ANNEX_A: + case SYS_DVBC_ANNEX_B: + case SYS_DVBC_ANNEX_C: + ret = cxd2841er_read_snr_c(priv, &tmp); + break; case SYS_DVBT: - cxd2841er_read_snr_t(priv, &tmp); + ret = cxd2841er_read_snr_t(priv, &tmp); break; case SYS_DVBT2: - cxd2841er_read_snr_t2(priv, &tmp); + ret = cxd2841er_read_snr_t2(priv, &tmp); + break; + case SYS_ISDBT: + ret = cxd2841er_read_snr_i(priv, &tmp); break; case SYS_DVBS: case SYS_DVBS2: - tmp = cxd2841er_dvbs_read_snr(priv, p->delivery_system); + ret = cxd2841er_dvbs_read_snr(priv, p->delivery_system, &tmp); break; default: dev_dbg(&priv->i2c->dev, "%s(): unknown delivery system %d\n", __func__, p->delivery_system); - break; + p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + return; + } + + if (!ret) { + p->cnr.stat[0].scale = FE_SCALE_DECIBEL; + p->cnr.stat[0].svalue = tmp; + } else { + p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; } - *snr = tmp & 0xffff; - return 0; } -static int cxd2841er_read_ucblocks(struct dvb_frontend *fe, u32 *ucblocks) +static void cxd2841er_read_ucblocks(struct dvb_frontend *fe) { struct dtv_frontend_properties *p = &fe->dtv_property_cache; struct cxd2841er_priv *priv = fe->demodulator_priv; + u32 ucblocks; dev_dbg(&priv->i2c->dev, "%s()\n", __func__); switch (p->delivery_system) { + case SYS_DVBC_ANNEX_A: + case SYS_DVBC_ANNEX_B: + case SYS_DVBC_ANNEX_C: + cxd2841er_read_packet_errors_c(priv, &ucblocks); + break; case SYS_DVBT: - cxd2841er_read_packet_errors_t(priv, ucblocks); + cxd2841er_read_packet_errors_t(priv, &ucblocks); break; case SYS_DVBT2: - cxd2841er_read_packet_errors_t2(priv, ucblocks); + cxd2841er_read_packet_errors_t2(priv, &ucblocks); break; - default: - *ucblocks = 0; + case SYS_ISDBT: + cxd2841er_read_packet_errors_i(priv, &ucblocks); break; + default: + p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + return; } dev_dbg(&priv->i2c->dev, "%s()\n", __func__); - return 0; + + p->block_error.stat[0].scale = FE_SCALE_COUNTER; + p->block_error.stat[0].uvalue = ucblocks; } static int cxd2841er_dvbt2_set_profile( @@ -1524,15 +2015,18 @@ static int cxd2841er_dvbt2_set_profile( switch (profile) { case DVBT2_PROFILE_BASE: tune_mode = 0x01; - seq_not2d_time = 12; + /* Set early unlock time */ + seq_not2d_time = (priv->xtal == SONY_XTAL_24000)?0x0E:0x0C; break; case DVBT2_PROFILE_LITE: tune_mode = 0x05; - seq_not2d_time = 40; + /* Set early unlock time */ + seq_not2d_time = (priv->xtal == SONY_XTAL_24000)?0x2E:0x28; break; case DVBT2_PROFILE_ANY: tune_mode = 0x00; - seq_not2d_time = 40; + /* Set early unlock time */ + seq_not2d_time = (priv->xtal == SONY_XTAL_24000)?0x2E:0x28; break; default: return -EINVAL; @@ -1574,254 +2068,617 @@ static int cxd2841er_sleep_tc_to_active_t2_band(struct cxd2841er_priv *priv, u32 bandwidth) { u32 iffreq; - u8 b20_9f[5]; - u8 b10_a6[14]; - u8 b10_b6[3]; - u8 b10_d7; + u8 data[MAX_WRITE_REGSIZE]; + + const uint8_t nominalRate8bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x11, 0xF0, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */ + {0x15, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x11, 0xF0, 0x00, 0x00, 0x00} /* 41MHz XTal */ + }; + + const uint8_t nominalRate7bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x14, 0x80, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */ + {0x18, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x14, 0x80, 0x00, 0x00, 0x00} /* 41MHz XTal */ + }; + + const uint8_t nominalRate6bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x17, 0xEA, 0xAA, 0xAA, 0xAA}, /* 20.5MHz XTal */ + {0x1C, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x17, 0xEA, 0xAA, 0xAA, 0xAA} /* 41MHz XTal */ + }; + + const uint8_t nominalRate5bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x1C, 0xB3, 0x33, 0x33, 0x33}, /* 20.5MHz XTal */ + {0x21, 0x99, 0x99, 0x99, 0x99}, /* 24MHz XTal */ + {0x1C, 0xB3, 0x33, 0x33, 0x33} /* 41MHz XTal */ + }; + + const uint8_t nominalRate17bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x58, 0xE2, 0xAF, 0xE0, 0xBC}, /* 20.5MHz XTal */ + {0x68, 0x0F, 0xA2, 0x32, 0xD0}, /* 24MHz XTal */ + {0x58, 0xE2, 0xAF, 0xE0, 0xBC} /* 41MHz XTal */ + }; + + const uint8_t itbCoef8bw[3][14] = { + {0x26, 0xAF, 0x06, 0xCD, 0x13, 0xBB, 0x28, 0xBA, + 0x23, 0xA9, 0x1F, 0xA8, 0x2C, 0xC8}, /* 20.5MHz XTal */ + {0x2F, 0xBA, 0x28, 0x9B, 0x28, 0x9D, 0x28, 0xA1, + 0x29, 0xA5, 0x2A, 0xAC, 0x29, 0xB5}, /* 24MHz XTal */ + {0x26, 0xAF, 0x06, 0xCD, 0x13, 0xBB, 0x28, 0xBA, + 0x23, 0xA9, 0x1F, 0xA8, 0x2C, 0xC8} /* 41MHz XTal */ + }; + + const uint8_t itbCoef7bw[3][14] = { + {0x2C, 0xBD, 0x02, 0xCF, 0x04, 0xF8, 0x23, 0xA6, + 0x29, 0xB0, 0x26, 0xA9, 0x21, 0xA5}, /* 20.5MHz XTal */ + {0x30, 0xB1, 0x29, 0x9A, 0x28, 0x9C, 0x28, 0xA0, + 0x29, 0xA2, 0x2B, 0xA6, 0x2B, 0xAD}, /* 24MHz XTal */ + {0x2C, 0xBD, 0x02, 0xCF, 0x04, 0xF8, 0x23, 0xA6, + 0x29, 0xB0, 0x26, 0xA9, 0x21, 0xA5} /* 41MHz XTal */ + }; + + const uint8_t itbCoef6bw[3][14] = { + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, + 0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */ + {0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E, + 0x29, 0xA4, 0x29, 0xA2, 0x29, 0xA8}, /* 24MHz XTal */ + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, + 0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4} /* 41MHz XTal */ + }; + + const uint8_t itbCoef5bw[3][14] = { + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, + 0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */ + {0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E, + 0x29, 0xA4, 0x29, 0xA2, 0x29, 0xA8}, /* 24MHz XTal */ + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, + 0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4} /* 41MHz XTal */ + }; + + const uint8_t itbCoef17bw[3][14] = { + {0x25, 0xA0, 0x36, 0x8D, 0x2E, 0x94, 0x28, 0x9B, + 0x32, 0x90, 0x2C, 0x9D, 0x29, 0x99}, /* 20.5MHz XTal */ + {0x33, 0x8E, 0x2B, 0x97, 0x2D, 0x95, 0x37, 0x8B, + 0x30, 0x97, 0x2D, 0x9A, 0x21, 0xA4}, /* 24MHz XTal */ + {0x25, 0xA0, 0x36, 0x8D, 0x2E, 0x94, 0x28, 0x9B, + 0x32, 0x90, 0x2C, 0x9D, 0x29, 0x99} /* 41MHz XTal */ + }; + + /* Set SLV-T Bank : 0x20 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x20); - dev_dbg(&priv->i2c->dev, "%s()\n", __func__); switch (bandwidth) { case 8000000: - /* bank 0x20, reg 0x9f */ - b20_9f[0] = 0x11; - b20_9f[1] = 0xf0; - b20_9f[2] = 0x00; - b20_9f[3] = 0x00; - b20_9f[4] = 0x00; - /* bank 0x10, reg 0xa6 */ - b10_a6[0] = 0x26; - b10_a6[1] = 0xaf; - b10_a6[2] = 0x06; - b10_a6[3] = 0xcd; - b10_a6[4] = 0x13; - b10_a6[5] = 0xbb; - b10_a6[6] = 0x28; - b10_a6[7] = 0xba; - b10_a6[8] = 0x23; - b10_a6[9] = 0xa9; - b10_a6[10] = 0x1f; - b10_a6[11] = 0xa8; - b10_a6[12] = 0x2c; - b10_a6[13] = 0xc8; - iffreq = MAKE_IFFREQ_CONFIG(4.80); - b10_d7 = 0x00; + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate8bw[priv->xtal], 5); + + /* Set SLV-T Bank : 0x27 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x27); + cxd2841er_set_reg_bits(priv, I2C_SLVT, + 0x7a, 0x00, 0x0f); + + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef8bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 4.80); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x00, 0x07); break; case 7000000: - /* bank 0x20, reg 0x9f */ - b20_9f[0] = 0x14; - b20_9f[1] = 0x80; - b20_9f[2] = 0x00; - b20_9f[3] = 0x00; - b20_9f[4] = 0x00; - /* bank 0x10, reg 0xa6 */ - b10_a6[0] = 0x2C; - b10_a6[1] = 0xBD; - b10_a6[2] = 0x02; - b10_a6[3] = 0xCF; - b10_a6[4] = 0x04; - b10_a6[5] = 0xF8; - b10_a6[6] = 0x23; - b10_a6[7] = 0xA6; - b10_a6[8] = 0x29; - b10_a6[9] = 0xB0; - b10_a6[10] = 0x26; - b10_a6[11] = 0xA9; - b10_a6[12] = 0x21; - b10_a6[13] = 0xA5; - iffreq = MAKE_IFFREQ_CONFIG(4.2); - b10_d7 = 0x02; + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate7bw[priv->xtal], 5); + + /* Set SLV-T Bank : 0x27 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x27); + cxd2841er_set_reg_bits(priv, I2C_SLVT, + 0x7a, 0x00, 0x0f); + + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef7bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 4.20); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x02, 0x07); break; case 6000000: - /* bank 0x20, reg 0x9f */ - b20_9f[0] = 0x17; - b20_9f[1] = 0xEA; - b20_9f[2] = 0xAA; - b20_9f[3] = 0xAA; - b20_9f[4] = 0xAA; - /* bank 0x10, reg 0xa6 */ - b10_a6[0] = 0x27; - b10_a6[1] = 0xA7; - b10_a6[2] = 0x28; - b10_a6[3] = 0xB3; - b10_a6[4] = 0x02; - b10_a6[5] = 0xF0; - b10_a6[6] = 0x01; - b10_a6[7] = 0xE8; - b10_a6[8] = 0x00; - b10_a6[9] = 0xCF; - b10_a6[10] = 0x00; - b10_a6[11] = 0xE6; - b10_a6[12] = 0x23; - b10_a6[13] = 0xA4; - iffreq = MAKE_IFFREQ_CONFIG(3.6); - b10_d7 = 0x04; + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate6bw[priv->xtal], 5); + + /* Set SLV-T Bank : 0x27 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x27); + cxd2841er_set_reg_bits(priv, I2C_SLVT, + 0x7a, 0x00, 0x0f); + + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef6bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 3.60); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x04, 0x07); break; case 5000000: - /* bank 0x20, reg 0x9f */ - b20_9f[0] = 0x1C; - b20_9f[1] = 0xB3; - b20_9f[2] = 0x33; - b20_9f[3] = 0x33; - b20_9f[4] = 0x33; - /* bank 0x10, reg 0xa6 */ - b10_a6[0] = 0x27; - b10_a6[1] = 0xA7; - b10_a6[2] = 0x28; - b10_a6[3] = 0xB3; - b10_a6[4] = 0x02; - b10_a6[5] = 0xF0; - b10_a6[6] = 0x01; - b10_a6[7] = 0xE8; - b10_a6[8] = 0x00; - b10_a6[9] = 0xCF; - b10_a6[10] = 0x00; - b10_a6[11] = 0xE6; - b10_a6[12] = 0x23; - b10_a6[13] = 0xA4; - iffreq = MAKE_IFFREQ_CONFIG(3.6); - b10_d7 = 0x06; + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate5bw[priv->xtal], 5); + + /* Set SLV-T Bank : 0x27 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x27); + cxd2841er_set_reg_bits(priv, I2C_SLVT, + 0x7a, 0x00, 0x0f); + + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef5bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 3.60); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x06, 0x07); break; case 1712000: - /* bank 0x20, reg 0x9f */ - b20_9f[0] = 0x58; - b20_9f[1] = 0xE2; - b20_9f[2] = 0xAF; - b20_9f[3] = 0xE0; - b20_9f[4] = 0xBC; - /* bank 0x10, reg 0xa6 */ - b10_a6[0] = 0x25; - b10_a6[1] = 0xA0; - b10_a6[2] = 0x36; - b10_a6[3] = 0x8D; - b10_a6[4] = 0x2E; - b10_a6[5] = 0x94; - b10_a6[6] = 0x28; - b10_a6[7] = 0x9B; - b10_a6[8] = 0x32; - b10_a6[9] = 0x90; - b10_a6[10] = 0x2C; - b10_a6[11] = 0x9D; - b10_a6[12] = 0x29; - b10_a6[13] = 0x99; - iffreq = MAKE_IFFREQ_CONFIG(3.5); - b10_d7 = 0x03; + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate17bw[priv->xtal], 5); + + /* Set SLV-T Bank : 0x27 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x27); + cxd2841er_set_reg_bits(priv, I2C_SLVT, + 0x7a, 0x03, 0x0f); + + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef17bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 3.50); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x03, 0x07); break; default: return -EINVAL; } - /* Set SLV-T Bank : 0x20 */ - cxd2841er_write_reg(priv, I2C_SLVX, 0x00, 0x20); - cxd2841er_write_regs(priv, I2C_SLVT, 0x9f, b20_9f, sizeof(b20_9f)); - /* Set SLV-T Bank : 0x27 */ - cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x27); - cxd2841er_set_reg_bits( - priv, I2C_SLVT, 0x7a, - (bandwidth == 1712000 ? 0x03 : 0x00), 0x0f); - /* Set SLV-T Bank : 0x10 */ - cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); - /* Group delay equaliser sett. for ASCOT2E */ - cxd2841er_write_regs(priv, I2C_SLVT, 0xa6, b10_a6, sizeof(b10_a6)); - /* */ - b10_b6[0] = (u8) ((iffreq >> 16) & 0xff); - b10_b6[1] = (u8)((iffreq >> 8) & 0xff); - b10_b6[2] = (u8)(iffreq & 0xff); - cxd2841er_write_regs(priv, I2C_SLVT, 0xb6, b10_b6, sizeof(b10_b6)); - /* System bandwidth setting */ - cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xd7, b10_d7, 0x07); return 0; } static int cxd2841er_sleep_tc_to_active_t_band( struct cxd2841er_priv *priv, u32 bandwidth) { - u8 b13_9c[2] = { 0x01, 0x14 }; - u8 bw8mhz_b10_9f[] = { 0x11, 0xF0, 0x00, 0x00, 0x00 }; - u8 bw8mhz_b10_a6[] = { 0x26, 0xAF, 0x06, 0xCD, 0x13, 0xBB, - 0x28, 0xBA, 0x23, 0xA9, 0x1F, 0xA8, 0x2C, 0xC8 }; - u8 bw8mhz_b10_d9[] = { 0x01, 0xE0 }; - u8 bw8mhz_b17_38[] = { 0x01, 0x02 }; - u8 bw7mhz_b10_9f[] = { 0x14, 0x80, 0x00, 0x00, 0x00 }; - u8 bw7mhz_b10_a6[] = { 0x2C, 0xBD, 0x02, 0xCF, 0x04, 0xF8, - 0x23, 0xA6, 0x29, 0xB0, 0x26, 0xA9, 0x21, 0xA5 }; - u8 bw7mhz_b10_d9[] = { 0x12, 0xF8 }; - u8 bw7mhz_b17_38[] = { 0x00, 0x03 }; - u8 bw6mhz_b10_9f[] = { 0x17, 0xEA, 0xAA, 0xAA, 0xAA }; - u8 bw6mhz_b10_a6[] = { 0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, - 0x01, 0xE8, 0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4 }; - u8 bw6mhz_b10_d9[] = { 0x1F, 0xDC }; - u8 bw6mhz_b17_38[] = { 0x00, 0x03 }; - u8 bw5mhz_b10_9f[] = { 0x1C, 0xB3, 0x33, 0x33, 0x33 }; - u8 bw5mhz_b10_a6[] = { 0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, - 0x01, 0xE8, 0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4 }; - u8 bw5mhz_b10_d9[] = { 0x26, 0x3C }; - u8 bw5mhz_b17_38[] = { 0x00, 0x03 }; - u8 b10_b6[3]; - u8 d7val; + u8 data[MAX_WRITE_REGSIZE]; u32 iffreq; - u8 *b10_9f; - u8 *b10_a6; - u8 *b10_d9; - u8 *b17_38; + u8 nominalRate8bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x11, 0xF0, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */ + {0x15, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x11, 0xF0, 0x00, 0x00, 0x00} /* 41MHz XTal */ + }; + u8 nominalRate7bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x14, 0x80, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */ + {0x18, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x14, 0x80, 0x00, 0x00, 0x00} /* 41MHz XTal */ + }; + u8 nominalRate6bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x17, 0xEA, 0xAA, 0xAA, 0xAA}, /* 20.5MHz XTal */ + {0x1C, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x17, 0xEA, 0xAA, 0xAA, 0xAA} /* 41MHz XTal */ + }; + u8 nominalRate5bw[3][5] = { + /* TRCG Nominal Rate [37:0] */ + {0x1C, 0xB3, 0x33, 0x33, 0x33}, /* 20.5MHz XTal */ + {0x21, 0x99, 0x99, 0x99, 0x99}, /* 24MHz XTal */ + {0x1C, 0xB3, 0x33, 0x33, 0x33} /* 41MHz XTal */ + }; - dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + u8 itbCoef8bw[3][14] = { + {0x26, 0xAF, 0x06, 0xCD, 0x13, 0xBB, 0x28, 0xBA, 0x23, 0xA9, + 0x1F, 0xA8, 0x2C, 0xC8}, /* 20.5MHz XTal */ + {0x2F, 0xBA, 0x28, 0x9B, 0x28, 0x9D, 0x28, 0xA1, 0x29, 0xA5, + 0x2A, 0xAC, 0x29, 0xB5}, /* 24MHz XTal */ + {0x26, 0xAF, 0x06, 0xCD, 0x13, 0xBB, 0x28, 0xBA, 0x23, 0xA9, + 0x1F, 0xA8, 0x2C, 0xC8} /* 41MHz XTal */ + }; + u8 itbCoef7bw[3][14] = { + {0x2C, 0xBD, 0x02, 0xCF, 0x04, 0xF8, 0x23, 0xA6, 0x29, 0xB0, + 0x26, 0xA9, 0x21, 0xA5}, /* 20.5MHz XTal */ + {0x30, 0xB1, 0x29, 0x9A, 0x28, 0x9C, 0x28, 0xA0, 0x29, 0xA2, + 0x2B, 0xA6, 0x2B, 0xAD}, /* 24MHz XTal */ + {0x2C, 0xBD, 0x02, 0xCF, 0x04, 0xF8, 0x23, 0xA6, 0x29, 0xB0, + 0x26, 0xA9, 0x21, 0xA5} /* 41MHz XTal */ + }; + u8 itbCoef6bw[3][14] = { + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, 0xCF, + 0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */ + {0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E, 0x29, 0xA4, + 0x29, 0xA2, 0x29, 0xA8}, /* 24MHz XTal */ + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, 0xCF, + 0x00, 0xE6, 0x23, 0xA4} /* 41MHz XTal */ + }; + u8 itbCoef5bw[3][14] = { + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, 0xCF, + 0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */ + {0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E, 0x29, 0xA4, + 0x29, 0xA2, 0x29, 0xA8}, /* 24MHz XTal */ + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, 0xCF, + 0x00, 0xE6, 0x23, 0xA4} /* 41MHz XTal */ + }; + + /* Set SLV-T Bank : 0x13 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x13); /* Echo performance optimization setting */ - cxd2841er_write_regs(priv, I2C_SLVT, 0x9c, b13_9c, sizeof(b13_9c)); + data[0] = 0x01; + data[1] = 0x14; + cxd2841er_write_regs(priv, I2C_SLVT, 0x9C, data, 2); + + /* Set SLV-T Bank : 0x10 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); switch (bandwidth) { case 8000000: - b10_9f = bw8mhz_b10_9f; - b10_a6 = bw8mhz_b10_a6; - b10_d9 = bw8mhz_b10_d9; - b17_38 = bw8mhz_b17_38; - d7val = 0; - iffreq = MAKE_IFFREQ_CONFIG(4.80); + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate8bw[priv->xtal], 5); + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef8bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 4.80); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x00, 0x07); + + /* Demod core latency setting */ + if (priv->xtal == SONY_XTAL_24000) { + data[0] = 0x15; + data[1] = 0x28; + } else { + data[0] = 0x01; + data[1] = 0xE0; + } + cxd2841er_write_regs(priv, I2C_SLVT, 0xD9, data, 2); + + /* Notch filter setting */ + data[0] = 0x01; + data[1] = 0x02; + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x17); + cxd2841er_write_regs(priv, I2C_SLVT, 0x38, data, 2); break; case 7000000: - b10_9f = bw7mhz_b10_9f; - b10_a6 = bw7mhz_b10_a6; - b10_d9 = bw7mhz_b10_d9; - b17_38 = bw7mhz_b17_38; - d7val = 2; - iffreq = MAKE_IFFREQ_CONFIG(4.20); + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate7bw[priv->xtal], 5); + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef7bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 4.20); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x02, 0x07); + + /* Demod core latency setting */ + if (priv->xtal == SONY_XTAL_24000) { + data[0] = 0x1F; + data[1] = 0xF8; + } else { + data[0] = 0x12; + data[1] = 0xF8; + } + cxd2841er_write_regs(priv, I2C_SLVT, 0xD9, data, 2); + + /* Notch filter setting */ + data[0] = 0x00; + data[1] = 0x03; + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x17); + cxd2841er_write_regs(priv, I2C_SLVT, 0x38, data, 2); break; case 6000000: - b10_9f = bw6mhz_b10_9f; - b10_a6 = bw6mhz_b10_a6; - b10_d9 = bw6mhz_b10_d9; - b17_38 = bw6mhz_b17_38; - d7val = 4; - iffreq = MAKE_IFFREQ_CONFIG(3.60); + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate6bw[priv->xtal], 5); + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef6bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 3.60); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x04, 0x07); + + /* Demod core latency setting */ + if (priv->xtal == SONY_XTAL_24000) { + data[0] = 0x25; + data[1] = 0x4C; + } else { + data[0] = 0x1F; + data[1] = 0xDC; + } + cxd2841er_write_regs(priv, I2C_SLVT, 0xD9, data, 2); + + /* Notch filter setting */ + data[0] = 0x00; + data[1] = 0x03; + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x17); + cxd2841er_write_regs(priv, I2C_SLVT, 0x38, data, 2); break; case 5000000: - b10_9f = bw5mhz_b10_9f; - b10_a6 = bw5mhz_b10_a6; - b10_d9 = bw5mhz_b10_d9; - b17_38 = bw5mhz_b17_38; - d7val = 6; - iffreq = MAKE_IFFREQ_CONFIG(3.60); + /* */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate5bw[priv->xtal], 5); + /* Group delay equaliser settings for + * ASCOT2D, ASCOT2E and ASCOT3 tuners + */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef5bw[priv->xtal], 14); + /* */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 3.60); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + /* System bandwidth setting */ + cxd2841er_set_reg_bits( + priv, I2C_SLVT, 0xD7, 0x06, 0x07); + + /* Demod core latency setting */ + if (priv->xtal == SONY_XTAL_24000) { + data[0] = 0x2C; + data[1] = 0xC2; + } else { + data[0] = 0x26; + data[1] = 0x3C; + } + cxd2841er_write_regs(priv, I2C_SLVT, 0xD9, data, 2); + + /* Notch filter setting */ + data[0] = 0x00; + data[1] = 0x03; + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x17); + cxd2841er_write_regs(priv, I2C_SLVT, 0x38, data, 2); + break; + } + + return 0; +} + +static int cxd2841er_sleep_tc_to_active_i_band( + struct cxd2841er_priv *priv, u32 bandwidth) +{ + u32 iffreq; + u8 data[3]; + + /* TRCG Nominal Rate */ + u8 nominalRate8bw[3][5] = { + {0x00, 0x00, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */ + {0x11, 0xB8, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x00, 0x00, 0x00, 0x00, 0x00} /* 41MHz XTal */ + }; + + u8 nominalRate7bw[3][5] = { + {0x00, 0x00, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */ + {0x14, 0x40, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x00, 0x00, 0x00, 0x00, 0x00} /* 41MHz XTal */ + }; + + u8 nominalRate6bw[3][5] = { + {0x14, 0x2E, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */ + {0x17, 0xA0, 0x00, 0x00, 0x00}, /* 24MHz XTal */ + {0x14, 0x2E, 0x00, 0x00, 0x00} /* 41MHz XTal */ + }; + + u8 itbCoef8bw[3][14] = { + {0x00}, /* 20.5MHz XTal */ + {0x2F, 0xBA, 0x28, 0x9B, 0x28, 0x9D, 0x28, 0xA1, 0x29, + 0xA5, 0x2A, 0xAC, 0x29, 0xB5}, /* 24MHz Xtal */ + {0x0}, /* 41MHz XTal */ + }; + + u8 itbCoef7bw[3][14] = { + {0x00}, /* 20.5MHz XTal */ + {0x30, 0xB1, 0x29, 0x9A, 0x28, 0x9C, 0x28, 0xA0, 0x29, + 0xA2, 0x2B, 0xA6, 0x2B, 0xAD}, /* 24MHz Xtal */ + {0x00}, /* 41MHz XTal */ + }; + + u8 itbCoef6bw[3][14] = { + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, + 0xCF, 0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */ + {0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E, 0x29, + 0xA4, 0x29, 0xA2, 0x29, 0xA8}, /* 24MHz Xtal */ + {0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, + 0xCF, 0x00, 0xE6, 0x23, 0xA4}, /* 41MHz XTal */ + }; + + dev_dbg(&priv->i2c->dev, "%s() bandwidth=%u\n", __func__, bandwidth); + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + + /* 20.5/41MHz Xtal support is not available + * on ISDB-T 7MHzBW and 8MHzBW + */ + if (priv->xtal != SONY_XTAL_24000 && bandwidth > 6000000) { + dev_err(&priv->i2c->dev, + "%s(): bandwidth %d supported only for 24MHz xtal\n", + __func__, bandwidth); + return -EINVAL; + } + + switch (bandwidth) { + case 8000000: + /* TRCG Nominal Rate */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate8bw[priv->xtal], 5); + /* Group delay equaliser settings for ASCOT tuners optimized */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef8bw[priv->xtal], 14); + + /* IF freq setting */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 4.75); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + + /* System bandwidth setting */ + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xd7, 0x0, 0x7); + + /* Demod core latency setting */ + data[0] = 0x13; + data[1] = 0xFC; + cxd2841er_write_regs(priv, I2C_SLVT, 0xD9, data, 2); + + /* Acquisition optimization setting */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x12); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x71, 0x03, 0x07); + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x15); + cxd2841er_write_reg(priv, I2C_SLVT, 0xBE, 0x03); + break; + case 7000000: + /* TRCG Nominal Rate */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate7bw[priv->xtal], 5); + /* Group delay equaliser settings for ASCOT tuners optimized */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef7bw[priv->xtal], 14); + + /* IF freq setting */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 4.15); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + + /* System bandwidth setting */ + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xd7, 0x02, 0x7); + + /* Demod core latency setting */ + data[0] = 0x1A; + data[1] = 0xFA; + cxd2841er_write_regs(priv, I2C_SLVT, 0xD9, data, 2); + + /* Acquisition optimization setting */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x12); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x71, 0x03, 0x07); + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x15); + cxd2841er_write_reg(priv, I2C_SLVT, 0xBE, 0x02); + break; + case 6000000: + /* TRCG Nominal Rate */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0x9F, nominalRate6bw[priv->xtal], 5); + /* Group delay equaliser settings for ASCOT tuners optimized */ + cxd2841er_write_regs(priv, I2C_SLVT, + 0xA6, itbCoef6bw[priv->xtal], 14); + + /* IF freq setting */ + iffreq = MAKE_IFFREQ_CONFIG_XTAL(priv->xtal, 3.55); + data[0] = (u8) ((iffreq >> 16) & 0xff); + data[1] = (u8)((iffreq >> 8) & 0xff); + data[2] = (u8)(iffreq & 0xff); + cxd2841er_write_regs(priv, I2C_SLVT, 0xB6, data, 3); + + /* System bandwidth setting */ + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xd7, 0x04, 0x7); + + /* Demod core latency setting */ + if (priv->xtal == SONY_XTAL_24000) { + data[0] = 0x1F; + data[1] = 0x79; + } else { + data[0] = 0x1A; + data[1] = 0xE2; + } + cxd2841er_write_regs(priv, I2C_SLVT, 0xD9, data, 2); + + /* Acquisition optimization setting */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x12); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x71, 0x07, 0x07); + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x15); + cxd2841er_write_reg(priv, I2C_SLVT, 0xBE, 0x02); break; default: dev_dbg(&priv->i2c->dev, "%s(): invalid bandwidth %d\n", - __func__, bandwidth); + __func__, bandwidth); return -EINVAL; } - /* */ - b10_b6[0] = (u8) ((iffreq >> 16) & 0xff); - b10_b6[1] = (u8)((iffreq >> 8) & 0xff); - b10_b6[2] = (u8)(iffreq & 0xff); - cxd2841er_write_regs( - priv, I2C_SLVT, 0x9f, b10_9f, sizeof(bw8mhz_b10_9f)); - cxd2841er_write_regs( - priv, I2C_SLVT, 0xa6, b10_a6, sizeof(bw8mhz_b10_a6)); - cxd2841er_write_regs(priv, I2C_SLVT, 0xb6, b10_b6, sizeof(b10_b6)); - cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xd7, d7val, 0x7); - cxd2841er_write_regs( - priv, I2C_SLVT, 0xd9, b10_d9, sizeof(bw8mhz_b10_d9)); - cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x17); - cxd2841er_write_regs( - priv, I2C_SLVT, 0x38, b17_38, sizeof(bw8mhz_b17_38)); return 0; } @@ -1837,7 +2694,7 @@ static int cxd2841er_sleep_tc_to_active_c_band(struct cxd2841er_priv *priv, u8 b10_b6[3]; u32 iffreq; - dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + dev_dbg(&priv->i2c->dev, "%s() bw=%d\n", __func__, bandwidth); cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); switch (bandwidth) { case 8000000: @@ -1854,7 +2711,7 @@ static int cxd2841er_sleep_tc_to_active_c_band(struct cxd2841er_priv *priv, iffreq = MAKE_IFFREQ_CONFIG(3.7); break; default: - dev_dbg(&priv->i2c->dev, "%s(): unsupported bandwidth %d\n", + dev_err(&priv->i2c->dev, "%s(): unsupported bandwidth %d\n", __func__, bandwidth); return -EINVAL; } @@ -1902,6 +2759,7 @@ static int cxd2841er_sleep_tc_to_active_t(struct cxd2841er_priv *priv, u32 bandwidth) { u8 data[2] = { 0x09, 0x54 }; + u8 data24m[3] = {0xDC, 0x6C, 0x00}; dev_dbg(&priv->i2c->dev, "%s()\n", __func__); cxd2841er_set_ts_clock_mode(priv, SYS_DVBT); @@ -1919,7 +2777,11 @@ static int cxd2841er_sleep_tc_to_active_t(struct cxd2841er_priv *priv, cxd2841er_write_reg(priv, I2C_SLVT, 0x30, 0x00); /* Enable ADC 1 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x41, 0x1a); - /* xtal freq 20.5MHz */ + /* Enable ADC 2 & 3 */ + if (priv->xtal == SONY_XTAL_41000) { + data[0] = 0x0A; + data[1] = 0xD4; + } cxd2841er_write_regs(priv, I2C_SLVT, 0x43, data, 2); /* Enable ADC 4 */ cxd2841er_write_reg(priv, I2C_SLVX, 0x18, 0x00); @@ -1947,6 +2809,15 @@ static int cxd2841er_sleep_tc_to_active_t(struct cxd2841er_priv *priv, /* TSIF setting */ cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xce, 0x01, 0x01); cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xcf, 0x01, 0x01); + + if (priv->xtal == SONY_XTAL_24000) { + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + cxd2841er_write_reg(priv, I2C_SLVT, 0xBF, 0x60); + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x18); + cxd2841er_write_regs(priv, I2C_SLVT, 0x24, data24m, 3); + } + cxd2841er_sleep_tc_to_active_t_band(priv, bandwidth); /* Set SLV-T Bank : 0x00 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); @@ -1961,7 +2832,7 @@ static int cxd2841er_sleep_tc_to_active_t(struct cxd2841er_priv *priv, static int cxd2841er_sleep_tc_to_active_t2(struct cxd2841er_priv *priv, u32 bandwidth) { - u8 data[2] = { 0x09, 0x54 }; + u8 data[MAX_WRITE_REGSIZE]; dev_dbg(&priv->i2c->dev, "%s()\n", __func__); cxd2841er_set_ts_clock_mode(priv, SYS_DVBT2); @@ -1974,12 +2845,21 @@ static int cxd2841er_sleep_tc_to_active_t2(struct cxd2841er_priv *priv, /* Enable demod clock */ cxd2841er_write_reg(priv, I2C_SLVT, 0x2c, 0x01); /* Disable RF level monitor */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x59, 0x00); cxd2841er_write_reg(priv, I2C_SLVT, 0x2f, 0x00); /* Enable ADC clock */ cxd2841er_write_reg(priv, I2C_SLVT, 0x30, 0x00); /* Enable ADC 1 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x41, 0x1a); - /* xtal freq 20.5MHz */ + + if (priv->xtal == SONY_XTAL_41000) { + data[0] = 0x0A; + data[1] = 0xD4; + } else { + data[0] = 0x09; + data[1] = 0x54; + } + cxd2841er_write_regs(priv, I2C_SLVT, 0x43, data, 2); /* Enable ADC 4 */ cxd2841er_write_reg(priv, I2C_SLVX, 0x18, 0x00); @@ -2002,6 +2882,10 @@ static int cxd2841er_sleep_tc_to_active_t2(struct cxd2841er_priv *priv, /* Set SLV-T Bank : 0x2b */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x2b); cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x76, 0x20, 0x70); + /* Set SLV-T Bank : 0x23 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x23); + /* L1 Control setting */ + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xE6, 0x00, 0x03); /* Set SLV-T Bank : 0x00 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); /* TSIF setting */ @@ -2020,6 +2904,72 @@ static int cxd2841er_sleep_tc_to_active_t2(struct cxd2841er_priv *priv, cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x2b); cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x11, 0x20, 0x3f); + /* 24MHz Xtal setting */ + if (priv->xtal == SONY_XTAL_24000) { + /* Set SLV-T Bank : 0x11 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x11); + data[0] = 0xEB; + data[1] = 0x03; + data[2] = 0x3B; + cxd2841er_write_regs(priv, I2C_SLVT, 0x33, data, 3); + + /* Set SLV-T Bank : 0x20 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x20); + data[0] = 0x5E; + data[1] = 0x5E; + data[2] = 0x47; + cxd2841er_write_regs(priv, I2C_SLVT, 0x95, data, 3); + + cxd2841er_write_reg(priv, I2C_SLVT, 0x99, 0x18); + + data[0] = 0x3F; + data[1] = 0xFF; + cxd2841er_write_regs(priv, I2C_SLVT, 0xD9, data, 2); + + /* Set SLV-T Bank : 0x24 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x24); + data[0] = 0x0B; + data[1] = 0x72; + cxd2841er_write_regs(priv, I2C_SLVT, 0x34, data, 2); + + data[0] = 0x93; + data[1] = 0xF3; + data[2] = 0x00; + cxd2841er_write_regs(priv, I2C_SLVT, 0xD2, data, 3); + + data[0] = 0x05; + data[1] = 0xB8; + data[2] = 0xD8; + cxd2841er_write_regs(priv, I2C_SLVT, 0xDD, data, 3); + + cxd2841er_write_reg(priv, I2C_SLVT, 0xE0, 0x00); + + /* Set SLV-T Bank : 0x25 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x25); + cxd2841er_write_reg(priv, I2C_SLVT, 0xED, 0x60); + + /* Set SLV-T Bank : 0x27 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x27); + cxd2841er_write_reg(priv, I2C_SLVT, 0xFA, 0x34); + + /* Set SLV-T Bank : 0x2B */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x2B); + cxd2841er_write_reg(priv, I2C_SLVT, 0x4B, 0x2F); + cxd2841er_write_reg(priv, I2C_SLVT, 0x9E, 0x0E); + + /* Set SLV-T Bank : 0x2D */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x2D); + data[0] = 0x89; + data[1] = 0x89; + cxd2841er_write_regs(priv, I2C_SLVT, 0x24, data, 2); + + /* Set SLV-T Bank : 0x5E */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x5E); + data[0] = 0x24; + data[1] = 0x95; + cxd2841er_write_regs(priv, I2C_SLVT, 0x8C, data, 2); + } + cxd2841er_sleep_tc_to_active_t2_band(priv, bandwidth); /* Set SLV-T Bank : 0x00 */ @@ -2032,6 +2982,84 @@ static int cxd2841er_sleep_tc_to_active_t2(struct cxd2841er_priv *priv, return 0; } +/* ISDB-Tb part */ +static int cxd2841er_sleep_tc_to_active_i(struct cxd2841er_priv *priv, + u32 bandwidth) +{ + u8 data[2] = { 0x09, 0x54 }; + u8 data24m[2] = {0x60, 0x00}; + u8 data24m2[3] = {0xB7, 0x1B, 0x00}; + + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + cxd2841er_set_ts_clock_mode(priv, SYS_DVBT); + /* Set SLV-X Bank : 0x00 */ + cxd2841er_write_reg(priv, I2C_SLVX, 0x00, 0x00); + /* Set demod mode */ + cxd2841er_write_reg(priv, I2C_SLVX, 0x17, 0x06); + /* Set SLV-T Bank : 0x00 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); + /* Enable demod clock */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x2c, 0x01); + /* Enable RF level monitor */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x2f, 0x01); + cxd2841er_write_reg(priv, I2C_SLVT, 0x59, 0x01); + /* Enable ADC clock */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x30, 0x00); + /* Enable ADC 1 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x41, 0x1a); + /* xtal freq 20.5MHz or 24M */ + cxd2841er_write_regs(priv, I2C_SLVT, 0x43, data, 2); + /* Enable ADC 4 */ + cxd2841er_write_reg(priv, I2C_SLVX, 0x18, 0x00); + /* ASCOT setting ON */ + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xa5, 0x01, 0x01); + /* FEC Auto Recovery setting */ + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x30, 0x01, 0x01); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x31, 0x00, 0x01); + /* ISDB-T initial setting */ + /* Set SLV-T Bank : 0x00 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xce, 0x00, 0x01); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xcf, 0x00, 0x01); + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x69, 0x04, 0x07); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x6B, 0x03, 0x07); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x9D, 0x50, 0xFF); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xD3, 0x06, 0x1F); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xED, 0x00, 0x01); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xE2, 0xCE, 0x80); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xF2, 0x13, 0x10); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xDE, 0x2E, 0x3F); + /* Set SLV-T Bank : 0x15 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x15); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xDE, 0x02, 0x03); + /* Set SLV-T Bank : 0x1E */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x1E); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x73, 0x68, 0xFF); + /* Set SLV-T Bank : 0x63 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x63); + cxd2841er_set_reg_bits(priv, I2C_SLVT, 0x81, 0x00, 0x01); + + /* for xtal 24MHz */ + /* Set SLV-T Bank : 0x10 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); + cxd2841er_write_regs(priv, I2C_SLVT, 0xBF, data24m, 2); + /* Set SLV-T Bank : 0x60 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x60); + cxd2841er_write_regs(priv, I2C_SLVT, 0xA8, data24m2, 3); + + cxd2841er_sleep_tc_to_active_i_band(priv, bandwidth); + /* Set SLV-T Bank : 0x00 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); + /* Disable HiZ Setting 1 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x80, 0x28); + /* Disable HiZ Setting 2 */ + cxd2841er_write_reg(priv, I2C_SLVT, 0x81, 0x00); + priv->state = STATE_ACTIVE_TC; + return 0; +} + static int cxd2841er_sleep_tc_to_active_c(struct cxd2841er_priv *priv, u32 bandwidth) { @@ -2079,7 +3107,7 @@ static int cxd2841er_sleep_tc_to_active_c(struct cxd2841er_priv *priv, cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xce, 0x01, 0x01); cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xcf, 0x01, 0x01); - cxd2841er_sleep_tc_to_active_c_band(priv, 8000000); + cxd2841er_sleep_tc_to_active_c_band(priv, bandwidth); /* Set SLV-T Bank : 0x00 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); /* Disable HiZ Setting 1 */ @@ -2094,8 +3122,6 @@ static int cxd2841er_get_frontend(struct dvb_frontend *fe, struct dtv_frontend_properties *p) { enum fe_status status = 0; - u16 strength = 0, snr = 0; - u32 errors = 0, ber = 0; struct cxd2841er_priv *priv = fe->demodulator_priv; dev_dbg(&priv->i2c->dev, "%s()\n", __func__); @@ -2104,32 +3130,18 @@ static int cxd2841er_get_frontend(struct dvb_frontend *fe, else if (priv->state == STATE_ACTIVE_TC) cxd2841er_read_status_tc(fe, &status); + cxd2841er_read_signal_strength(fe); + if (status & FE_HAS_LOCK) { - cxd2841er_read_signal_strength(fe, &strength); - p->strength.len = 1; - p->strength.stat[0].scale = FE_SCALE_RELATIVE; - p->strength.stat[0].uvalue = strength; - cxd2841er_read_snr(fe, &snr); - p->cnr.len = 1; - p->cnr.stat[0].scale = FE_SCALE_DECIBEL; - p->cnr.stat[0].svalue = snr; - cxd2841er_read_ucblocks(fe, &errors); - p->block_error.len = 1; - p->block_error.stat[0].scale = FE_SCALE_COUNTER; - p->block_error.stat[0].uvalue = errors; - cxd2841er_read_ber(fe, &ber); - p->post_bit_error.len = 1; - p->post_bit_error.stat[0].scale = FE_SCALE_COUNTER; - p->post_bit_error.stat[0].uvalue = ber; + cxd2841er_read_snr(fe); + cxd2841er_read_ucblocks(fe); + + cxd2841er_read_ber(fe); } else { - p->strength.len = 1; - p->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - p->cnr.len = 1; p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - p->block_error.len = 1; p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - p->post_bit_error.len = 1; p->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; } return 0; } @@ -2142,10 +3154,10 @@ static int cxd2841er_set_frontend_s(struct dvb_frontend *fe) struct dtv_frontend_properties *p = &fe->dtv_property_cache; u32 symbol_rate = p->symbol_rate/1000; - dev_dbg(&priv->i2c->dev, "%s(): %s frequency=%d symbol_rate=%d\n", + dev_dbg(&priv->i2c->dev, "%s(): %s frequency=%d symbol_rate=%d xtal=%d\n", __func__, (p->delivery_system == SYS_DVBS ? "DVB-S" : "DVB-S2"), - p->frequency, symbol_rate); + p->frequency, symbol_rate, priv->xtal); switch (priv->state) { case STATE_SLEEP_S: ret = cxd2841er_sleep_s_to_active_s( @@ -2189,6 +3201,13 @@ static int cxd2841er_set_frontend_s(struct dvb_frontend *fe) __func__, carr_offset); } done: + /* Reset stats */ + p->strength.stat[0].scale = FE_SCALE_RELATIVE; + p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + return ret; } @@ -2199,7 +3218,8 @@ static int cxd2841er_set_frontend_tc(struct dvb_frontend *fe) struct cxd2841er_priv *priv = fe->demodulator_priv; struct dtv_frontend_properties *p = &fe->dtv_property_cache; - dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + dev_dbg(&priv->i2c->dev, "%s() delivery_system=%d bandwidth_hz=%d\n", + __func__, p->delivery_system, p->bandwidth_hz); if (p->delivery_system == SYS_DVBT) { priv->system = SYS_DVBT; switch (priv->state) { @@ -2233,9 +3253,33 @@ static int cxd2841er_set_frontend_tc(struct dvb_frontend *fe) __func__, priv->state); ret = -EINVAL; } + } else if (p->delivery_system == SYS_ISDBT) { + priv->system = SYS_ISDBT; + switch (priv->state) { + case STATE_SLEEP_TC: + ret = cxd2841er_sleep_tc_to_active_i( + priv, p->bandwidth_hz); + break; + case STATE_ACTIVE_TC: + ret = cxd2841er_retune_active(priv, p); + break; + default: + dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n", + __func__, priv->state); + ret = -EINVAL; + } } else if (p->delivery_system == SYS_DVBC_ANNEX_A || p->delivery_system == SYS_DVBC_ANNEX_C) { priv->system = SYS_DVBC_ANNEX_A; + /* correct bandwidth */ + if (p->bandwidth_hz != 6000000 && + p->bandwidth_hz != 7000000 && + p->bandwidth_hz != 8000000) { + p->bandwidth_hz = 8000000; + dev_dbg(&priv->i2c->dev, "%s(): forcing bandwidth to %d\n", + __func__, p->bandwidth_hz); + } + switch (priv->state) { case STATE_SLEEP_TC: ret = cxd2841er_sleep_tc_to_active_c( @@ -2321,7 +3365,8 @@ static int cxd2841er_tune_tc(struct dvb_frontend *fe, struct cxd2841er_priv *priv = fe->demodulator_priv; struct dtv_frontend_properties *p = &fe->dtv_property_cache; - dev_dbg(&priv->i2c->dev, "%s(): re_tune %d\n", __func__, re_tune); + dev_dbg(&priv->i2c->dev, "%s(): re_tune %d bandwidth=%d\n", __func__, + re_tune, p->bandwidth_hz); if (re_tune) { ret = cxd2841er_set_frontend_tc(fe); if (ret) @@ -2329,7 +3374,16 @@ static int cxd2841er_tune_tc(struct dvb_frontend *fe, cxd2841er_read_status_tc(fe, status); if (*status & FE_HAS_LOCK) { switch (priv->system) { + case SYS_ISDBT: + ret = cxd2841er_get_carrier_offset_i( + priv, p->bandwidth_hz, + &carrier_offset); + break; case SYS_DVBT: + ret = cxd2841er_get_carrier_offset_t( + priv, p->bandwidth_hz, + &carrier_offset); + break; case SYS_DVBT2: ret = cxd2841er_get_carrier_offset_t2( priv, p->bandwidth_hz, @@ -2382,6 +3436,9 @@ static int cxd2841er_sleep_tc(struct dvb_frontend *fe) case SYS_DVBT2: cxd2841er_active_t2_to_sleep_tc(priv); break; + case SYS_ISDBT: + cxd2841er_active_i_to_sleep_tc(priv); + break; case SYS_DVBC_ANNEX_A: cxd2841er_active_c_to_sleep_tc(priv); break; @@ -2512,23 +3569,57 @@ static enum dvbfe_algo cxd2841er_get_algo(struct dvb_frontend *fe) return DVBFE_ALGO_HW; } +static void cxd2841er_init_stats(struct dvb_frontend *fe) +{ + struct dtv_frontend_properties *p = &fe->dtv_property_cache; + + p->strength.len = 1; + p->strength.stat[0].scale = FE_SCALE_RELATIVE; + p->cnr.len = 1; + p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->block_error.len = 1; + p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->post_bit_error.len = 1; + p->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + p->post_bit_count.len = 1; + p->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; +} + + static int cxd2841er_init_s(struct dvb_frontend *fe) { struct cxd2841er_priv *priv = fe->demodulator_priv; + /* sanity. force demod to SHUTDOWN state */ + if (priv->state == STATE_SLEEP_S) { + dev_dbg(&priv->i2c->dev, "%s() forcing sleep->shutdown\n", + __func__); + cxd2841er_sleep_s_to_shutdown(priv); + } else if (priv->state == STATE_ACTIVE_S) { + dev_dbg(&priv->i2c->dev, "%s() forcing active->sleep->shutdown\n", + __func__); + cxd2841er_active_s_to_sleep_s(priv); + cxd2841er_sleep_s_to_shutdown(priv); + } + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); cxd2841er_shutdown_to_sleep_s(priv); /* SONY_DEMOD_CONFIG_SAT_IFAGCNEG set to 1 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0xa0); cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xb9, 0x01, 0x01); + + cxd2841er_init_stats(fe); + return 0; } static int cxd2841er_init_tc(struct dvb_frontend *fe) { struct cxd2841er_priv *priv = fe->demodulator_priv; + struct dtv_frontend_properties *p = &fe->dtv_property_cache; - dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + dev_dbg(&priv->i2c->dev, "%s() bandwidth_hz=%d\n", + __func__, p->bandwidth_hz); cxd2841er_shutdown_to_sleep_tc(priv); /* SONY_DEMOD_CONFIG_IFAGCNEG = 1 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10); @@ -2538,12 +3629,14 @@ static int cxd2841er_init_tc(struct dvb_frontend *fe) /* SONY_DEMOD_CONFIG_PARALLEL_SEL = 1 */ cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x00); cxd2841er_set_reg_bits(priv, I2C_SLVT, 0xc4, 0x00, 0x80); + + cxd2841er_init_stats(fe); + return 0; } static struct dvb_frontend_ops cxd2841er_dvbs_s2_ops; -static struct dvb_frontend_ops cxd2841er_dvbt_t2_ops; -static struct dvb_frontend_ops cxd2841er_dvbc_ops; +static struct dvb_frontend_ops cxd2841er_t_c_ops; static struct dvb_frontend *cxd2841er_attach(struct cxd2841er_config *cfg, struct i2c_adapter *i2c, @@ -2551,6 +3644,7 @@ static struct dvb_frontend *cxd2841er_attach(struct cxd2841er_config *cfg, { u8 chip_id = 0; const char *type; + const char *name; struct cxd2841er_priv *priv = NULL; /* allocate memory for the internal state */ @@ -2561,46 +3655,49 @@ static struct dvb_frontend *cxd2841er_attach(struct cxd2841er_config *cfg, priv->config = cfg; priv->i2c_addr_slvx = (cfg->i2c_addr + 4) >> 1; priv->i2c_addr_slvt = (cfg->i2c_addr) >> 1; - /* create dvb_frontend */ - switch (system) { - case SYS_DVBS: - memcpy(&priv->frontend.ops, - &cxd2841er_dvbs_s2_ops, - sizeof(struct dvb_frontend_ops)); - type = "S/S2"; - break; - case SYS_DVBT: - memcpy(&priv->frontend.ops, - &cxd2841er_dvbt_t2_ops, - sizeof(struct dvb_frontend_ops)); - type = "T/T2"; - break; - case SYS_DVBC_ANNEX_A: - memcpy(&priv->frontend.ops, - &cxd2841er_dvbc_ops, - sizeof(struct dvb_frontend_ops)); - type = "C/C2"; - break; - default: - kfree(priv); - return NULL; - } + priv->xtal = cfg->xtal; priv->frontend.demodulator_priv = priv; - dev_info(&priv->i2c->dev, - "%s(): attaching CXD2841ER DVB-%s frontend\n", - __func__, type); dev_info(&priv->i2c->dev, "%s(): I2C adapter %p SLVX addr %x SLVT addr %x\n", __func__, priv->i2c, priv->i2c_addr_slvx, priv->i2c_addr_slvt); chip_id = cxd2841er_chip_id(priv); - if (chip_id != CXD2841ER_CHIP_ID) { + switch (chip_id) { + case CXD2841ER_CHIP_ID: + snprintf(cxd2841er_t_c_ops.info.name, 128, + "Sony CXD2841ER DVB-T/T2/C demodulator"); + name = "CXD2841ER"; + break; + case CXD2854ER_CHIP_ID: + snprintf(cxd2841er_t_c_ops.info.name, 128, + "Sony CXD2854ER DVB-T/T2/C and ISDB-T demodulator"); + cxd2841er_t_c_ops.delsys[3] = SYS_ISDBT; + name = "CXD2854ER"; + break; + default: dev_err(&priv->i2c->dev, "%s(): invalid chip ID 0x%02x\n", - __func__, chip_id); + __func__, chip_id); priv->frontend.demodulator_priv = NULL; kfree(priv); return NULL; } + + /* create dvb_frontend */ + if (system == SYS_DVBS) { + memcpy(&priv->frontend.ops, + &cxd2841er_dvbs_s2_ops, + sizeof(struct dvb_frontend_ops)); + type = "S/S2"; + } else { + memcpy(&priv->frontend.ops, + &cxd2841er_t_c_ops, + sizeof(struct dvb_frontend_ops)); + type = "T/T2/C/ISDB-T"; + } + + dev_info(&priv->i2c->dev, + "%s(): attaching %s DVB-%s frontend\n", + __func__, name, type); dev_info(&priv->i2c->dev, "%s(): chip ID 0x%02x OK.\n", __func__, chip_id); return &priv->frontend; @@ -2613,19 +3710,12 @@ struct dvb_frontend *cxd2841er_attach_s(struct cxd2841er_config *cfg, } EXPORT_SYMBOL(cxd2841er_attach_s); -struct dvb_frontend *cxd2841er_attach_t(struct cxd2841er_config *cfg, - struct i2c_adapter *i2c) -{ - return cxd2841er_attach(cfg, i2c, SYS_DVBT); -} -EXPORT_SYMBOL(cxd2841er_attach_t); - -struct dvb_frontend *cxd2841er_attach_c(struct cxd2841er_config *cfg, +struct dvb_frontend *cxd2841er_attach_t_c(struct cxd2841er_config *cfg, struct i2c_adapter *i2c) { - return cxd2841er_attach(cfg, i2c, SYS_DVBC_ANNEX_A); + return cxd2841er_attach(cfg, i2c, 0); } -EXPORT_SYMBOL(cxd2841er_attach_c); +EXPORT_SYMBOL(cxd2841er_attach_t_c); static struct dvb_frontend_ops cxd2841er_dvbs_s2_ops = { .delsys = { SYS_DVBS, SYS_DVBS2 }, @@ -2655,10 +3745,10 @@ static struct dvb_frontend_ops cxd2841er_dvbs_s2_ops = { .tune = cxd2841er_tune_s }; -static struct dvb_frontend_ops cxd2841er_dvbt_t2_ops = { - .delsys = { SYS_DVBT, SYS_DVBT2 }, +static struct dvb_frontend_ops cxd2841er_t_c_ops = { + .delsys = { SYS_DVBT, SYS_DVBT2, SYS_DVBC_ANNEX_A }, .info = { - .name = "Sony CXD2841ER DVB-T/T2 demodulator", + .name = "", /* will set in attach function */ .caps = FE_CAN_FEC_1_2 | FE_CAN_FEC_2_3 | FE_CAN_FEC_3_4 | @@ -2691,37 +3781,6 @@ static struct dvb_frontend_ops cxd2841er_dvbt_t2_ops = { .get_frontend_algo = cxd2841er_get_algo }; -static struct dvb_frontend_ops cxd2841er_dvbc_ops = { - .delsys = { SYS_DVBC_ANNEX_A }, - .info = { - .name = "Sony CXD2841ER DVB-C demodulator", - .caps = FE_CAN_FEC_1_2 | - FE_CAN_FEC_2_3 | - FE_CAN_FEC_3_4 | - FE_CAN_FEC_5_6 | - FE_CAN_FEC_7_8 | - FE_CAN_FEC_AUTO | - FE_CAN_QAM_16 | - FE_CAN_QAM_32 | - FE_CAN_QAM_64 | - FE_CAN_QAM_128 | - FE_CAN_QAM_256 | - FE_CAN_QAM_AUTO | - FE_CAN_INVERSION_AUTO, - .frequency_min = 42000000, - .frequency_max = 1002000000 - }, - .init = cxd2841er_init_tc, - .sleep = cxd2841er_sleep_tc, - .release = cxd2841er_release, - .set_frontend = cxd2841er_set_frontend_tc, - .get_frontend = cxd2841er_get_frontend, - .read_status = cxd2841er_read_status_tc, - .tune = cxd2841er_tune_tc, - .i2c_gate_ctrl = cxd2841er_i2c_gate_ctrl, - .get_frontend_algo = cxd2841er_get_algo, -}; - -MODULE_DESCRIPTION("Sony CXD2841ER DVB-C/C2/T/T2/S/S2 demodulator driver"); -MODULE_AUTHOR("Sergey Kozlov "); +MODULE_DESCRIPTION("Sony CXD2841ER/CXD2854ER DVB-C/C2/T/T2/S/S2 demodulator driver"); +MODULE_AUTHOR("Sergey Kozlov , Abylay Ospan "); MODULE_LICENSE("GPL"); diff --git a/drivers/media/dvb-frontends/cxd2841er.h b/drivers/media/dvb-frontends/cxd2841er.h index 3472bdd58949..62ad5f07390b 100644 --- a/drivers/media/dvb-frontends/cxd2841er.h +++ b/drivers/media/dvb-frontends/cxd2841er.h @@ -25,41 +25,39 @@ #include #include +enum cxd2841er_xtal { + SONY_XTAL_20500, /* 20.5 MHz */ + SONY_XTAL_24000, /* 24 MHz */ + SONY_XTAL_41000 /* 41 MHz */ +}; + struct cxd2841er_config { u8 i2c_addr; + enum cxd2841er_xtal xtal; }; #if IS_REACHABLE(CONFIG_DVB_CXD2841ER) extern struct dvb_frontend *cxd2841er_attach_s(struct cxd2841er_config *cfg, struct i2c_adapter *i2c); -extern struct dvb_frontend *cxd2841er_attach_t(struct cxd2841er_config *cfg, - struct i2c_adapter *i2c); - -extern struct dvb_frontend *cxd2841er_attach_c(struct cxd2841er_config *cfg, +extern struct dvb_frontend *cxd2841er_attach_t_c(struct cxd2841er_config *cfg, struct i2c_adapter *i2c); #else static inline struct dvb_frontend *cxd2841er_attach_s( struct cxd2841er_config *cfg, struct i2c_adapter *i2c) { - printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__); + pr_warn("%s: driver disabled by Kconfig\n", __func__); return NULL; } -static inline struct dvb_frontend *cxd2841er_attach_t( +static inline struct dvb_frontend *cxd2841er_attach_t_c( struct cxd2841er_config *cfg, struct i2c_adapter *i2c) { - printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__); + pr_warn("%s: driver disabled by Kconfig\n", __func__); return NULL; } -static inline struct dvb_frontend *cxd2841er_attach_c( - struct cxd2841er_config *cfg, struct i2c_adapter *i2c) -{ - printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__); - return NULL; -} #endif #endif diff --git a/drivers/media/dvb-frontends/cxd2841er_priv.h b/drivers/media/dvb-frontends/cxd2841er_priv.h index 33e2f495277b..0bbce451149f 100644 --- a/drivers/media/dvb-frontends/cxd2841er_priv.h +++ b/drivers/media/dvb-frontends/cxd2841er_priv.h @@ -26,6 +26,7 @@ #define I2C_SLVT 1 #define CXD2841ER_CHIP_ID 0xa7 +#define CXD2854ER_CHIP_ID 0xc1 #define CXD2841ER_DVBS_POLLING_INVL 10 diff --git a/drivers/media/dvb-frontends/dib0090.c b/drivers/media/dvb-frontends/dib0090.c index d879dc0607f4..14c403254fe0 100644 --- a/drivers/media/dvb-frontends/dib0090.c +++ b/drivers/media/dvb-frontends/dib0090.c @@ -797,6 +797,8 @@ static const u16 bb_ramp_pwm_normal[] = { (0 << 9) | 400, /* BB_RAMP6 */ }; +#if 0 +/* Currently unused */ static const u16 bb_ramp_pwm_boost[] = { 550, /* max BB gain in 10th of dB */ 8, /* ramp_slope = 1dB of gain -> clock_ticks_per_db = clk_khz / ramp_slope -> BB_RAMP2 */ @@ -806,6 +808,7 @@ static const u16 bb_ramp_pwm_boost[] = { (2 << 9) | 208, /* BB_RAMP5 = 29dB */ (0 << 9) | 440, /* BB_RAMP6 */ }; +#endif static const u16 rf_ramp_pwm_cband[] = { 314, /* max RF gain in 10th of dB */ @@ -849,6 +852,8 @@ static const u16 rf_ramp_pwm_uhf[] = { (0 << 10) | 580, /* GAIN_4_2, LNA 4 */ }; +#if 0 +/* Currently unused */ static const u16 rf_ramp_pwm_sband[] = { 253, /* max RF gain in 10th of dB */ 38, /* ramp_slope = 1dB of gain -> clock_ticks_per_db = clk_khz / ramp_slope -> RF_RAMP2 */ @@ -862,6 +867,7 @@ static const u16 rf_ramp_pwm_sband[] = { (0 << 10) | 0, /* GAIN_4_1, LNA 4 = 0dB */ (0 << 10) | 0, /* GAIN_4_2, LNA 4 */ }; +#endif struct slope { s16 range; diff --git a/drivers/media/dvb-frontends/drx39xyj/drxj.c b/drivers/media/dvb-frontends/drx39xyj/drxj.c index e48b741d439e..bd6d2ee0f7c9 100644 --- a/drivers/media/dvb-frontends/drx39xyj/drxj.c +++ b/drivers/media/dvb-frontends/drx39xyj/drxj.c @@ -1240,12 +1240,15 @@ static u32 frac_times1e6(u32 N, u32 D) * and rounded. For calc used formula: 16*10^(prescaleGain[dB]/20). * */ +#if 0 +/* Currently, unused as we lack support for analog TV */ static const u16 nicam_presc_table_val[43] = { 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10, 11, 13, 14, 16, 18, 20, 23, 25, 28, 32, 36, 40, 45, 51, 57, 64, 71, 80, 90, 101, 113, 127 }; +#endif /*============================================================================*/ /*== END HELPER FUNCTIONS ==*/ diff --git a/drivers/media/dvb-frontends/ds3000.c b/drivers/media/dvb-frontends/ds3000.c index addffc33993a..447b518e287a 100644 --- a/drivers/media/dvb-frontends/ds3000.c +++ b/drivers/media/dvb-frontends/ds3000.c @@ -959,6 +959,15 @@ static int ds3000_set_frontend(struct dvb_frontend *fe) /* enable ac coupling */ ds3000_writereg(state, 0x25, 0x8a); + if ((c->symbol_rate < ds3000_ops.info.symbol_rate_min) || + (c->symbol_rate > ds3000_ops.info.symbol_rate_max)) { + dprintk("%s() symbol_rate %u out of range (%u ... %u)\n", + __func__, c->symbol_rate, + ds3000_ops.info.symbol_rate_min, + ds3000_ops.info.symbol_rate_max); + return -EINVAL; + } + /* enhance symbol rate performance */ if ((c->symbol_rate / 1000) <= 5000) { value = 29777 / (c->symbol_rate / 1000) + 1; diff --git a/drivers/media/dvb-frontends/helene.c b/drivers/media/dvb-frontends/helene.c new file mode 100644 index 000000000000..97a8982740a6 --- /dev/null +++ b/drivers/media/dvb-frontends/helene.c @@ -0,0 +1,1042 @@ +/* + * helene.c + * + * Sony HELENE DVB-S/S2 DVB-T/T2 DVB-C/C2 ISDB-T/S tuner driver (CXD2858ER) + * + * Copyright 2012 Sony Corporation + * Copyright (C) 2014 NetUP Inc. + * Copyright (C) 2014 Abylay Ospan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include "helene.h" +#include "dvb_frontend.h" + +#define MAX_WRITE_REGSIZE 20 + +enum helene_state { + STATE_UNKNOWN, + STATE_SLEEP, + STATE_ACTIVE +}; + +struct helene_priv { + u32 frequency; + u8 i2c_address; + struct i2c_adapter *i2c; + enum helene_state state; + void *set_tuner_data; + int (*set_tuner)(void *, int); + enum helene_xtal xtal; +}; + +#define TERR_INTERNAL_LOOPFILTER_AVAILABLE(tv_system) \ + (((tv_system) != SONY_HELENE_DTV_DVBC_6) && \ + ((tv_system) != SONY_HELENE_DTV_DVBC_8)\ + && ((tv_system) != SONY_HELENE_DTV_DVBC2_6) && \ + ((tv_system) != SONY_HELENE_DTV_DVBC2_8)) + +#define HELENE_AUTO 0xff +#define HELENE_OFFSET(ofs) ((u8)(ofs) & 0x1F) +#define HELENE_BW_6 0x00 +#define HELENE_BW_7 0x01 +#define HELENE_BW_8 0x02 +#define HELENE_BW_1_7 0x03 + +enum helene_tv_system_t { + SONY_HELENE_TV_SYSTEM_UNKNOWN, + /* Terrestrial Analog */ + SONY_HELENE_ATV_MN_EIAJ, + /**< System-M (Japan) (IF: Fp=5.75MHz in default) */ + SONY_HELENE_ATV_MN_SAP, + /**< System-M (US) (IF: Fp=5.75MHz in default) */ + SONY_HELENE_ATV_MN_A2, + /**< System-M (Korea) (IF: Fp=5.9MHz in default) */ + SONY_HELENE_ATV_BG, + /**< System-B/G (IF: Fp=7.3MHz in default) */ + SONY_HELENE_ATV_I, + /**< System-I (IF: Fp=7.85MHz in default) */ + SONY_HELENE_ATV_DK, + /**< System-D/K (IF: Fp=7.85MHz in default) */ + SONY_HELENE_ATV_L, + /**< System-L (IF: Fp=7.85MHz in default) */ + SONY_HELENE_ATV_L_DASH, + /**< System-L DASH (IF: Fp=2.2MHz in default) */ + /* Terrestrial/Cable Digital */ + SONY_HELENE_DTV_8VSB, + /**< ATSC 8VSB (IF: Fc=3.7MHz in default) */ + SONY_HELENE_DTV_QAM, + /**< US QAM (IF: Fc=3.7MHz in default) */ + SONY_HELENE_DTV_ISDBT_6, + /**< ISDB-T 6MHzBW (IF: Fc=3.55MHz in default) */ + SONY_HELENE_DTV_ISDBT_7, + /**< ISDB-T 7MHzBW (IF: Fc=4.15MHz in default) */ + SONY_HELENE_DTV_ISDBT_8, + /**< ISDB-T 8MHzBW (IF: Fc=4.75MHz in default) */ + SONY_HELENE_DTV_DVBT_5, + /**< DVB-T 5MHzBW (IF: Fc=3.6MHz in default) */ + SONY_HELENE_DTV_DVBT_6, + /**< DVB-T 6MHzBW (IF: Fc=3.6MHz in default) */ + SONY_HELENE_DTV_DVBT_7, + /**< DVB-T 7MHzBW (IF: Fc=4.2MHz in default) */ + SONY_HELENE_DTV_DVBT_8, + /**< DVB-T 8MHzBW (IF: Fc=4.8MHz in default) */ + SONY_HELENE_DTV_DVBT2_1_7, + /**< DVB-T2 1.7MHzBW (IF: Fc=3.5MHz in default) */ + SONY_HELENE_DTV_DVBT2_5, + /**< DVB-T2 5MHzBW (IF: Fc=3.6MHz in default) */ + SONY_HELENE_DTV_DVBT2_6, + /**< DVB-T2 6MHzBW (IF: Fc=3.6MHz in default) */ + SONY_HELENE_DTV_DVBT2_7, + /**< DVB-T2 7MHzBW (IF: Fc=4.2MHz in default) */ + SONY_HELENE_DTV_DVBT2_8, + /**< DVB-T2 8MHzBW (IF: Fc=4.8MHz in default) */ + SONY_HELENE_DTV_DVBC_6, + /**< DVB-C 6MHzBW (IF: Fc=3.7MHz in default) */ + SONY_HELENE_DTV_DVBC_8, + /**< DVB-C 8MHzBW (IF: Fc=4.9MHz in default) */ + SONY_HELENE_DTV_DVBC2_6, + /**< DVB-C2 6MHzBW (IF: Fc=3.7MHz in default) */ + SONY_HELENE_DTV_DVBC2_8, + /**< DVB-C2 8MHzBW (IF: Fc=4.9MHz in default) */ + SONY_HELENE_DTV_DTMB, + /**< DTMB (IF: Fc=5.1MHz in default) */ + /* Satellite */ + SONY_HELENE_STV_ISDBS, + /**< ISDB-S */ + SONY_HELENE_STV_DVBS, + /**< DVB-S */ + SONY_HELENE_STV_DVBS2, + /**< DVB-S2 */ + + SONY_HELENE_ATV_MIN = SONY_HELENE_ATV_MN_EIAJ, + /**< Minimum analog terrestrial system */ + SONY_HELENE_ATV_MAX = SONY_HELENE_ATV_L_DASH, + /**< Maximum analog terrestrial system */ + SONY_HELENE_DTV_MIN = SONY_HELENE_DTV_8VSB, + /**< Minimum digital terrestrial system */ + SONY_HELENE_DTV_MAX = SONY_HELENE_DTV_DTMB, + /**< Maximum digital terrestrial system */ + SONY_HELENE_TERR_TV_SYSTEM_NUM, + /**< Number of supported terrestrial broadcasting system */ + SONY_HELENE_STV_MIN = SONY_HELENE_STV_ISDBS, + /**< Minimum satellite system */ + SONY_HELENE_STV_MAX = SONY_HELENE_STV_DVBS2 + /**< Maximum satellite system */ +}; + +struct helene_terr_adjust_param_t { + /* < Addr:0x69 Bit[6:4] : RFVGA gain. + * 0xFF means Auto. (RF_GAIN_SEL = 1) + */ + uint8_t RF_GAIN; + /* < Addr:0x69 Bit[3:0] : IF_BPF gain. + */ + uint8_t IF_BPF_GC; + /* < Addr:0x6B Bit[3:0] : RF overload + * RF input detect level. (FRF <= 172MHz) + */ + uint8_t RFOVLD_DET_LV1_VL; + /* < Addr:0x6B Bit[3:0] : RF overload + * RF input detect level. (172MHz < FRF <= 464MHz) + */ + uint8_t RFOVLD_DET_LV1_VH; + /* < Addr:0x6B Bit[3:0] : RF overload + * RF input detect level. (FRF > 464MHz) + */ + uint8_t RFOVLD_DET_LV1_U; + /* < Addr:0x6C Bit[2:0] : + * Internal RFAGC detect level. (FRF <= 172MHz) + */ + uint8_t IFOVLD_DET_LV_VL; + /* < Addr:0x6C Bit[2:0] : + * Internal RFAGC detect level. (172MHz < FRF <= 464MHz) + */ + uint8_t IFOVLD_DET_LV_VH; + /* < Addr:0x6C Bit[2:0] : + * Internal RFAGC detect level. (FRF > 464MHz) + */ + uint8_t IFOVLD_DET_LV_U; + /* < Addr:0x6D Bit[5:4] : + * IF filter center offset. + */ + uint8_t IF_BPF_F0; + /* < Addr:0x6D Bit[1:0] : + * 6MHzBW(0x00) or 7MHzBW(0x01) + * or 8MHzBW(0x02) or 1.7MHzBW(0x03) + */ + uint8_t BW; + /* < Addr:0x6E Bit[4:0] : + * 5bit signed. IF offset (kHz) = FIF_OFFSET x 50 + */ + uint8_t FIF_OFFSET; + /* < Addr:0x6F Bit[4:0] : + * 5bit signed. BW offset (kHz) = + * BW_OFFSET x 50 (BW_OFFSET x 10 in 1.7MHzBW) + */ + uint8_t BW_OFFSET; + /* < Addr:0x9C Bit[0] : + * Local polarity. (0: Upper Local, 1: Lower Local) + */ + uint8_t IS_LOWERLOCAL; +}; + +static const struct helene_terr_adjust_param_t +terr_params[SONY_HELENE_TERR_TV_SYSTEM_NUM] = { + /*< SONY_HELENE_TV_SYSTEM_UNKNOWN */ + {HELENE_AUTO, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + HELENE_BW_6, HELENE_OFFSET(0), HELENE_OFFSET(0), 0x00}, + /* Analog */ + /**< SONY_HELENE_ATV_MN_EIAJ (System-M (Japan)) */ + {HELENE_AUTO, 0x05, 0x03, 0x06, 0x03, 0x01, 0x01, 0x01, 0x00, + HELENE_BW_6, HELENE_OFFSET(0), HELENE_OFFSET(1), 0x00}, + /**< SONY_HELENE_ATV_MN_SAP (System-M (US)) */ + {HELENE_AUTO, 0x05, 0x03, 0x06, 0x03, 0x01, 0x01, 0x01, 0x00, + HELENE_BW_6, HELENE_OFFSET(0), HELENE_OFFSET(1), 0x00}, + {HELENE_AUTO, 0x05, 0x03, 0x06, 0x03, 0x01, 0x01, 0x01, 0x00, + HELENE_BW_6, HELENE_OFFSET(3), HELENE_OFFSET(1), 0x00}, + /**< SONY_HELENE_ATV_MN_A2 (System-M (Korea)) */ + {HELENE_AUTO, 0x05, 0x03, 0x06, 0x03, 0x01, 0x01, 0x01, 0x00, + HELENE_BW_7, HELENE_OFFSET(11), HELENE_OFFSET(5), 0x00}, + /**< SONY_HELENE_ATV_BG (System-B/G) */ + {HELENE_AUTO, 0x05, 0x03, 0x06, 0x03, 0x01, 0x01, 0x01, 0x00, + HELENE_BW_8, HELENE_OFFSET(2), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_ATV_I (System-I) */ + {HELENE_AUTO, 0x05, 0x03, 0x06, 0x03, 0x01, 0x01, 0x01, 0x00, + HELENE_BW_8, HELENE_OFFSET(2), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_ATV_DK (System-D/K) */ + {HELENE_AUTO, 0x03, 0x04, 0x0A, 0x04, 0x04, 0x04, 0x04, 0x00, + HELENE_BW_8, HELENE_OFFSET(2), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_ATV_L (System-L) */ + {HELENE_AUTO, 0x03, 0x04, 0x0A, 0x04, 0x04, 0x04, 0x04, 0x00, + HELENE_BW_8, HELENE_OFFSET(-1), HELENE_OFFSET(4), 0x00}, + /**< SONY_HELENE_ATV_L_DASH (System-L DASH) */ + /* Digital */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x03, 0x03, 0x03, 0x00, + HELENE_BW_6, HELENE_OFFSET(-6), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_DTV_8VSB (ATSC 8VSB) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_6, HELENE_OFFSET(-6), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_DTV_QAM (US QAM) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_6, HELENE_OFFSET(-9), HELENE_OFFSET(-5), 0x00}, + /**< SONY_HELENE_DTV_ISDBT_6 (ISDB-T 6MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_7, HELENE_OFFSET(-7), HELENE_OFFSET(-6), 0x00}, + /**< SONY_HELENE_DTV_ISDBT_7 (ISDB-T 7MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_8, HELENE_OFFSET(-5), HELENE_OFFSET(-7), 0x00}, + /**< SONY_HELENE_DTV_ISDBT_8 (ISDB-T 8MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_6, HELENE_OFFSET(-8), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_DTV_DVBT_5 (DVB-T 5MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_6, HELENE_OFFSET(-8), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_DTV_DVBT_6 (DVB-T 6MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_7, HELENE_OFFSET(-6), HELENE_OFFSET(-5), 0x00}, + /**< SONY_HELENE_DTV_DVBT_7 (DVB-T 7MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_8, HELENE_OFFSET(-4), HELENE_OFFSET(-6), 0x00}, + /**< SONY_HELENE_DTV_DVBT_8 (DVB-T 8MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_1_7, HELENE_OFFSET(-10), HELENE_OFFSET(-10), 0x00}, + /**< SONY_HELENE_DTV_DVBT2_1_7 (DVB-T2 1.7MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_6, HELENE_OFFSET(-8), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_DTV_DVBT2_5 (DVB-T2 5MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_6, HELENE_OFFSET(-8), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_DTV_DVBT2_6 (DVB-T2 6MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_7, HELENE_OFFSET(-6), HELENE_OFFSET(-5), 0x00}, + /**< SONY_HELENE_DTV_DVBT2_7 (DVB-T2 7MHzBW) */ + {HELENE_AUTO, 0x09, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_8, HELENE_OFFSET(-4), HELENE_OFFSET(-6), 0x00}, + /**< SONY_HELENE_DTV_DVBT2_8 (DVB-T2 8MHzBW) */ + {HELENE_AUTO, 0x05, 0x02, 0x02, 0x02, 0x01, 0x01, 0x01, 0x00, + HELENE_BW_6, HELENE_OFFSET(-6), HELENE_OFFSET(-4), 0x00}, + /**< SONY_HELENE_DTV_DVBC_6 (DVB-C 6MHzBW) */ + {HELENE_AUTO, 0x05, 0x02, 0x02, 0x02, 0x01, 0x01, 0x01, 0x00, + HELENE_BW_8, HELENE_OFFSET(-2), HELENE_OFFSET(-3), 0x00}, + /**< SONY_HELENE_DTV_DVBC_8 (DVB-C 8MHzBW) */ + {HELENE_AUTO, 0x03, 0x09, 0x09, 0x09, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_6, HELENE_OFFSET(-6), HELENE_OFFSET(-2), 0x00}, + /**< SONY_HELENE_DTV_DVBC2_6 (DVB-C2 6MHzBW) */ + {HELENE_AUTO, 0x03, 0x09, 0x09, 0x09, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_8, HELENE_OFFSET(-2), HELENE_OFFSET(0), 0x00}, + /**< SONY_HELENE_DTV_DVBC2_8 (DVB-C2 8MHzBW) */ + {HELENE_AUTO, 0x04, 0x0B, 0x0B, 0x0B, 0x02, 0x02, 0x02, 0x00, + HELENE_BW_8, HELENE_OFFSET(2), HELENE_OFFSET(1), 0x00} + /**< SONY_HELENE_DTV_DTMB (DTMB) */ +}; + +static void helene_i2c_debug(struct helene_priv *priv, + u8 reg, u8 write, const u8 *data, u32 len) +{ + dev_dbg(&priv->i2c->dev, "helene: I2C %s reg 0x%02x size %d\n", + (write == 0 ? "read" : "write"), reg, len); + print_hex_dump_bytes("helene: I2C data: ", + DUMP_PREFIX_OFFSET, data, len); +} + +static int helene_write_regs(struct helene_priv *priv, + u8 reg, const u8 *data, u32 len) +{ + int ret; + u8 buf[MAX_WRITE_REGSIZE + 1]; + struct i2c_msg msg[1] = { + { + .addr = priv->i2c_address, + .flags = 0, + .len = len + 1, + .buf = buf, + } + }; + + if (len + 1 > sizeof(buf)) { + dev_warn(&priv->i2c->dev, + "wr reg=%04x: len=%d vs %Zu is too big!\n", + reg, len + 1, sizeof(buf)); + return -E2BIG; + } + + helene_i2c_debug(priv, reg, 1, data, len); + buf[0] = reg; + memcpy(&buf[1], data, len); + ret = i2c_transfer(priv->i2c, msg, 1); + if (ret >= 0 && ret != 1) + ret = -EREMOTEIO; + if (ret < 0) { + dev_warn(&priv->i2c->dev, + "%s: i2c wr failed=%d reg=%02x len=%d\n", + KBUILD_MODNAME, ret, reg, len); + return ret; + } + return 0; +} + +static int helene_write_reg(struct helene_priv *priv, u8 reg, u8 val) +{ + return helene_write_regs(priv, reg, &val, 1); +} + +static int helene_read_regs(struct helene_priv *priv, + u8 reg, u8 *val, u32 len) +{ + int ret; + struct i2c_msg msg[2] = { + { + .addr = priv->i2c_address, + .flags = 0, + .len = 1, + .buf = ®, + }, { + .addr = priv->i2c_address, + .flags = I2C_M_RD, + .len = len, + .buf = val, + } + }; + + ret = i2c_transfer(priv->i2c, &msg[0], 1); + if (ret >= 0 && ret != 1) + ret = -EREMOTEIO; + if (ret < 0) { + dev_warn(&priv->i2c->dev, + "%s: I2C rw failed=%d addr=%02x reg=%02x\n", + KBUILD_MODNAME, ret, priv->i2c_address, reg); + return ret; + } + ret = i2c_transfer(priv->i2c, &msg[1], 1); + if (ret >= 0 && ret != 1) + ret = -EREMOTEIO; + if (ret < 0) { + dev_warn(&priv->i2c->dev, + "%s: i2c rd failed=%d addr=%02x reg=%02x\n", + KBUILD_MODNAME, ret, priv->i2c_address, reg); + return ret; + } + helene_i2c_debug(priv, reg, 0, val, len); + return 0; +} + +static int helene_read_reg(struct helene_priv *priv, u8 reg, u8 *val) +{ + return helene_read_regs(priv, reg, val, 1); +} + +static int helene_set_reg_bits(struct helene_priv *priv, + u8 reg, u8 data, u8 mask) +{ + int res; + u8 rdata; + + if (mask != 0xff) { + res = helene_read_reg(priv, reg, &rdata); + if (res != 0) + return res; + data = ((data & mask) | (rdata & (mask ^ 0xFF))); + } + return helene_write_reg(priv, reg, data); +} + +static int helene_enter_power_save(struct helene_priv *priv) +{ + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + if (priv->state == STATE_SLEEP) + return 0; + + /* Standby setting for CPU */ + helene_write_reg(priv, 0x88, 0x0); + + /* Standby setting for internal logic block */ + helene_write_reg(priv, 0x87, 0xC0); + + priv->state = STATE_SLEEP; + return 0; +} + +static int helene_leave_power_save(struct helene_priv *priv) +{ + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + if (priv->state == STATE_ACTIVE) + return 0; + + /* Standby setting for internal logic block */ + helene_write_reg(priv, 0x87, 0xC4); + + /* Standby setting for CPU */ + helene_write_reg(priv, 0x88, 0x40); + + priv->state = STATE_ACTIVE; + return 0; +} + +static int helene_init(struct dvb_frontend *fe) +{ + struct helene_priv *priv = fe->tuner_priv; + + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + return helene_leave_power_save(priv); +} + +static int helene_release(struct dvb_frontend *fe) +{ + struct helene_priv *priv = fe->tuner_priv; + + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + kfree(fe->tuner_priv); + fe->tuner_priv = NULL; + return 0; +} + +static int helene_sleep(struct dvb_frontend *fe) +{ + struct helene_priv *priv = fe->tuner_priv; + + dev_dbg(&priv->i2c->dev, "%s()\n", __func__); + helene_enter_power_save(priv); + return 0; +} + +static enum helene_tv_system_t helene_get_tv_system(struct dvb_frontend *fe) +{ + enum helene_tv_system_t system = SONY_HELENE_TV_SYSTEM_UNKNOWN; + struct dtv_frontend_properties *p = &fe->dtv_property_cache; + struct helene_priv *priv = fe->tuner_priv; + + if (p->delivery_system == SYS_DVBT) { + if (p->bandwidth_hz <= 5000000) + system = SONY_HELENE_DTV_DVBT_5; + else if (p->bandwidth_hz <= 6000000) + system = SONY_HELENE_DTV_DVBT_6; + else if (p->bandwidth_hz <= 7000000) + system = SONY_HELENE_DTV_DVBT_7; + else if (p->bandwidth_hz <= 8000000) + system = SONY_HELENE_DTV_DVBT_8; + else { + system = SONY_HELENE_DTV_DVBT_8; + p->bandwidth_hz = 8000000; + } + } else if (p->delivery_system == SYS_DVBT2) { + if (p->bandwidth_hz <= 5000000) + system = SONY_HELENE_DTV_DVBT2_5; + else if (p->bandwidth_hz <= 6000000) + system = SONY_HELENE_DTV_DVBT2_6; + else if (p->bandwidth_hz <= 7000000) + system = SONY_HELENE_DTV_DVBT2_7; + else if (p->bandwidth_hz <= 8000000) + system = SONY_HELENE_DTV_DVBT2_8; + else { + system = SONY_HELENE_DTV_DVBT2_8; + p->bandwidth_hz = 8000000; + } + } else if (p->delivery_system == SYS_DVBS) { + system = SONY_HELENE_STV_DVBS; + } else if (p->delivery_system == SYS_DVBS2) { + system = SONY_HELENE_STV_DVBS2; + } else if (p->delivery_system == SYS_ISDBS) { + system = SONY_HELENE_STV_ISDBS; + } else if (p->delivery_system == SYS_ISDBT) { + if (p->bandwidth_hz <= 6000000) + system = SONY_HELENE_DTV_ISDBT_6; + else if (p->bandwidth_hz <= 7000000) + system = SONY_HELENE_DTV_ISDBT_7; + else if (p->bandwidth_hz <= 8000000) + system = SONY_HELENE_DTV_ISDBT_8; + else { + system = SONY_HELENE_DTV_ISDBT_8; + p->bandwidth_hz = 8000000; + } + } else if (p->delivery_system == SYS_DVBC_ANNEX_A) { + if (p->bandwidth_hz <= 6000000) + system = SONY_HELENE_DTV_DVBC_6; + else if (p->bandwidth_hz <= 8000000) + system = SONY_HELENE_DTV_DVBC_8; + } + dev_dbg(&priv->i2c->dev, + "%s(): HELENE DTV system %d (delsys %d, bandwidth %d)\n", + __func__, (int)system, p->delivery_system, + p->bandwidth_hz); + return system; +} + +static int helene_set_params_s(struct dvb_frontend *fe) +{ + u8 data[MAX_WRITE_REGSIZE]; + u32 frequency; + enum helene_tv_system_t tv_system; + struct dtv_frontend_properties *p = &fe->dtv_property_cache; + struct helene_priv *priv = fe->tuner_priv; + int frequencykHz = p->frequency; + uint32_t frequency4kHz = 0; + u32 symbol_rate = p->symbol_rate/1000; + + dev_dbg(&priv->i2c->dev, "%s(): tune frequency %dkHz sr=%uKsps\n", + __func__, frequencykHz, symbol_rate); + tv_system = helene_get_tv_system(fe); + + if (tv_system == SONY_HELENE_TV_SYSTEM_UNKNOWN) { + dev_err(&priv->i2c->dev, "%s(): unknown DTV system\n", + __func__); + return -EINVAL; + } + /* RF switch turn to satellite */ + if (priv->set_tuner) + priv->set_tuner(priv->set_tuner_data, 0); + frequency = roundup(p->frequency / 1000, 1); + + /* Disable IF signal output */ + helene_write_reg(priv, 0x15, 0x02); + + /* RFIN matching in power save (Sat) reset */ + helene_write_reg(priv, 0x43, 0x06); + + /* Analog block setting (0x6A, 0x6B) */ + data[0] = 0x00; + data[1] = 0x00; + helene_write_regs(priv, 0x6A, data, 2); + helene_write_reg(priv, 0x75, 0x99); + helene_write_reg(priv, 0x9D, 0x00); + + /* Tuning setting for CPU (0x61) */ + helene_write_reg(priv, 0x61, 0x07); + + /* Satellite mode select (0x01) */ + helene_write_reg(priv, 0x01, 0x01); + + /* Clock enable for internal logic block, CPU wake-up (0x04, 0x05) */ + data[0] = 0xC4; + data[1] = 0x40; + + switch (priv->xtal) { + case SONY_HELENE_XTAL_16000: + data[2] = 0x02; + break; + case SONY_HELENE_XTAL_20500: + data[2] = 0x02; + break; + case SONY_HELENE_XTAL_24000: + data[2] = 0x03; + break; + case SONY_HELENE_XTAL_41000: + data[2] = 0x05; + break; + default: + dev_err(&priv->i2c->dev, "%s(): unknown xtal %d\n", + __func__, priv->xtal); + return -EINVAL; + } + + /* Setting for analog block (0x07). LOOPFILTER INTERNAL */ + data[3] = 0x80; + + /* Tuning setting for analog block + * (0x08, 0x09, 0x0A, 0x0B). LOOPFILTER INTERNAL + */ + if (priv->xtal == SONY_HELENE_XTAL_20500) + data[4] = 0x58; + else + data[4] = 0x70; + + data[5] = 0x1E; + data[6] = 0x02; + data[7] = 0x24; + + /* Enable for analog block (0x0C, 0x0D, 0x0E). SAT LNA ON */ + data[8] = 0x0F; + data[8] |= 0xE0; /* POWERSAVE_TERR_RF_ACTIVE */ + data[9] = 0x02; + data[10] = 0x1E; + + /* Setting for LPF cutoff frequency (0x0F) */ + switch (tv_system) { + case SONY_HELENE_STV_ISDBS: + data[11] = 0x22; /* 22MHz */ + break; + case SONY_HELENE_STV_DVBS: + if (symbol_rate <= 4000) + data[11] = 0x05; + else if (symbol_rate <= 10000) + data[11] = (uint8_t)((symbol_rate * 47 + + (40000-1)) / 40000); + else + data[11] = (uint8_t)((symbol_rate * 27 + + (40000-1)) / 40000 + 5); + + if (data[11] > 36) + data[11] = 36; /* 5 <= lpf_cutoff <= 36 is valid */ + break; + case SONY_HELENE_STV_DVBS2: + if (symbol_rate <= 4000) + data[11] = 0x05; + else if (symbol_rate <= 10000) + data[11] = (uint8_t)((symbol_rate * 11 + + (10000-1)) / 10000); + else + data[11] = (uint8_t)((symbol_rate * 3 + + (5000-1)) / 5000 + 5); + + if (data[11] > 36) + data[11] = 36; /* 5 <= lpf_cutoff <= 36 is valid */ + break; + default: + dev_err(&priv->i2c->dev, "%s(): unknown standard %d\n", + __func__, tv_system); + return -EINVAL; + } + + /* RF tuning frequency setting (0x10, 0x11, 0x12) */ + frequency4kHz = (frequencykHz + 2) / 4; + data[12] = (uint8_t)(frequency4kHz & 0xFF); /* FRF_L */ + data[13] = (uint8_t)((frequency4kHz >> 8) & 0xFF); /* FRF_M */ + /* FRF_H (bit[3:0]) */ + data[14] = (uint8_t)((frequency4kHz >> 16) & 0x0F); + + /* Tuning command (0x13) */ + data[15] = 0xFF; + + /* Setting for IQOUT_LIMIT (0x14) 0.75Vpp */ + data[16] = 0x00; + + /* Enable IQ output (0x15) */ + data[17] = 0x01; + + helene_write_regs(priv, 0x04, data, 18); + + dev_dbg(&priv->i2c->dev, "%s(): tune done\n", + __func__); + + priv->frequency = frequency; + return 0; +} + +static int helene_set_params(struct dvb_frontend *fe) +{ + u8 data[MAX_WRITE_REGSIZE]; + u32 frequency; + enum helene_tv_system_t tv_system; + struct dtv_frontend_properties *p = &fe->dtv_property_cache; + struct helene_priv *priv = fe->tuner_priv; + int frequencykHz = p->frequency / 1000; + + dev_dbg(&priv->i2c->dev, "%s(): tune frequency %dkHz\n", + __func__, frequencykHz); + tv_system = helene_get_tv_system(fe); + + if (tv_system == SONY_HELENE_TV_SYSTEM_UNKNOWN) { + dev_dbg(&priv->i2c->dev, "%s(): unknown DTV system\n", + __func__); + return -EINVAL; + } + if (priv->set_tuner) + priv->set_tuner(priv->set_tuner_data, 1); + frequency = roundup(p->frequency / 1000, 25); + + /* mode select */ + helene_write_reg(priv, 0x01, 0x00); + + /* Disable IF signal output */ + helene_write_reg(priv, 0x74, 0x02); + + if (priv->state == STATE_SLEEP) + helene_leave_power_save(priv); + + /* Initial setting for internal analog block (0x91, 0x92) */ + if ((tv_system == SONY_HELENE_DTV_DVBC_6) || + (tv_system == SONY_HELENE_DTV_DVBC_8)) { + data[0] = 0x16; + data[1] = 0x26; + } else { + data[0] = 0x10; + data[1] = 0x20; + } + helene_write_regs(priv, 0x91, data, 2); + + /* Setting for analog block */ + if (TERR_INTERNAL_LOOPFILTER_AVAILABLE(tv_system)) + data[0] = 0x90; + else + data[0] = 0x00; + + /* Setting for local polarity (0x9D) */ + data[1] = (uint8_t)(terr_params[tv_system].IS_LOWERLOCAL & 0x01); + helene_write_regs(priv, 0x9C, data, 2); + + /* Enable for analog block */ + data[0] = 0xEE; + data[1] = 0x02; + data[2] = 0x1E; + data[3] = 0x67; /* Tuning setting for CPU */ + + /* Setting for PLL reference divider for xtal=24MHz */ + if ((tv_system == SONY_HELENE_DTV_DVBC_6) || + (tv_system == SONY_HELENE_DTV_DVBC_8)) + data[4] = 0x18; + else + data[4] = 0x03; + + /* Tuning setting for analog block */ + if (TERR_INTERNAL_LOOPFILTER_AVAILABLE(tv_system)) { + data[5] = 0x38; + data[6] = 0x1E; + data[7] = 0x02; + data[8] = 0x24; + } else if ((tv_system == SONY_HELENE_DTV_DVBC_6) || + (tv_system == SONY_HELENE_DTV_DVBC_8)) { + data[5] = 0x1C; + data[6] = 0x78; + data[7] = 0x08; + data[8] = 0x1C; + } else { + data[5] = 0xB4; + data[6] = 0x78; + data[7] = 0x08; + data[8] = 0x30; + } + helene_write_regs(priv, 0x5E, data, 9); + + /* LT_AMP_EN should be 0 */ + helene_set_reg_bits(priv, 0x67, 0x0, 0x02); + + /* Setting for IFOUT_LIMIT */ + data[0] = 0x00; /* 1.5Vpp */ + + /* RF_GAIN setting */ + if (terr_params[tv_system].RF_GAIN == HELENE_AUTO) + data[1] = 0x80; /* RF_GAIN_SEL = 1 */ + else + data[1] = (uint8_t)((terr_params[tv_system].RF_GAIN + << 4) & 0x70); + + /* IF_BPF_GC setting */ + data[1] |= (uint8_t)(terr_params[tv_system].IF_BPF_GC & 0x0F); + + /* Setting for internal RFAGC (0x6A, 0x6B, 0x6C) */ + data[2] = 0x00; + if (frequencykHz <= 172000) { + data[3] = (uint8_t)(terr_params[tv_system].RFOVLD_DET_LV1_VL + & 0x0F); + data[4] = (uint8_t)(terr_params[tv_system].IFOVLD_DET_LV_VL + & 0x07); + } else if (frequencykHz <= 464000) { + data[3] = (uint8_t)(terr_params[tv_system].RFOVLD_DET_LV1_VH + & 0x0F); + data[4] = (uint8_t)(terr_params[tv_system].IFOVLD_DET_LV_VH + & 0x07); + } else { + data[3] = (uint8_t)(terr_params[tv_system].RFOVLD_DET_LV1_U + & 0x0F); + data[4] = (uint8_t)(terr_params[tv_system].IFOVLD_DET_LV_U + & 0x07); + } + data[4] |= 0x20; + + /* Setting for IF frequency and bandwidth */ + + /* IF filter center frequency offset (IF_BPF_F0) (0x6D) */ + data[5] = (uint8_t)((terr_params[tv_system].IF_BPF_F0 << 4) & 0x30); + + /* IF filter band width (BW) (0x6D) */ + data[5] |= (uint8_t)(terr_params[tv_system].BW & 0x03); + + /* IF frequency offset value (FIF_OFFSET) (0x6E) */ + data[6] = (uint8_t)(terr_params[tv_system].FIF_OFFSET & 0x1F); + + /* IF band width offset value (BW_OFFSET) (0x6F) */ + data[7] = (uint8_t)(terr_params[tv_system].BW_OFFSET & 0x1F); + + /* RF tuning frequency setting (0x70, 0x71, 0x72) */ + data[8] = (uint8_t)(frequencykHz & 0xFF); /* FRF_L */ + data[9] = (uint8_t)((frequencykHz >> 8) & 0xFF); /* FRF_M */ + data[10] = (uint8_t)((frequencykHz >> 16) + & 0x0F); /* FRF_H (bit[3:0]) */ + + /* Tuning command */ + data[11] = 0xFF; + + /* Enable IF output, AGC and IFOUT pin selection (0x74) */ + data[12] = 0x01; + + if ((tv_system == SONY_HELENE_DTV_DVBC_6) || + (tv_system == SONY_HELENE_DTV_DVBC_8)) { + data[13] = 0xD9; + data[14] = 0x0F; + data[15] = 0x24; + data[16] = 0x87; + } else { + data[13] = 0x99; + data[14] = 0x00; + data[15] = 0x24; + data[16] = 0x87; + } + + helene_write_regs(priv, 0x68, data, 17); + + dev_dbg(&priv->i2c->dev, "%s(): tune done\n", + __func__); + + priv->frequency = frequency; + return 0; +} + +static int helene_get_frequency(struct dvb_frontend *fe, u32 *frequency) +{ + struct helene_priv *priv = fe->tuner_priv; + + *frequency = priv->frequency * 1000; + return 0; +} + +static struct dvb_tuner_ops helene_tuner_ops = { + .info = { + .name = "Sony HELENE Ter tuner", + .frequency_min = 1000000, + .frequency_max = 1200000000, + .frequency_step = 25000, + }, + .init = helene_init, + .release = helene_release, + .sleep = helene_sleep, + .set_params = helene_set_params, + .get_frequency = helene_get_frequency, +}; + +static struct dvb_tuner_ops helene_tuner_ops_s = { + .info = { + .name = "Sony HELENE Sat tuner", + .frequency_min = 500000, + .frequency_max = 2500000, + .frequency_step = 1000, + }, + .init = helene_init, + .release = helene_release, + .sleep = helene_sleep, + .set_params = helene_set_params_s, + .get_frequency = helene_get_frequency, +}; + +/* power-on tuner + * call once after reset + */ +static int helene_x_pon(struct helene_priv *priv) +{ + /* RFIN matching in power save (terrestrial) = ACTIVE */ + /* RFIN matching in power save (satellite) = ACTIVE */ + u8 dataT[] = { 0x06, 0x00, 0x02, 0x00 }; + /* SAT_RF_ACTIVE = true, lnaOff = false, terrRfActive = true */ + u8 dataS[] = { 0x05, 0x06 }; + u8 cdata[] = {0x7A, 0x01}; + u8 data[20]; + u8 rdata[2]; + + /* mode select */ + helene_write_reg(priv, 0x01, 0x00); + + helene_write_reg(priv, 0x67, dataT[3]); + helene_write_reg(priv, 0x43, dataS[1]); + helene_write_regs(priv, 0x5E, dataT, 3); + helene_write_reg(priv, 0x0C, dataS[0]); + + /* Initial setting for internal logic block */ + helene_write_regs(priv, 0x99, cdata, sizeof(cdata)); + + /* 0x81 - 0x94 */ + data[0] = 0x18; /* xtal 24 MHz */ + data[1] = (uint8_t)(0x80 | (0x04 & 0x1F)); /* 4 x 25 = 100uA */ + data[2] = (uint8_t)(0x80 | (0x26 & 0x7F)); /* 38 x 0.25 = 9.5pF */ + data[3] = 0x80; /* REFOUT signal output 500mVpp */ + data[4] = 0x00; /* GPIO settings */ + data[5] = 0x00; /* GPIO settings */ + data[6] = 0xC4; /* Clock enable for internal logic block */ + data[7] = 0x40; /* Start CPU boot-up */ + data[8] = 0x10; /* For burst-write */ + + /* Setting for internal RFAGC */ + data[9] = 0x00; + data[10] = 0x45; + data[11] = 0x75; + + data[12] = 0x07; /* Setting for analog block */ + + /* Initial setting for internal analog block */ + data[13] = 0x1C; + data[14] = 0x3F; + data[15] = 0x02; + data[16] = 0x10; + data[17] = 0x20; + data[18] = 0x0A; + data[19] = 0x00; + + helene_write_regs(priv, 0x81, data, sizeof(data)); + + /* Setting for internal RFAGC */ + helene_write_reg(priv, 0x9B, 0x00); + + msleep(20); + + /* Check CPU_STT/CPU_ERR */ + helene_read_regs(priv, 0x1A, rdata, sizeof(rdata)); + + if (rdata[0] != 0x00) { + dev_err(&priv->i2c->dev, + "HELENE tuner CPU error 0x%x\n", rdata[0]); + return -EIO; + } + + /* VCO current setting */ + cdata[0] = 0x90; + cdata[1] = 0x06; + helene_write_regs(priv, 0x17, cdata, sizeof(cdata)); + msleep(20); + helene_read_reg(priv, 0x19, data); + helene_write_reg(priv, 0x95, (uint8_t)((data[0] >> 4) & 0x0F)); + + /* Disable IF signal output */ + helene_write_reg(priv, 0x74, 0x02); + + /* Standby setting for CPU */ + helene_write_reg(priv, 0x88, 0x00); + + /* Standby setting for internal logic block */ + helene_write_reg(priv, 0x87, 0xC0); + + /* Load capacitance control setting for crystal oscillator */ + helene_write_reg(priv, 0x80, 0x01); + + /* Satellite initial setting */ + cdata[0] = 0x07; + cdata[1] = 0x00; + helene_write_regs(priv, 0x41, cdata, sizeof(cdata)); + + dev_info(&priv->i2c->dev, + "HELENE tuner x_pon done\n"); + + return 0; +} + +struct dvb_frontend *helene_attach_s(struct dvb_frontend *fe, + const struct helene_config *config, + struct i2c_adapter *i2c) +{ + struct helene_priv *priv = NULL; + + priv = kzalloc(sizeof(struct helene_priv), GFP_KERNEL); + if (priv == NULL) + return NULL; + priv->i2c_address = (config->i2c_address >> 1); + priv->i2c = i2c; + priv->set_tuner_data = config->set_tuner_priv; + priv->set_tuner = config->set_tuner_callback; + priv->xtal = config->xtal; + + if (fe->ops.i2c_gate_ctrl) + fe->ops.i2c_gate_ctrl(fe, 1); + + if (helene_x_pon(priv) != 0) + return NULL; + + if (fe->ops.i2c_gate_ctrl) + fe->ops.i2c_gate_ctrl(fe, 0); + + memcpy(&fe->ops.tuner_ops, &helene_tuner_ops_s, + sizeof(struct dvb_tuner_ops)); + fe->tuner_priv = priv; + dev_info(&priv->i2c->dev, + "Sony HELENE Sat attached on addr=%x at I2C adapter %p\n", + priv->i2c_address, priv->i2c); + return fe; +} +EXPORT_SYMBOL(helene_attach_s); + +struct dvb_frontend *helene_attach(struct dvb_frontend *fe, + const struct helene_config *config, + struct i2c_adapter *i2c) +{ + struct helene_priv *priv = NULL; + + priv = kzalloc(sizeof(struct helene_priv), GFP_KERNEL); + if (priv == NULL) + return NULL; + priv->i2c_address = (config->i2c_address >> 1); + priv->i2c = i2c; + priv->set_tuner_data = config->set_tuner_priv; + priv->set_tuner = config->set_tuner_callback; + priv->xtal = config->xtal; + + if (fe->ops.i2c_gate_ctrl) + fe->ops.i2c_gate_ctrl(fe, 1); + + if (helene_x_pon(priv) != 0) + return NULL; + + if (fe->ops.i2c_gate_ctrl) + fe->ops.i2c_gate_ctrl(fe, 0); + + memcpy(&fe->ops.tuner_ops, &helene_tuner_ops, + sizeof(struct dvb_tuner_ops)); + fe->tuner_priv = priv; + dev_info(&priv->i2c->dev, + "Sony HELENE Ter attached on addr=%x at I2C adapter %p\n", + priv->i2c_address, priv->i2c); + return fe; +} +EXPORT_SYMBOL(helene_attach); + +MODULE_DESCRIPTION("Sony HELENE Sat/Ter tuner driver"); +MODULE_AUTHOR("Abylay Ospan "); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/dvb-frontends/helene.h b/drivers/media/dvb-frontends/helene.h new file mode 100644 index 000000000000..e1b9224cfc55 --- /dev/null +++ b/drivers/media/dvb-frontends/helene.h @@ -0,0 +1,79 @@ +/* + * helene.h + * + * Sony HELENE DVB-S/S2/T/T2/C/C2/ISDB-T/S tuner driver (CXD2858ER) + * + * Copyright 2012 Sony Corporation + * Copyright (C) 2014 NetUP Inc. + * Copyright (C) 2014 Abylay Ospan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __DVB_HELENE_H__ +#define __DVB_HELENE_H__ + +#include +#include +#include + +enum helene_xtal { + SONY_HELENE_XTAL_16000, /* 16 MHz */ + SONY_HELENE_XTAL_20500, /* 20.5 MHz */ + SONY_HELENE_XTAL_24000, /* 24 MHz */ + SONY_HELENE_XTAL_41000 /* 41 MHz */ +}; + +/** + * struct helene_config - the configuration of 'Helene' tuner driver + * @i2c_address: I2C address of the tuner + * @xtal_freq_mhz: Oscillator frequency, MHz + * @set_tuner_priv: Callback function private context + * @set_tuner_callback: Callback function that notifies the parent driver + * which tuner is active now + */ +struct helene_config { + u8 i2c_address; + u8 xtal_freq_mhz; + void *set_tuner_priv; + int (*set_tuner_callback)(void *, int); + enum helene_xtal xtal; +}; + +#if IS_REACHABLE(CONFIG_DVB_HELENE) +extern struct dvb_frontend *helene_attach(struct dvb_frontend *fe, + const struct helene_config *config, + struct i2c_adapter *i2c); +#else +static inline struct dvb_frontend *helene_attach(struct dvb_frontend *fe, + const struct helene_config *config, + struct i2c_adapter *i2c) +{ + pr_warn("%s: driver disabled by Kconfig\n", __func__); + return NULL; +} +#endif + +#if IS_REACHABLE(CONFIG_DVB_HELENE) +extern struct dvb_frontend *helene_attach_s(struct dvb_frontend *fe, + const struct helene_config *config, + struct i2c_adapter *i2c); +#else +static inline struct dvb_frontend *helene_attach_s(struct dvb_frontend *fe, + const struct helene_config *config, + struct i2c_adapter *i2c) +{ + pr_warn("%s: driver disabled by Kconfig\n", __func__); + return NULL; +} +#endif + +#endif diff --git a/drivers/media/dvb-frontends/horus3a.c b/drivers/media/dvb-frontends/horus3a.c index 000606af70f7..a98bca5270d9 100644 --- a/drivers/media/dvb-frontends/horus3a.c +++ b/drivers/media/dvb-frontends/horus3a.c @@ -66,7 +66,7 @@ static int horus3a_write_regs(struct horus3a_priv *priv, } }; - if (len + 1 >= sizeof(buf)) { + if (len + 1 > sizeof(buf)) { dev_warn(&priv->i2c->dev,"wr reg=%04x: len=%d is too big!\n", reg, len + 1); return -E2BIG; @@ -272,24 +272,6 @@ static int horus3a_set_params(struct dvb_frontend *fe) if (fc_lpf > 36) fc_lpf = 36; } else if (p->delivery_system == SYS_DVBS2) { - int rolloff; - - switch (p->rolloff) { - case ROLLOFF_35: - rolloff = 35; - break; - case ROLLOFF_25: - rolloff = 25; - break; - case ROLLOFF_20: - rolloff = 20; - break; - case ROLLOFF_AUTO: - default: - dev_err(&priv->i2c->dev, - "horus3a: auto roll-off is not supported\n"); - return -EINVAL; - } /* * SR <= 4.5: * fc_lpf = 5 @@ -302,11 +284,9 @@ static int horus3a_set_params(struct dvb_frontend *fe) if (symbol_rate <= 4500) fc_lpf = 5; else if (symbol_rate <= 10000) - fc_lpf = (u8)DIV_ROUND_UP( - symbol_rate * (200 + rolloff), 200000); + fc_lpf = (u8)((symbol_rate * 11 + (10000-1)) / 10000); else - fc_lpf = (u8)DIV_ROUND_UP( - symbol_rate * (100 + rolloff), 200000) + 5; + fc_lpf = (u8)((symbol_rate * 3 + (5000-1)) / 5000 + 5); /* 5 <= fc_lpf <= 36 is valid */ if (fc_lpf > 36) fc_lpf = 36; diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c index 5557ef8fc704..e0fe5bc9dbce 100644 --- a/drivers/media/dvb-frontends/m88ds3103.c +++ b/drivers/media/dvb-frontends/m88ds3103.c @@ -306,8 +306,8 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) const struct m88ds3103_reg_val *init; u8 u8tmp, u8tmp1 = 0, u8tmp2 = 0; /* silence compiler warning */ u8 buf[3]; - u16 u16tmp, divide_ratio = 0; - u32 tuner_frequency, target_mclk; + u16 u16tmp; + u32 tuner_frequency_khz, target_mclk; s32 s32tmp; dev_dbg(&client->dev, @@ -344,7 +344,7 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) } if (fe->ops.tuner_ops.get_frequency) { - ret = fe->ops.tuner_ops.get_frequency(fe, &tuner_frequency); + ret = fe->ops.tuner_ops.get_frequency(fe, &tuner_frequency_khz); if (ret) goto err; } else { @@ -353,20 +353,20 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) * actual frequency used. Carrier offset calculation is not * valid. */ - tuner_frequency = c->frequency; + tuner_frequency_khz = c->frequency; } /* select M88RS6000 demod main mclk and ts mclk from tuner die. */ if (dev->chip_id == M88RS6000_CHIP_ID) { if (c->symbol_rate > 45010000) - dev->mclk_khz = 110250; + dev->mclk = 110250000; else - dev->mclk_khz = 96000; + dev->mclk = 96000000; if (c->delivery_system == SYS_DVBS) - target_mclk = 96000; + target_mclk = 96000000; else - target_mclk = 144000; + target_mclk = 144000000; /* Enable demod clock path */ ret = regmap_write(dev->regmap, 0x06, 0x00); @@ -375,7 +375,7 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) usleep_range(10000, 20000); } else { /* set M88DS3103 mclk and ts mclk. */ - dev->mclk_khz = 96000; + dev->mclk = 96000000; switch (dev->cfg->ts_mode) { case M88DS3103_TS_SERIAL: @@ -385,14 +385,14 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) case M88DS3103_TS_PARALLEL: case M88DS3103_TS_CI: if (c->delivery_system == SYS_DVBS) - target_mclk = 96000; + target_mclk = 96000000; else { if (c->symbol_rate < 18000000) - target_mclk = 96000; + target_mclk = 96000000; else if (c->symbol_rate < 28000000) - target_mclk = 144000; + target_mclk = 144000000; else - target_mclk = 192000; + target_mclk = 192000000; } break; default: @@ -402,15 +402,15 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) } switch (target_mclk) { - case 96000: + case 96000000: u8tmp1 = 0x02; /* 0b10 */ u8tmp2 = 0x01; /* 0b01 */ break; - case 144000: + case 144000000: u8tmp1 = 0x00; /* 0b00 */ u8tmp2 = 0x01; /* 0b01 */ break; - case 192000: + case 192000000: u8tmp1 = 0x03; /* 0b11 */ u8tmp2 = 0x00; /* 0b00 */ break; @@ -464,8 +464,8 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) } if (dev->chip_id == M88RS6000_CHIP_ID) { - if ((c->delivery_system == SYS_DVBS2) - && ((c->symbol_rate / 1000) <= 5000)) { + if (c->delivery_system == SYS_DVBS2 && + c->symbol_rate <= 5000000) { ret = regmap_write(dev->regmap, 0xc0, 0x04); if (ret) goto err; @@ -522,37 +522,25 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) ret = m88ds3103_update_bits(dev, 0x29, 0x20, u8tmp1); if (ret) goto err; - u8tmp1 = 0; - u8tmp2 = 0; + u16tmp = 0; + u8tmp1 = 0x3f; + u8tmp2 = 0x3f; break; default: - if (dev->cfg->ts_clk) { - divide_ratio = DIV_ROUND_UP(target_mclk, dev->cfg->ts_clk); - u8tmp1 = divide_ratio / 2; - u8tmp2 = DIV_ROUND_UP(divide_ratio, 2); - } + u16tmp = DIV_ROUND_UP(target_mclk, dev->cfg->ts_clk); + u8tmp1 = u16tmp / 2 - 1; + u8tmp2 = DIV_ROUND_UP(u16tmp, 2) - 1; } - dev_dbg(&client->dev, - "target_mclk=%d ts_clk=%d divide_ratio=%d\n", - target_mclk, dev->cfg->ts_clk, divide_ratio); + dev_dbg(&client->dev, "target_mclk=%u ts_clk=%u ts_clk_divide_ratio=%u\n", + target_mclk, dev->cfg->ts_clk, u16tmp); - u8tmp1--; - u8tmp2--; /* u8tmp1[5:2] => fe[3:0], u8tmp1[1:0] => ea[7:6] */ - u8tmp1 &= 0x3f; /* u8tmp2[5:0] => ea[5:0] */ - u8tmp2 &= 0x3f; - - ret = regmap_bulk_read(dev->regmap, 0xfe, &u8tmp, 1); + u8tmp = (u8tmp1 >> 2) & 0x0f; + ret = regmap_update_bits(dev->regmap, 0xfe, 0x0f, u8tmp); if (ret) goto err; - - u8tmp = ((u8tmp & 0xf0) << 0) | u8tmp1 >> 2; - ret = regmap_write(dev->regmap, 0xfe, u8tmp); - if (ret) - goto err; - u8tmp = ((u8tmp1 & 0x03) << 6) | u8tmp2 >> 0; ret = regmap_write(dev->regmap, 0xea, u8tmp); if (ret) @@ -581,7 +569,7 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) if (ret) goto err; - u16tmp = DIV_ROUND_CLOSEST((c->symbol_rate / 1000) << 15, dev->mclk_khz / 2); + u16tmp = DIV_ROUND_CLOSEST_ULL((u64)c->symbol_rate * 0x10000, dev->mclk); buf[0] = (u16tmp >> 0) & 0xff; buf[1] = (u16tmp >> 8) & 0xff; ret = regmap_bulk_write(dev->regmap, 0x61, buf, 2); @@ -601,13 +589,11 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe) goto err; dev_dbg(&client->dev, "carrier offset=%d\n", - (tuner_frequency - c->frequency)); - - s32tmp = 0x10000 * (tuner_frequency - c->frequency); - s32tmp = DIV_ROUND_CLOSEST(s32tmp, dev->mclk_khz); - if (s32tmp < 0) - s32tmp += 0x10000; + (tuner_frequency_khz - c->frequency)); + /* Use 32-bit calc as there is no s64 version of DIV_ROUND_CLOSEST() */ + s32tmp = 0x10000 * (tuner_frequency_khz - c->frequency); + s32tmp = DIV_ROUND_CLOSEST(s32tmp, dev->mclk / 1000); buf[0] = (s32tmp >> 0) & 0xff; buf[1] = (s32tmp >> 8) & 0xff; ret = regmap_bulk_write(dev->regmap, 0x5e, buf, 2); @@ -635,10 +621,10 @@ static int m88ds3103_init(struct dvb_frontend *fe) struct m88ds3103_dev *dev = fe->demodulator_priv; struct i2c_client *client = dev->client; struct dtv_frontend_properties *c = &fe->dtv_property_cache; - int ret, len, remaining; + int ret, len, rem; unsigned int utmp; - const struct firmware *fw = NULL; - u8 *fw_file; + const struct firmware *firmware; + const char *name; dev_dbg(&client->dev, "\n"); @@ -664,7 +650,7 @@ static int m88ds3103_init(struct dvb_frontend *fe) dev_dbg(&client->dev, "firmware=%02x\n", utmp); if (utmp) - goto skip_fw_download; + goto warm; /* global reset, global diseqc reset, golbal fec reset */ ret = regmap_write(dev->regmap, 0x07, 0xe0); @@ -679,52 +665,47 @@ static int m88ds3103_init(struct dvb_frontend *fe) m88ds3103_ops.info.name); if (dev->chip_id == M88RS6000_CHIP_ID) - fw_file = M88RS6000_FIRMWARE; + name = M88RS6000_FIRMWARE; else - fw_file = M88DS3103_FIRMWARE; + name = M88DS3103_FIRMWARE; /* request the firmware, this will block and timeout */ - ret = request_firmware(&fw, fw_file, &client->dev); + ret = request_firmware(&firmware, name, &client->dev); if (ret) { - dev_err(&client->dev, "firmware file '%s' not found\n", fw_file); + dev_err(&client->dev, "firmware file '%s' not found\n", name); goto err; } - dev_info(&client->dev, "downloading firmware from file '%s'\n", - fw_file); + dev_info(&client->dev, "downloading firmware from file '%s'\n", name); ret = regmap_write(dev->regmap, 0xb2, 0x01); if (ret) - goto error_fw_release; - - for (remaining = fw->size; remaining > 0; - remaining -= (dev->cfg->i2c_wr_max - 1)) { - len = remaining; - if (len > (dev->cfg->i2c_wr_max - 1)) - len = (dev->cfg->i2c_wr_max - 1); + goto err_release_firmware; + for (rem = firmware->size; rem > 0; rem -= (dev->cfg->i2c_wr_max - 1)) { + len = min(dev->cfg->i2c_wr_max - 1, rem); ret = regmap_bulk_write(dev->regmap, 0xb0, - &fw->data[fw->size - remaining], len); + &firmware->data[firmware->size - rem], + len); if (ret) { - dev_err(&client->dev, "firmware download failed=%d\n", + dev_err(&client->dev, "firmware download failed %d\n", ret); - goto error_fw_release; + goto err_release_firmware; } } ret = regmap_write(dev->regmap, 0xb2, 0x00); if (ret) - goto error_fw_release; + goto err_release_firmware; - release_firmware(fw); - fw = NULL; + release_firmware(firmware); ret = regmap_read(dev->regmap, 0xb9, &utmp); if (ret) goto err; if (!utmp) { + ret = -EINVAL; dev_info(&client->dev, "firmware did not run\n"); - ret = -EFAULT; goto err; } @@ -733,7 +714,7 @@ static int m88ds3103_init(struct dvb_frontend *fe) dev_info(&client->dev, "firmware version: %X.%X\n", (utmp >> 4) & 0xf, (utmp >> 0 & 0xf)); -skip_fw_download: +warm: /* warm state */ dev->warm = true; @@ -746,8 +727,8 @@ skip_fw_download: c->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; return 0; -error_fw_release: - release_firmware(fw); +err_release_firmware: + release_firmware(firmware); err: dev_dbg(&client->dev, "failed=%d\n", ret); return ret; @@ -952,8 +933,7 @@ static int m88ds3103_get_frontend(struct dvb_frontend *fe, if (ret) goto err; - c->symbol_rate = 1ull * ((buf[1] << 8) | (buf[0] << 0)) * - dev->mclk_khz * 1000 / 0x10000; + c->symbol_rate = DIV_ROUND_CLOSEST_ULL((u64)(buf[1] << 8 | buf[0] << 0) * dev->mclk, 0x10000); return 0; err: @@ -1119,8 +1099,9 @@ static int m88ds3103_diseqc_send_master_cmd(struct dvb_frontend *fe, #define SEND_MASTER_CMD_TIMEOUT 120 timeout = jiffies + msecs_to_jiffies(SEND_MASTER_CMD_TIMEOUT); - /* DiSEqC message typical period is 54 ms */ - usleep_range(50000, 54000); + /* DiSEqC message period is 13.5 ms per byte */ + utmp = diseqc_cmd->msg_len * 13500; + usleep_range(utmp - 4000, utmp); for (utmp = 1; !time_after(jiffies, timeout) && utmp;) { ret = regmap_read(dev->regmap, 0xa1, &utmp); @@ -1395,7 +1376,7 @@ static int m88ds3103_probe(struct i2c_client *client, dev->config.clock = pdata->clk; dev->config.i2c_wr_max = pdata->i2c_wr_max; dev->config.ts_mode = pdata->ts_mode; - dev->config.ts_clk = pdata->ts_clk; + dev->config.ts_clk = pdata->ts_clk * 1000; dev->config.ts_clk_pol = pdata->ts_clk_pol; dev->config.spec_inv = pdata->spec_inv; dev->config.agc_inv = pdata->agc_inv; @@ -1446,6 +1427,11 @@ static int m88ds3103_probe(struct i2c_client *client, goto err_kfree; } + if (!pdata->ts_clk) { + ret = -EINVAL; + goto err_kfree; + } + /* 0x29 register is defined differently for m88rs6000. */ /* set internal tuner address to 0x21 */ if (dev->chip_id == M88RS6000_CHIP_ID) diff --git a/drivers/media/dvb-frontends/m88ds3103_priv.h b/drivers/media/dvb-frontends/m88ds3103_priv.h index d78e467295d2..07f20c269c67 100644 --- a/drivers/media/dvb-frontends/m88ds3103_priv.h +++ b/drivers/media/dvb-frontends/m88ds3103_priv.h @@ -27,7 +27,6 @@ #define M88DS3103_FIRMWARE "dvb-demod-m88ds3103.fw" #define M88RS6000_FIRMWARE "dvb-demod-m88rs6000.fw" -#define M88DS3103_MCLK_KHZ 96000 #define M88RS6000_CHIP_ID 0x74 #define M88DS3103_CHIP_ID 0x70 @@ -46,7 +45,7 @@ struct m88ds3103_dev { /* auto detect chip id to do different config */ u8 chip_id; /* main mclk is calculated for M88RS6000 dynamically */ - s32 mclk_khz; + s32 mclk; u64 post_bit_error; u64 post_bit_count; }; diff --git a/drivers/media/dvb-frontends/m88rs2000.c b/drivers/media/dvb-frontends/m88rs2000.c index a09b12313a73..ef79a4ec31e2 100644 --- a/drivers/media/dvb-frontends/m88rs2000.c +++ b/drivers/media/dvb-frontends/m88rs2000.c @@ -609,7 +609,7 @@ static int m88rs2000_set_frontend(struct dvb_frontend *fe) { struct m88rs2000_state *state = fe->demodulator_priv; struct dtv_frontend_properties *c = &fe->dtv_property_cache; - enum fe_status status; + enum fe_status status = 0; int i, ret = 0; u32 tuner_freq; s16 offset = 0; diff --git a/drivers/media/dvb-frontends/mb86a20s.c b/drivers/media/dvb-frontends/mb86a20s.c index fb88dddaf3a3..41325328a22e 100644 --- a/drivers/media/dvb-frontends/mb86a20s.c +++ b/drivers/media/dvb-frontends/mb86a20s.c @@ -301,10 +301,11 @@ static int mb86a20s_read_status(struct dvb_frontend *fe, enum fe_status *status) *status = 0; - val = mb86a20s_readreg(state, 0x0a) & 0xf; + val = mb86a20s_readreg(state, 0x0a); if (val < 0) return val; + val &= 0xf; if (val >= 2) *status |= FE_HAS_SIGNAL; diff --git a/drivers/staging/media/mn88472/mn88472.c b/drivers/media/dvb-frontends/mn88472.c similarity index 58% rename from drivers/staging/media/mn88472/mn88472.c rename to drivers/media/dvb-frontends/mn88472.c index 7ea749cf19f9..18fb2df1e2bd 100644 --- a/drivers/staging/media/mn88472/mn88472.c +++ b/drivers/media/dvb-frontends/mn88472.c @@ -17,28 +17,90 @@ #include "mn88472_priv.h" static int mn88472_get_tune_settings(struct dvb_frontend *fe, - struct dvb_frontend_tune_settings *s) + struct dvb_frontend_tune_settings *s) { - s->min_delay_ms = 800; + s->min_delay_ms = 1000; return 0; } +static int mn88472_read_status(struct dvb_frontend *fe, enum fe_status *status) +{ + struct i2c_client *client = fe->demodulator_priv; + struct mn88472_dev *dev = i2c_get_clientdata(client); + struct dtv_frontend_properties *c = &fe->dtv_property_cache; + int ret; + unsigned int utmp; + + if (!dev->active) { + ret = -EAGAIN; + goto err; + } + + switch (c->delivery_system) { + case SYS_DVBT: + ret = regmap_read(dev->regmap[0], 0x7f, &utmp); + if (ret) + goto err; + if ((utmp & 0x0f) >= 0x09) + *status = FE_HAS_SIGNAL | FE_HAS_CARRIER | + FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK; + else + *status = 0; + break; + case SYS_DVBT2: + ret = regmap_read(dev->regmap[2], 0x92, &utmp); + if (ret) + goto err; + if ((utmp & 0x0f) >= 0x0d) + *status = FE_HAS_SIGNAL | FE_HAS_CARRIER | + FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK; + else if ((utmp & 0x0f) >= 0x0a) + *status = FE_HAS_SIGNAL | FE_HAS_CARRIER | + FE_HAS_VITERBI; + else if ((utmp & 0x0f) >= 0x07) + *status = FE_HAS_SIGNAL | FE_HAS_CARRIER; + else + *status = 0; + break; + case SYS_DVBC_ANNEX_A: + ret = regmap_read(dev->regmap[1], 0x84, &utmp); + if (ret) + goto err; + if ((utmp & 0x0f) >= 0x08) + *status = FE_HAS_SIGNAL | FE_HAS_CARRIER | + FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK; + else + *status = 0; + break; + default: + ret = -EINVAL; + goto err; + } + + return 0; +err: + dev_dbg(&client->dev, "failed=%d\n", ret); + return ret; +} + static int mn88472_set_frontend(struct dvb_frontend *fe) { struct i2c_client *client = fe->demodulator_priv; struct mn88472_dev *dev = i2c_get_clientdata(client); struct dtv_frontend_properties *c = &fe->dtv_property_cache; int ret, i; - u32 if_frequency = 0; - u64 tmp; - u8 delivery_system_val, if_val[3], bw_val[7], bw_val2; + unsigned int utmp; + u32 if_frequency; + u8 buf[3], delivery_system_val, bandwidth_val, *bandwidth_vals_ptr; + u8 reg_bank0_b4_val, reg_bank0_cd_val, reg_bank0_d4_val; + u8 reg_bank0_d6_val; dev_dbg(&client->dev, - "delivery_system=%d modulation=%d frequency=%d symbol_rate=%d inversion=%d\n", - c->delivery_system, c->modulation, - c->frequency, c->symbol_rate, c->inversion); + "delivery_system=%u modulation=%u frequency=%u bandwidth_hz=%u symbol_rate=%u inversion=%d stream_id=%d\n", + c->delivery_system, c->modulation, c->frequency, + c->bandwidth_hz, c->symbol_rate, c->inversion, c->stream_id); - if (!dev->warm) { + if (!dev->active) { ret = -EAGAIN; goto err; } @@ -46,39 +108,64 @@ static int mn88472_set_frontend(struct dvb_frontend *fe) switch (c->delivery_system) { case SYS_DVBT: delivery_system_val = 0x02; + reg_bank0_b4_val = 0x00; + reg_bank0_cd_val = 0x1f; + reg_bank0_d4_val = 0x0a; + reg_bank0_d6_val = 0x48; break; case SYS_DVBT2: delivery_system_val = 0x03; + reg_bank0_b4_val = 0xf6; + reg_bank0_cd_val = 0x01; + reg_bank0_d4_val = 0x09; + reg_bank0_d6_val = 0x46; break; case SYS_DVBC_ANNEX_A: delivery_system_val = 0x04; + reg_bank0_b4_val = 0x00; + reg_bank0_cd_val = 0x17; + reg_bank0_d4_val = 0x09; + reg_bank0_d6_val = 0x48; break; default: ret = -EINVAL; goto err; } - if (c->bandwidth_hz <= 5000000) { - memcpy(bw_val, "\xe5\x99\x9a\x1b\xa9\x1b\xa9", 7); - bw_val2 = 0x03; - } else if (c->bandwidth_hz <= 6000000) { - /* IF 3570000 Hz, BW 6000000 Hz */ - memcpy(bw_val, "\xbf\x55\x55\x15\x6b\x15\x6b", 7); - bw_val2 = 0x02; - } else if (c->bandwidth_hz <= 7000000) { - /* IF 4570000 Hz, BW 7000000 Hz */ - memcpy(bw_val, "\xa4\x00\x00\x0f\x2c\x0f\x2c", 7); - bw_val2 = 0x01; - } else if (c->bandwidth_hz <= 8000000) { - /* IF 4570000 Hz, BW 8000000 Hz */ - memcpy(bw_val, "\x8f\x80\x00\x08\xee\x08\xee", 7); - bw_val2 = 0x00; - } else { - ret = -EINVAL; - goto err; + switch (c->delivery_system) { + case SYS_DVBT: + case SYS_DVBT2: + switch (c->bandwidth_hz) { + case 5000000: + bandwidth_vals_ptr = "\xe5\x99\x9a\x1b\xa9\x1b\xa9"; + bandwidth_val = 0x03; + break; + case 6000000: + bandwidth_vals_ptr = "\xbf\x55\x55\x15\x6b\x15\x6b"; + bandwidth_val = 0x02; + break; + case 7000000: + bandwidth_vals_ptr = "\xa4\x00\x00\x0f\x2c\x0f\x2c"; + bandwidth_val = 0x01; + break; + case 8000000: + bandwidth_vals_ptr = "\x8f\x80\x00\x08\xee\x08\xee"; + bandwidth_val = 0x00; + break; + default: + ret = -EINVAL; + goto err; + } + break; + case SYS_DVBC_ANNEX_A: + bandwidth_vals_ptr = NULL; + bandwidth_val = 0x00; + break; + default: + break; } - /* program tuner */ + /* Program tuner */ if (fe->ops.tuner_ops.set_params) { ret = fe->ops.tuner_ops.set_params(fe); if (ret) @@ -91,20 +178,10 @@ static int mn88472_set_frontend(struct dvb_frontend *fe) goto err; dev_dbg(&client->dev, "get_if_frequency=%d\n", if_frequency); - } - - /* Calculate IF registers ( (1<<24)*IF / Xtal ) */ - tmp = div_u64(if_frequency * (u64)(1<<24) + (dev->xtal / 2), - dev->xtal); - if_val[0] = (tmp >> 16) & 0xff; - if_val[1] = (tmp >> 8) & 0xff; - if_val[2] = (tmp >> 0) & 0xff; - - ret = regmap_write(dev->regmap[2], 0xfb, 0x13); - ret = regmap_write(dev->regmap[2], 0xef, 0x13); - ret = regmap_write(dev->regmap[2], 0xf9, 0x13); - if (ret) + } else { + ret = -EINVAL; goto err; + } ret = regmap_write(dev->regmap[2], 0x00, 0x66); if (ret) @@ -118,157 +195,81 @@ static int mn88472_set_frontend(struct dvb_frontend *fe) ret = regmap_write(dev->regmap[2], 0x03, delivery_system_val); if (ret) goto err; - ret = regmap_write(dev->regmap[2], 0x04, bw_val2); + ret = regmap_write(dev->regmap[2], 0x04, bandwidth_val); if (ret) goto err; - for (i = 0; i < sizeof(if_val); i++) { - ret = regmap_write(dev->regmap[2], 0x10 + i, if_val[i]); + /* IF */ + utmp = DIV_ROUND_CLOSEST_ULL((u64)if_frequency * 0x1000000, dev->clk); + buf[0] = (utmp >> 16) & 0xff; + buf[1] = (utmp >> 8) & 0xff; + buf[2] = (utmp >> 0) & 0xff; + for (i = 0; i < 3; i++) { + ret = regmap_write(dev->regmap[2], 0x10 + i, buf[i]); if (ret) goto err; } - for (i = 0; i < sizeof(bw_val); i++) { - ret = regmap_write(dev->regmap[2], 0x13 + i, bw_val[i]); - if (ret) - goto err; + /* Bandwidth */ + if (bandwidth_vals_ptr) { + for (i = 0; i < 7; i++) { + ret = regmap_write(dev->regmap[2], 0x13 + i, + bandwidth_vals_ptr[i]); + if (ret) + goto err; + } } + ret = regmap_write(dev->regmap[0], 0xb4, reg_bank0_b4_val); + if (ret) + goto err; + ret = regmap_write(dev->regmap[0], 0xcd, reg_bank0_cd_val); + if (ret) + goto err; + ret = regmap_write(dev->regmap[0], 0xd4, reg_bank0_d4_val); + if (ret) + goto err; + ret = regmap_write(dev->regmap[0], 0xd6, reg_bank0_d6_val); + if (ret) + goto err; + switch (c->delivery_system) { case SYS_DVBT: ret = regmap_write(dev->regmap[0], 0x07, 0x26); - ret = regmap_write(dev->regmap[0], 0xb0, 0x0a); - ret = regmap_write(dev->regmap[0], 0xb4, 0x00); - ret = regmap_write(dev->regmap[0], 0xcd, 0x1f); - ret = regmap_write(dev->regmap[0], 0xd4, 0x0a); - ret = regmap_write(dev->regmap[0], 0xd6, 0x48); + if (ret) + goto err; ret = regmap_write(dev->regmap[0], 0x00, 0xba); + if (ret) + goto err; ret = regmap_write(dev->regmap[0], 0x01, 0x13); if (ret) goto err; break; case SYS_DVBT2: ret = regmap_write(dev->regmap[2], 0x2b, 0x13); + if (ret) + goto err; ret = regmap_write(dev->regmap[2], 0x4f, 0x05); + if (ret) + goto err; ret = regmap_write(dev->regmap[1], 0xf6, 0x05); - ret = regmap_write(dev->regmap[0], 0xb0, 0x0a); - ret = regmap_write(dev->regmap[0], 0xb4, 0xf6); - ret = regmap_write(dev->regmap[0], 0xcd, 0x01); - ret = regmap_write(dev->regmap[0], 0xd4, 0x09); - ret = regmap_write(dev->regmap[0], 0xd6, 0x46); - ret = regmap_write(dev->regmap[2], 0x30, 0x80); - ret = regmap_write(dev->regmap[2], 0x32, 0x00); if (ret) goto err; - break; - case SYS_DVBC_ANNEX_A: - ret = regmap_write(dev->regmap[0], 0xb0, 0x0b); - ret = regmap_write(dev->regmap[0], 0xb4, 0x00); - ret = regmap_write(dev->regmap[0], 0xcd, 0x17); - ret = regmap_write(dev->regmap[0], 0xd4, 0x09); - ret = regmap_write(dev->regmap[0], 0xd6, 0x48); - ret = regmap_write(dev->regmap[1], 0x00, 0xb0); + ret = regmap_write(dev->regmap[2], 0x32, c->stream_id); if (ret) goto err; break; - default: - ret = -EINVAL; - goto err; - } - - ret = regmap_write(dev->regmap[0], 0x46, 0x00); - ret = regmap_write(dev->regmap[0], 0xae, 0x00); - - switch (dev->ts_mode) { - case SERIAL_TS_MODE: - ret = regmap_write(dev->regmap[2], 0x08, 0x1d); - break; - case PARALLEL_TS_MODE: - ret = regmap_write(dev->regmap[2], 0x08, 0x00); + case SYS_DVBC_ANNEX_A: break; default: - dev_dbg(&client->dev, "ts_mode error: %d\n", dev->ts_mode); - ret = -EINVAL; - goto err; - } - - switch (dev->ts_clock) { - case VARIABLE_TS_CLOCK: - ret = regmap_write(dev->regmap[0], 0xd9, 0xe3); break; - case FIXED_TS_CLOCK: - ret = regmap_write(dev->regmap[0], 0xd9, 0xe1); - break; - default: - dev_dbg(&client->dev, "ts_clock error: %d\n", dev->ts_clock); - ret = -EINVAL; - goto err; } - /* Reset demod */ + /* Reset FSM */ ret = regmap_write(dev->regmap[2], 0xf8, 0x9f); if (ret) goto err; - dev->delivery_system = c->delivery_system; - - return 0; -err: - dev_dbg(&client->dev, "failed=%d\n", ret); - return ret; -} - -static int mn88472_read_status(struct dvb_frontend *fe, enum fe_status *status) -{ - struct i2c_client *client = fe->demodulator_priv; - struct mn88472_dev *dev = i2c_get_clientdata(client); - struct dtv_frontend_properties *c = &fe->dtv_property_cache; - int ret; - unsigned int utmp; - int lock = 0; - - *status = 0; - - if (!dev->warm) { - ret = -EAGAIN; - goto err; - } - - switch (c->delivery_system) { - case SYS_DVBT: - ret = regmap_read(dev->regmap[0], 0x7F, &utmp); - if (ret) - goto err; - if ((utmp & 0xF) >= 0x09) - lock = 1; - break; - case SYS_DVBT2: - ret = regmap_read(dev->regmap[2], 0x92, &utmp); - if (ret) - goto err; - if ((utmp & 0xF) >= 0x07) - *status |= FE_HAS_SIGNAL; - if ((utmp & 0xF) >= 0x0a) - *status |= FE_HAS_CARRIER; - if ((utmp & 0xF) >= 0x0d) - *status |= FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK; - break; - case SYS_DVBC_ANNEX_A: - ret = regmap_read(dev->regmap[1], 0x84, &utmp); - if (ret) - goto err; - if ((utmp & 0xF) >= 0x08) - lock = 1; - break; - default: - ret = -EINVAL; - goto err; - } - - if (lock) - *status = FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_VITERBI | - FE_HAS_SYNC | FE_HAS_LOCK; - return 0; err: dev_dbg(&client->dev, "failed=%d\n", ret); @@ -279,93 +280,107 @@ static int mn88472_init(struct dvb_frontend *fe) { struct i2c_client *client = fe->demodulator_priv; struct mn88472_dev *dev = i2c_get_clientdata(client); - int ret, len, remaining; - const struct firmware *fw = NULL; - u8 *fw_file = MN88472_FIRMWARE; - unsigned int tmp; + int ret, len, rem; + unsigned int utmp; + const struct firmware *firmware; + const char *name = MN88472_FIRMWARE; dev_dbg(&client->dev, "\n"); - /* set cold state by default */ - dev->warm = false; - - /* power on */ + /* Power up */ ret = regmap_write(dev->regmap[2], 0x05, 0x00); if (ret) goto err; - - ret = regmap_bulk_write(dev->regmap[2], 0x0b, "\x00\x00", 2); + ret = regmap_write(dev->regmap[2], 0x0b, 0x00); if (ret) goto err; - - /* check if firmware is already running */ - ret = regmap_read(dev->regmap[0], 0xf5, &tmp); + ret = regmap_write(dev->regmap[2], 0x0c, 0x00); if (ret) goto err; - if (!(tmp & 0x1)) { - dev_info(&client->dev, "firmware already running\n"); - dev->warm = true; - return 0; - } + /* Check if firmware is already running */ + ret = regmap_read(dev->regmap[0], 0xf5, &utmp); + if (ret) + goto err; + if (!(utmp & 0x01)) + goto warm; - /* request the firmware, this will block and timeout */ - ret = request_firmware(&fw, fw_file, &client->dev); + ret = request_firmware(&firmware, name, &client->dev); if (ret) { - dev_err(&client->dev, "firmare file '%s' not found\n", - fw_file); + dev_err(&client->dev, "firmware file '%s' not found\n", name); goto err; } - dev_info(&client->dev, "downloading firmware from file '%s'\n", - fw_file); + dev_info(&client->dev, "downloading firmware from file '%s'\n", name); ret = regmap_write(dev->regmap[0], 0xf5, 0x03); if (ret) - goto firmware_release; - - for (remaining = fw->size; remaining > 0; - remaining -= (dev->i2c_wr_max - 1)) { - len = remaining; - if (len > (dev->i2c_wr_max - 1)) - len = dev->i2c_wr_max - 1; + goto err_release_firmware; + for (rem = firmware->size; rem > 0; rem -= (dev->i2c_write_max - 1)) { + len = min(dev->i2c_write_max - 1, rem); ret = regmap_bulk_write(dev->regmap[0], 0xf6, - &fw->data[fw->size - remaining], len); + &firmware->data[firmware->size - rem], + len); if (ret) { - dev_err(&client->dev, - "firmware download failed=%d\n", ret); - goto firmware_release; + dev_err(&client->dev, "firmware download failed %d\n", + ret); + goto err_release_firmware; } } - /* parity check of firmware */ - ret = regmap_read(dev->regmap[0], 0xf8, &tmp); - if (ret) { - dev_err(&client->dev, - "parity reg read failed=%d\n", ret); - goto firmware_release; - } - if (tmp & 0x10) { - dev_err(&client->dev, - "firmware parity check failed=0x%x\n", tmp); - goto firmware_release; + /* Parity check of firmware */ + ret = regmap_read(dev->regmap[0], 0xf8, &utmp); + if (ret) + goto err_release_firmware; + if (utmp & 0x10) { + ret = -EINVAL; + dev_err(&client->dev, "firmware did not run\n"); + goto err_release_firmware; } - dev_err(&client->dev, "firmware parity check succeeded=0x%x\n", tmp); ret = regmap_write(dev->regmap[0], 0xf5, 0x00); if (ret) - goto firmware_release; + goto err_release_firmware; + + release_firmware(firmware); +warm: + /* TS config */ + switch (dev->ts_mode) { + case SERIAL_TS_MODE: + utmp = 0x1d; + break; + case PARALLEL_TS_MODE: + utmp = 0x00; + break; + default: + ret = -EINVAL; + goto err; + } + ret = regmap_write(dev->regmap[2], 0x08, utmp); + if (ret) + goto err; - release_firmware(fw); - fw = NULL; + switch (dev->ts_clk) { + case VARIABLE_TS_CLOCK: + utmp = 0xe3; + break; + case FIXED_TS_CLOCK: + utmp = 0xe1; + break; + default: + ret = -EINVAL; + goto err; + } + ret = regmap_write(dev->regmap[0], 0xd9, utmp); + if (ret) + goto err; - /* warm state */ - dev->warm = true; + dev->active = true; return 0; -firmware_release: - release_firmware(fw); +err_release_firmware: + release_firmware(firmware); err: dev_dbg(&client->dev, "failed=%d\n", ret); return ret; @@ -379,18 +394,17 @@ static int mn88472_sleep(struct dvb_frontend *fe) dev_dbg(&client->dev, "\n"); - /* power off */ + /* Power down */ + ret = regmap_write(dev->regmap[2], 0x0c, 0x30); + if (ret) + goto err; ret = regmap_write(dev->regmap[2], 0x0b, 0x30); - if (ret) goto err; - ret = regmap_write(dev->regmap[2], 0x05, 0x3e); if (ret) goto err; - dev->delivery_system = SYS_UNDEFINED; - return 0; err: dev_dbg(&client->dev, "failed=%d\n", ret); @@ -434,10 +448,19 @@ static struct dvb_frontend_ops mn88472_ops = { .read_status = mn88472_read_status, }; +static struct dvb_frontend *mn88472_get_dvb_frontend(struct i2c_client *client) +{ + struct mn88472_dev *dev = i2c_get_clientdata(client); + + dev_dbg(&client->dev, "\n"); + + return &dev->fe; +} + static int mn88472_probe(struct i2c_client *client, - const struct i2c_device_id *id) + const struct i2c_device_id *id) { - struct mn88472_config *config = client->dev.platform_data; + struct mn88472_config *pdata = client->dev.platform_data; struct mn88472_dev *dev; int ret; unsigned int utmp; @@ -448,23 +471,16 @@ static int mn88472_probe(struct i2c_client *client, dev_dbg(&client->dev, "\n"); - /* Caller really need to provide pointer for frontend we create. */ - if (config->fe == NULL) { - dev_err(&client->dev, "frontend pointer not defined\n"); - ret = -EINVAL; - goto err; - } - dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) { ret = -ENOMEM; goto err; } - dev->i2c_wr_max = config->i2c_wr_max; - dev->xtal = config->xtal; - dev->ts_mode = config->ts_mode; - dev->ts_clock = config->ts_clock; + dev->i2c_write_max = pdata->i2c_wr_max ? pdata->i2c_wr_max : ~0; + dev->clk = pdata->xtal; + dev->ts_mode = pdata->ts_mode; + dev->ts_clk = pdata->ts_clock; dev->client[0] = client; dev->regmap[0] = regmap_init_i2c(dev->client[0], ®map_config); if (IS_ERR(dev->regmap[0])) { @@ -472,15 +488,25 @@ static int mn88472_probe(struct i2c_client *client, goto err_kfree; } - /* check demod answers to I2C */ - ret = regmap_read(dev->regmap[0], 0x00, &utmp); + /* Check demod answers with correct chip id */ + ret = regmap_read(dev->regmap[0], 0xff, &utmp); if (ret) goto err_regmap_0_regmap_exit; + dev_dbg(&client->dev, "chip id=%02x\n", utmp); + + if (utmp != 0x02) { + ret = -ENODEV; + goto err_regmap_0_regmap_exit; + } + /* - * Chip has three I2C addresses for different register pages. Used + * Chip has three I2C addresses for different register banks. Used * addresses are 0x18, 0x1a and 0x1c. We register two dummy clients, - * 0x1a and 0x1c, in order to get own I2C client for each register page. + * 0x1a and 0x1c, in order to get own I2C client for each register bank. + * + * Also, register bank 2 do not support sequential I/O. Only single + * register write or read is allowed to that bank. */ dev->client[1] = i2c_new_dummy(client->adapter, 0x1a); if (!dev->client[1]) { @@ -510,15 +536,25 @@ static int mn88472_probe(struct i2c_client *client, } i2c_set_clientdata(dev->client[2], dev); - /* create dvb_frontend */ + /* Sleep because chip is active by default */ + ret = regmap_write(dev->regmap[2], 0x05, 0x3e); + if (ret) + goto err_regmap_2_regmap_exit; + + /* Create dvb frontend */ memcpy(&dev->fe.ops, &mn88472_ops, sizeof(struct dvb_frontend_ops)); dev->fe.demodulator_priv = client; - *config->fe = &dev->fe; + *pdata->fe = &dev->fe; i2c_set_clientdata(client, dev); - dev_info(&client->dev, "Panasonic MN88472 successfully attached\n"); - return 0; + /* Setup callbacks */ + pdata->get_dvb_frontend = mn88472_get_dvb_frontend; + dev_info(&client->dev, "Panasonic MN88472 successfully identified\n"); + + return 0; +err_regmap_2_regmap_exit: + regmap_exit(dev->regmap[2]); err_client_2_i2c_unregister_device: i2c_unregister_device(dev->client[2]); err_regmap_1_regmap_exit: @@ -561,11 +597,12 @@ MODULE_DEVICE_TABLE(i2c, mn88472_id_table); static struct i2c_driver mn88472_driver = { .driver = { - .name = "mn88472", + .name = "mn88472", + .suppress_bind_attrs = true, }, - .probe = mn88472_probe, - .remove = mn88472_remove, - .id_table = mn88472_id_table, + .probe = mn88472_probe, + .remove = mn88472_remove, + .id_table = mn88472_id_table, }; module_i2c_driver(mn88472_driver); diff --git a/drivers/media/dvb-frontends/mn88472.h b/drivers/media/dvb-frontends/mn88472.h index 095294d292f3..323632523876 100644 --- a/drivers/media/dvb-frontends/mn88472.h +++ b/drivers/media/dvb-frontends/mn88472.h @@ -19,23 +19,33 @@ #include -enum ts_clock { - VARIABLE_TS_CLOCK, - FIXED_TS_CLOCK, -}; +/** + * struct mn88472_config - Platform data for the mn88472 driver + * @xtal: Clock frequency. + * @ts_mode: TS mode. + * @ts_clock: TS clock config. + * @i2c_wr_max: Max number of bytes driver writes to I2C at once. + * @get_dvb_frontend: Get DVB frontend. + */ -enum ts_mode { - SERIAL_TS_MODE, - PARALLEL_TS_MODE, -}; +/* Define old names for backward compatibility */ +#define VARIABLE_TS_CLOCK MN88472_TS_CLK_VARIABLE +#define FIXED_TS_CLOCK MN88472_TS_CLK_FIXED +#define SERIAL_TS_MODE MN88472_TS_MODE_SERIAL +#define PARALLEL_TS_MODE MN88472_TS_MODE_PARALLEL struct mn88472_config { - /* - * Max num of bytes given I2C adapter could write at once. - * Default: none - */ - u16 i2c_wr_max; + unsigned int xtal; + +#define MN88472_TS_MODE_SERIAL 0 +#define MN88472_TS_MODE_PARALLEL 1 + int ts_mode; +#define MN88472_TS_CLK_FIXED 0 +#define MN88472_TS_CLK_VARIABLE 1 + int ts_clock; + + u16 i2c_wr_max; /* Everything after that is returned by the driver. */ @@ -43,14 +53,7 @@ struct mn88472_config { * DVB frontend. */ struct dvb_frontend **fe; - - /* - * Xtal frequency. - * Hz - */ - u32 xtal; - int ts_mode; - int ts_clock; + struct dvb_frontend* (*get_dvb_frontend)(struct i2c_client *); }; #endif diff --git a/drivers/staging/media/mn88472/mn88472_priv.h b/drivers/media/dvb-frontends/mn88472_priv.h similarity index 88% rename from drivers/staging/media/mn88472/mn88472_priv.h rename to drivers/media/dvb-frontends/mn88472_priv.h index 1a0de9e46b66..cdf2597a25d1 100644 --- a/drivers/staging/media/mn88472/mn88472_priv.h +++ b/drivers/media/dvb-frontends/mn88472_priv.h @@ -28,12 +28,11 @@ struct mn88472_dev { struct i2c_client *client[3]; struct regmap *regmap[3]; struct dvb_frontend fe; - u16 i2c_wr_max; - enum fe_delivery_system delivery_system; - bool warm; /* FW running */ - u32 xtal; - int ts_mode; - int ts_clock; + u16 i2c_write_max; + unsigned int clk; + unsigned int active:1; + unsigned int ts_mode:1; + unsigned int ts_clk:1; }; #endif diff --git a/drivers/media/dvb-frontends/mn88473.c b/drivers/media/dvb-frontends/mn88473.c index 6c5d592161d4..451974a1d7ed 100644 --- a/drivers/media/dvb-frontends/mn88473.c +++ b/drivers/media/dvb-frontends/mn88473.c @@ -330,7 +330,7 @@ static int mn88473_init(struct dvb_frontend *fe) /* Request the firmware, this will block and timeout */ ret = request_firmware(&fw, name, &client->dev); if (ret) { - dev_err(&client->dev, "firmare file '%s' not found\n", name); + dev_err(&client->dev, "firmware file '%s' not found\n", name); goto err; } @@ -536,7 +536,7 @@ static int mn88473_probe(struct i2c_client *client, /* Sleep because chip is active by default */ ret = regmap_write(dev->regmap[2], 0x05, 0x3e); if (ret) - goto err_client_2_i2c_unregister_device; + goto err_regmap_2_regmap_exit; /* Create dvb frontend */ memcpy(&dev->frontend.ops, &mn88473_ops, sizeof(dev->frontend.ops)); @@ -547,7 +547,8 @@ static int mn88473_probe(struct i2c_client *client, dev_info(&client->dev, "Panasonic MN88473 successfully identified\n"); return 0; - +err_regmap_2_regmap_exit: + regmap_exit(dev->regmap[2]); err_client_2_i2c_unregister_device: i2c_unregister_device(dev->client[2]); err_regmap_1_regmap_exit: diff --git a/drivers/media/dvb-frontends/rtl2830.c b/drivers/media/dvb-frontends/rtl2830.c index d25d1e0cd4ca..87226056f226 100644 --- a/drivers/media/dvb-frontends/rtl2830.c +++ b/drivers/media/dvb-frontends/rtl2830.c @@ -135,8 +135,6 @@ static int rtl2830_init(struct dvb_frontend *fe) c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; c->post_bit_count.len = 1; c->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - /* start statistics polling */ - schedule_delayed_work(&dev->stat_work, msecs_to_jiffies(2000)); dev->sleeping = false; @@ -152,8 +150,6 @@ static int rtl2830_sleep(struct dvb_frontend *fe) struct rtl2830_dev *dev = i2c_get_clientdata(client); dev->sleeping = true; - /* stop statistics polling */ - cancel_delayed_work_sync(&dev->stat_work); dev->fe_status = 0; return 0; @@ -396,8 +392,10 @@ static int rtl2830_read_status(struct dvb_frontend *fe, enum fe_status *status) { struct i2c_client *client = fe->demodulator_priv; struct rtl2830_dev *dev = i2c_get_clientdata(client); - int ret; - u8 u8tmp; + struct dtv_frontend_properties *c = &dev->fe.dtv_property_cache; + int ret, stmp; + unsigned int utmp; + u8 u8tmp, buf[2]; *status = 0; @@ -419,6 +417,89 @@ static int rtl2830_read_status(struct dvb_frontend *fe, enum fe_status *status) dev->fe_status = *status; + /* Signal strength */ + if (dev->fe_status & FE_HAS_SIGNAL) { + /* Read IF AGC */ + ret = rtl2830_bulk_read(client, 0x359, buf, 2); + if (ret) + goto err; + + stmp = buf[0] << 8 | buf[1] << 0; + stmp = sign_extend32(stmp, 13); + utmp = clamp_val(-4 * stmp + 32767, 0x0000, 0xffff); + + dev_dbg(&client->dev, "IF AGC=%d\n", stmp); + + c->strength.stat[0].scale = FE_SCALE_RELATIVE; + c->strength.stat[0].uvalue = utmp; + } else { + c->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + } + + /* CNR */ + if (dev->fe_status & FE_HAS_VITERBI) { + unsigned int hierarchy, constellation; + #define CONSTELLATION_NUM 3 + #define HIERARCHY_NUM 4 + static const u32 constant[CONSTELLATION_NUM][HIERARCHY_NUM] = { + {70705899, 70705899, 70705899, 70705899}, + {82433173, 82433173, 87483115, 94445660}, + {92888734, 92888734, 95487525, 99770748}, + }; + + ret = rtl2830_bulk_read(client, 0x33c, &u8tmp, 1); + if (ret) + goto err; + + constellation = (u8tmp >> 2) & 0x03; /* [3:2] */ + if (constellation > CONSTELLATION_NUM - 1) + goto err; + + hierarchy = (u8tmp >> 4) & 0x07; /* [6:4] */ + if (hierarchy > HIERARCHY_NUM - 1) + goto err; + + ret = rtl2830_bulk_read(client, 0x40c, buf, 2); + if (ret) + goto err; + + utmp = buf[0] << 8 | buf[1] << 0; + if (utmp) + stmp = (constant[constellation][hierarchy] - + intlog10(utmp)) / ((1 << 24) / 10000); + else + stmp = 0; + + dev_dbg(&client->dev, "CNR raw=%u\n", utmp); + + c->cnr.stat[0].scale = FE_SCALE_DECIBEL; + c->cnr.stat[0].svalue = stmp; + } else { + c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + } + + /* BER */ + if (dev->fe_status & FE_HAS_LOCK) { + ret = rtl2830_bulk_read(client, 0x34e, buf, 2); + if (ret) + goto err; + + utmp = buf[0] << 8 | buf[1] << 0; + dev->post_bit_error += utmp; + dev->post_bit_count += 1000000; + + dev_dbg(&client->dev, "BER errors=%u total=1000000\n", utmp); + + c->post_bit_error.stat[0].scale = FE_SCALE_COUNTER; + c->post_bit_error.stat[0].uvalue = dev->post_bit_error; + c->post_bit_count.stat[0].scale = FE_SCALE_COUNTER; + c->post_bit_count.stat[0].uvalue = dev->post_bit_count; + } else { + c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + c->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; + } + + return ret; err: dev_dbg(&client->dev, "failed=%d\n", ret); @@ -503,109 +584,6 @@ static struct dvb_frontend_ops rtl2830_ops = { .read_signal_strength = rtl2830_read_signal_strength, }; -static void rtl2830_stat_work(struct work_struct *work) -{ - struct rtl2830_dev *dev = container_of(work, struct rtl2830_dev, stat_work.work); - struct i2c_client *client = dev->client; - struct dtv_frontend_properties *c = &dev->fe.dtv_property_cache; - int ret, tmp; - u8 u8tmp, buf[2]; - u16 u16tmp; - - dev_dbg(&client->dev, "\n"); - - /* signal strength */ - if (dev->fe_status & FE_HAS_SIGNAL) { - struct {signed int x:14; } s; - - /* read IF AGC */ - ret = rtl2830_bulk_read(client, 0x359, buf, 2); - if (ret) - goto err; - - u16tmp = buf[0] << 8 | buf[1] << 0; - u16tmp &= 0x3fff; /* [13:0] */ - tmp = s.x = u16tmp; /* 14-bit bin to 2 complement */ - u16tmp = clamp_val(-4 * tmp + 32767, 0x0000, 0xffff); - - dev_dbg(&client->dev, "IF AGC=%d\n", tmp); - - c->strength.stat[0].scale = FE_SCALE_RELATIVE; - c->strength.stat[0].uvalue = u16tmp; - } else { - c->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - } - - /* CNR */ - if (dev->fe_status & FE_HAS_VITERBI) { - unsigned hierarchy, constellation; - #define CONSTELLATION_NUM 3 - #define HIERARCHY_NUM 4 - static const u32 constant[CONSTELLATION_NUM][HIERARCHY_NUM] = { - {70705899, 70705899, 70705899, 70705899}, - {82433173, 82433173, 87483115, 94445660}, - {92888734, 92888734, 95487525, 99770748}, - }; - - ret = rtl2830_bulk_read(client, 0x33c, &u8tmp, 1); - if (ret) - goto err; - - constellation = (u8tmp >> 2) & 0x03; /* [3:2] */ - if (constellation > CONSTELLATION_NUM - 1) - goto err_schedule_delayed_work; - - hierarchy = (u8tmp >> 4) & 0x07; /* [6:4] */ - if (hierarchy > HIERARCHY_NUM - 1) - goto err_schedule_delayed_work; - - ret = rtl2830_bulk_read(client, 0x40c, buf, 2); - if (ret) - goto err; - - u16tmp = buf[0] << 8 | buf[1] << 0; - if (u16tmp) - tmp = (constant[constellation][hierarchy] - - intlog10(u16tmp)) / ((1 << 24) / 10000); - else - tmp = 0; - - dev_dbg(&client->dev, "CNR raw=%u\n", u16tmp); - - c->cnr.stat[0].scale = FE_SCALE_DECIBEL; - c->cnr.stat[0].svalue = tmp; - } else { - c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - } - - /* BER */ - if (dev->fe_status & FE_HAS_LOCK) { - ret = rtl2830_bulk_read(client, 0x34e, buf, 2); - if (ret) - goto err; - - u16tmp = buf[0] << 8 | buf[1] << 0; - dev->post_bit_error += u16tmp; - dev->post_bit_count += 1000000; - - dev_dbg(&client->dev, "BER errors=%u total=1000000\n", u16tmp); - - c->post_bit_error.stat[0].scale = FE_SCALE_COUNTER; - c->post_bit_error.stat[0].uvalue = dev->post_bit_error; - c->post_bit_count.stat[0].scale = FE_SCALE_COUNTER; - c->post_bit_count.stat[0].uvalue = dev->post_bit_count; - } else { - c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - c->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE; - } - -err_schedule_delayed_work: - schedule_delayed_work(&dev->stat_work, msecs_to_jiffies(2000)); - return; -err: - dev_dbg(&client->dev, "failed=%d\n", ret); -} - static int rtl2830_pid_filter_ctrl(struct dvb_frontend *fe, int onoff) { struct i2c_client *client = fe->demodulator_priv; @@ -851,7 +829,6 @@ static int rtl2830_probe(struct i2c_client *client, dev->client = client; dev->pdata = client->dev.platform_data; dev->sleeping = true; - INIT_DELAYED_WORK(&dev->stat_work, rtl2830_stat_work); dev->regmap = regmap_init(&client->dev, ®map_bus, client, ®map_config); if (IS_ERR(dev->regmap)) { @@ -904,9 +881,6 @@ static int rtl2830_remove(struct i2c_client *client) dev_dbg(&client->dev, "\n"); - /* stop statistics polling */ - cancel_delayed_work_sync(&dev->stat_work); - i2c_mux_del_adapters(dev->muxc); regmap_exit(dev->regmap); kfree(dev); @@ -922,7 +896,8 @@ MODULE_DEVICE_TABLE(i2c, rtl2830_id_table); static struct i2c_driver rtl2830_driver = { .driver = { - .name = "rtl2830", + .name = "rtl2830", + .suppress_bind_attrs = true, }, .probe = rtl2830_probe, .remove = rtl2830_remove, diff --git a/drivers/media/dvb-frontends/rtl2830_priv.h b/drivers/media/dvb-frontends/rtl2830_priv.h index da4909543da2..8ec4721d79ac 100644 --- a/drivers/media/dvb-frontends/rtl2830_priv.h +++ b/drivers/media/dvb-frontends/rtl2830_priv.h @@ -24,6 +24,7 @@ #include #include #include +#include struct rtl2830_dev { struct rtl2830_platform_data *pdata; @@ -33,7 +34,6 @@ struct rtl2830_dev { struct dvb_frontend fe; bool sleeping; unsigned long filters; - struct delayed_work stat_work; enum fe_status fe_status; u64 post_bit_error_prev; /* for old DVBv3 read_ber() calculation */ u64 post_bit_error; diff --git a/drivers/media/dvb-frontends/rtl2832.c b/drivers/media/dvb-frontends/rtl2832.c index bfb6beedd40b..0ced01f1012e 100644 --- a/drivers/media/dvb-frontends/rtl2832.c +++ b/drivers/media/dvb-frontends/rtl2832.c @@ -947,6 +947,8 @@ static int rtl2832_slave_ts_ctrl(struct i2c_client *client, bool enable) goto err; } + dev->slave_ts = enable; + return 0; err: dev_dbg(&client->dev, "failed=%d\n", ret); @@ -960,7 +962,7 @@ static int rtl2832_pid_filter_ctrl(struct dvb_frontend *fe, int onoff) int ret; u8 u8tmp; - dev_dbg(&client->dev, "onoff=%d\n", onoff); + dev_dbg(&client->dev, "onoff=%d, slave_ts=%d\n", onoff, dev->slave_ts); /* enable / disable PID filter */ if (onoff) @@ -968,7 +970,10 @@ static int rtl2832_pid_filter_ctrl(struct dvb_frontend *fe, int onoff) else u8tmp = 0x00; - ret = regmap_update_bits(dev->regmap, 0x061, 0xc0, u8tmp); + if (dev->slave_ts) + ret = regmap_update_bits(dev->regmap, 0x021, 0xc0, u8tmp); + else + ret = regmap_update_bits(dev->regmap, 0x061, 0xc0, u8tmp); if (ret) goto err; @@ -986,8 +991,8 @@ static int rtl2832_pid_filter(struct dvb_frontend *fe, u8 index, u16 pid, int ret; u8 buf[4]; - dev_dbg(&client->dev, "index=%d pid=%04x onoff=%d\n", - index, pid, onoff); + dev_dbg(&client->dev, "index=%d pid=%04x onoff=%d slave_ts=%d\n", + index, pid, onoff, dev->slave_ts); /* skip invalid PIDs (0x2000) */ if (pid > 0x1fff || index > 32) @@ -1003,14 +1008,22 @@ static int rtl2832_pid_filter(struct dvb_frontend *fe, u8 index, u16 pid, buf[1] = (dev->filters >> 8) & 0xff; buf[2] = (dev->filters >> 16) & 0xff; buf[3] = (dev->filters >> 24) & 0xff; - ret = regmap_bulk_write(dev->regmap, 0x062, buf, 4); + + if (dev->slave_ts) + ret = regmap_bulk_write(dev->regmap, 0x022, buf, 4); + else + ret = regmap_bulk_write(dev->regmap, 0x062, buf, 4); if (ret) goto err; /* add PID */ buf[0] = (pid >> 8) & 0xff; buf[1] = (pid >> 0) & 0xff; - ret = regmap_bulk_write(dev->regmap, 0x066 + 2 * index, buf, 2); + + if (dev->slave_ts) + ret = regmap_bulk_write(dev->regmap, 0x026 + 2 * index, buf, 2); + else + ret = regmap_bulk_write(dev->regmap, 0x066 + 2 * index, buf, 2); if (ret) goto err; @@ -1135,6 +1148,7 @@ MODULE_DEVICE_TABLE(i2c, rtl2832_id_table); static struct i2c_driver rtl2832_driver = { .driver = { .name = "rtl2832", + .suppress_bind_attrs = true, }, .probe = rtl2832_probe, .remove = rtl2832_remove, diff --git a/drivers/media/dvb-frontends/rtl2832_priv.h b/drivers/media/dvb-frontends/rtl2832_priv.h index c1a8a69e9015..9a6d01a9c690 100644 --- a/drivers/media/dvb-frontends/rtl2832_priv.h +++ b/drivers/media/dvb-frontends/rtl2832_priv.h @@ -44,6 +44,7 @@ struct rtl2832_dev { bool sleeping; struct delayed_work i2c_gate_work; unsigned long filters; /* PID filter */ + bool slave_ts; }; struct rtl2832_reg_entry { diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c index 47a480a7d46c..6e22af36b637 100644 --- a/drivers/media/dvb-frontends/rtl2832_sdr.c +++ b/drivers/media/dvb-frontends/rtl2832_sdr.c @@ -452,7 +452,7 @@ static int rtl2832_sdr_querycap(struct file *file, void *fh, /* Videobuf2 operations */ static int rtl2832_sdr_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, - unsigned int *nplanes, unsigned int sizes[], void *alloc_ctxs[]) + unsigned int *nplanes, unsigned int sizes[], struct device *alloc_devs[]) { struct rtl2832_sdr_dev *dev = vb2_get_drv_priv(vq); struct platform_device *pdev = dev->pdev; diff --git a/drivers/media/dvb-frontends/si2168.c b/drivers/media/dvb-frontends/si2168.c index 108a069fa1ae..20b4a659e2e4 100644 --- a/drivers/media/dvb-frontends/si2168.c +++ b/drivers/media/dvb-frontends/si2168.c @@ -357,9 +357,7 @@ static int si2168_init(struct dvb_frontend *fe) struct si2168_dev *dev = i2c_get_clientdata(client); int ret, len, remaining; const struct firmware *fw; - const char *fw_name; struct si2168_cmd cmd; - unsigned int chip_id; dev_dbg(&client->dev, "\n"); @@ -371,7 +369,7 @@ static int si2168_init(struct dvb_frontend *fe) if (ret) goto err; - if (dev->fw_loaded) { + if (dev->warm) { /* resume */ memcpy(cmd.args, "\xc0\x06\x08\x0f\x00\x20\x21\x01", 8); cmd.wlen = 8; @@ -398,49 +396,14 @@ static int si2168_init(struct dvb_frontend *fe) if (ret) goto err; - /* query chip revision */ - memcpy(cmd.args, "\x02", 1); - cmd.wlen = 1; - cmd.rlen = 13; - ret = si2168_cmd_execute(client, &cmd); - if (ret) - goto err; - - chip_id = cmd.args[1] << 24 | cmd.args[2] << 16 | cmd.args[3] << 8 | - cmd.args[4] << 0; - - #define SI2168_A20 ('A' << 24 | 68 << 16 | '2' << 8 | '0' << 0) - #define SI2168_A30 ('A' << 24 | 68 << 16 | '3' << 8 | '0' << 0) - #define SI2168_B40 ('B' << 24 | 68 << 16 | '4' << 8 | '0' << 0) - - switch (chip_id) { - case SI2168_A20: - fw_name = SI2168_A20_FIRMWARE; - break; - case SI2168_A30: - fw_name = SI2168_A30_FIRMWARE; - break; - case SI2168_B40: - fw_name = SI2168_B40_FIRMWARE; - break; - default: - dev_err(&client->dev, "unknown chip version Si21%d-%c%c%c\n", - cmd.args[2], cmd.args[1], - cmd.args[3], cmd.args[4]); - ret = -EINVAL; - goto err; - } - - dev_info(&client->dev, "found a 'Silicon Labs Si21%d-%c%c%c'\n", - cmd.args[2], cmd.args[1], cmd.args[3], cmd.args[4]); - /* request the firmware, this will block and timeout */ - ret = request_firmware(&fw, fw_name, &client->dev); + ret = request_firmware(&fw, dev->firmware_name, &client->dev); if (ret) { /* fallback mechanism to handle old name for Si2168 B40 fw */ - if (chip_id == SI2168_B40) { - fw_name = SI2168_B40_FIRMWARE_FALLBACK; - ret = request_firmware(&fw, fw_name, &client->dev); + if (dev->chip_id == SI2168_CHIP_ID_B40) { + dev->firmware_name = SI2168_B40_FIRMWARE_FALLBACK; + ret = request_firmware(&fw, dev->firmware_name, + &client->dev); } if (ret == 0) { @@ -450,13 +413,13 @@ static int si2168_init(struct dvb_frontend *fe) } else { dev_err(&client->dev, "firmware file '%s' not found\n", - fw_name); + dev->firmware_name); goto err_release_firmware; } } dev_info(&client->dev, "downloading firmware from file '%s'\n", - fw_name); + dev->firmware_name); if ((fw->size % 17 == 0) && (fw->data[0] > 5)) { /* firmware is in the new format */ @@ -511,8 +474,11 @@ static int si2168_init(struct dvb_frontend *fe) if (ret) goto err; - dev_info(&client->dev, "firmware version: %c.%c.%d\n", - cmd.args[6], cmd.args[7], cmd.args[8]); + dev->version = (cmd.args[9] + '@') << 24 | (cmd.args[6] - '0') << 16 | + (cmd.args[7] - '0') << 8 | (cmd.args[8]) << 0; + dev_info(&client->dev, "firmware version: %c %d.%d.%d\n", + dev->version >> 24 & 0xff, dev->version >> 16 & 0xff, + dev->version >> 8 & 0xff, dev->version >> 0 & 0xff); /* set ts mode */ memcpy(cmd.args, "\x14\x00\x01\x10\x10\x00", 6); @@ -525,7 +491,7 @@ static int si2168_init(struct dvb_frontend *fe) if (ret) goto err; - dev->fw_loaded = true; + dev->warm = true; warm: dev->active = true; @@ -549,6 +515,10 @@ static int si2168_sleep(struct dvb_frontend *fe) dev->active = false; + /* Firmware B 4.0-11 or later loses warm state during sleep */ + if (dev->version > ('B' << 24 | 4 << 16 | 0 << 8 | 11 << 0)) + dev->warm = false; + memcpy(cmd.args, "\x13", 1); cmd.wlen = 1; cmd.rlen = 0; @@ -653,6 +623,7 @@ static int si2168_probe(struct i2c_client *client, struct si2168_config *config = client->dev.platform_data; struct si2168_dev *dev; int ret; + struct si2168_cmd cmd; dev_dbg(&client->dev, "\n"); @@ -663,8 +634,56 @@ static int si2168_probe(struct i2c_client *client, goto err; } + i2c_set_clientdata(client, dev); mutex_init(&dev->i2c_mutex); + /* Initialize */ + memcpy(cmd.args, "\xc0\x12\x00\x0c\x00\x0d\x16\x00\x00\x00\x00\x00\x00", 13); + cmd.wlen = 13; + cmd.rlen = 0; + ret = si2168_cmd_execute(client, &cmd); + if (ret) + goto err_kfree; + + /* Power up */ + memcpy(cmd.args, "\xc0\x06\x01\x0f\x00\x20\x20\x01", 8); + cmd.wlen = 8; + cmd.rlen = 1; + ret = si2168_cmd_execute(client, &cmd); + if (ret) + goto err_kfree; + + /* Query chip revision */ + memcpy(cmd.args, "\x02", 1); + cmd.wlen = 1; + cmd.rlen = 13; + ret = si2168_cmd_execute(client, &cmd); + if (ret) + goto err_kfree; + + dev->chip_id = cmd.args[1] << 24 | cmd.args[2] << 16 | + cmd.args[3] << 8 | cmd.args[4] << 0; + + switch (dev->chip_id) { + case SI2168_CHIP_ID_A20: + dev->firmware_name = SI2168_A20_FIRMWARE; + break; + case SI2168_CHIP_ID_A30: + dev->firmware_name = SI2168_A30_FIRMWARE; + break; + case SI2168_CHIP_ID_B40: + dev->firmware_name = SI2168_B40_FIRMWARE; + break; + default: + dev_dbg(&client->dev, "unknown chip version Si21%d-%c%c%c\n", + cmd.args[2], cmd.args[1], cmd.args[3], cmd.args[4]); + ret = -ENODEV; + goto err_kfree; + } + + dev->version = (cmd.args[1]) << 24 | (cmd.args[3] - '0') << 16 | + (cmd.args[4] - '0') << 8 | (cmd.args[5]) << 0; + /* create mux i2c adapter for tuner */ dev->muxc = i2c_mux_alloc(client->adapter, &client->dev, 1, 0, I2C_MUX_LOCKED, @@ -686,11 +705,14 @@ static int si2168_probe(struct i2c_client *client, dev->ts_mode = config->ts_mode; dev->ts_clock_inv = config->ts_clock_inv; dev->ts_clock_gapped = config->ts_clock_gapped; - dev->fw_loaded = false; - i2c_set_clientdata(client, dev); + dev_info(&client->dev, "Silicon Labs Si2168-%c%d%d successfully identified\n", + dev->version >> 24 & 0xff, dev->version >> 16 & 0xff, + dev->version >> 8 & 0xff); + dev_info(&client->dev, "firmware version: %c %d.%d.%d\n", + dev->version >> 24 & 0xff, dev->version >> 16 & 0xff, + dev->version >> 8 & 0xff, dev->version >> 0 & 0xff); - dev_info(&client->dev, "Silicon Labs Si2168 successfully attached\n"); return 0; err_kfree: kfree(dev); @@ -723,7 +745,8 @@ MODULE_DEVICE_TABLE(i2c, si2168_id_table); static struct i2c_driver si2168_driver = { .driver = { - .name = "si2168", + .name = "si2168", + .suppress_bind_attrs = true, }, .probe = si2168_probe, .remove = si2168_remove, diff --git a/drivers/media/dvb-frontends/si2168_priv.h b/drivers/media/dvb-frontends/si2168_priv.h index 8a1f36d2014d..7843ccb448a0 100644 --- a/drivers/media/dvb-frontends/si2168_priv.h +++ b/drivers/media/dvb-frontends/si2168_priv.h @@ -34,8 +34,14 @@ struct si2168_dev { struct dvb_frontend fe; enum fe_delivery_system delivery_system; enum fe_status fe_status; + #define SI2168_CHIP_ID_A20 ('A' << 24 | 68 << 16 | '2' << 8 | '0' << 0) + #define SI2168_CHIP_ID_A30 ('A' << 24 | 68 << 16 | '3' << 8 | '0' << 0) + #define SI2168_CHIP_ID_B40 ('B' << 24 | 68 << 16 | '4' << 8 | '0' << 0) + unsigned int chip_id; + unsigned int version; + const char *firmware_name; bool active; - bool fw_loaded; + bool warm; u8 ts_mode; bool ts_clock_inv; bool ts_clock_gapped; diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig index 993dc50c12db..ce9006e10a30 100644 --- a/drivers/media/i2c/Kconfig +++ b/drivers/media/i2c/Kconfig @@ -209,6 +209,7 @@ config VIDEO_ADV7604 depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API depends on GPIOLIB || COMPILE_TEST select HDMI + select MEDIA_CEC_EDID ---help--- Support for the Analog Devices ADV7604 video decoder. @@ -218,10 +219,18 @@ config VIDEO_ADV7604 To compile this driver as a module, choose M here: the module will be called adv7604. +config VIDEO_ADV7604_CEC + bool "Enable Analog Devices ADV7604 CEC support" + depends on VIDEO_ADV7604 && MEDIA_CEC + ---help--- + When selected the adv7604 will support the optional + HDMI CEC feature. + config VIDEO_ADV7842 tristate "Analog Devices ADV7842 decoder" depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API select HDMI + select MEDIA_CEC_EDID ---help--- Support for the Analog Devices ADV7842 video decoder. @@ -231,6 +240,13 @@ config VIDEO_ADV7842 To compile this driver as a module, choose M here: the module will be called adv7842. +config VIDEO_ADV7842_CEC + bool "Enable Analog Devices ADV7842 CEC support" + depends on VIDEO_ADV7842 && MEDIA_CEC + ---help--- + When selected the adv7842 will support the optional + HDMI CEC feature. + config VIDEO_BT819 tristate "BT819A VideoStream decoder" depends on VIDEO_V4L2 && I2C @@ -447,6 +463,7 @@ config VIDEO_ADV7511 tristate "Analog Devices ADV7511 encoder" depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API select HDMI + select MEDIA_CEC_EDID ---help--- Support for the Analog Devices ADV7511 video encoder. @@ -455,6 +472,13 @@ config VIDEO_ADV7511 To compile this driver as a module, choose M here: the module will be called adv7511. +config VIDEO_ADV7511_CEC + bool "Enable Analog Devices ADV7511 CEC support" + depends on VIDEO_ADV7511 && MEDIA_CEC + ---help--- + When selected the adv7511 will support the optional + HDMI CEC feature. + config VIDEO_AD9389B tristate "Analog Devices AD9389B encoder" depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API diff --git a/drivers/media/i2c/adv7511.c b/drivers/media/i2c/adv7511.c index 39271c35da48..6d7cad54a65d 100644 --- a/drivers/media/i2c/adv7511.c +++ b/drivers/media/i2c/adv7511.c @@ -33,6 +33,7 @@ #include #include #include +#include static int debug; module_param(debug, int, 0644); @@ -59,6 +60,8 @@ MODULE_LICENSE("GPL v2"); #define ADV7511_MIN_PIXELCLOCK 20000000 #define ADV7511_MAX_PIXELCLOCK 225000000 +#define ADV7511_MAX_ADDRS (3) + /* ********************************************************************** * @@ -90,12 +93,20 @@ struct adv7511_state { struct v4l2_ctrl_handler hdl; int chip_revision; u8 i2c_edid_addr; - u8 i2c_cec_addr; u8 i2c_pktmem_addr; + u8 i2c_cec_addr; + + struct i2c_client *i2c_cec; + struct cec_adapter *cec_adap; + u8 cec_addr[ADV7511_MAX_ADDRS]; + u8 cec_valid_addrs; + bool cec_enabled_adap; + /* Is the adv7511 powered on? */ bool power_on; /* Did we receive hotplug and rx-sense signals? */ bool have_monitor; + bool enabled_irq; /* timings from s_dv_timings */ struct v4l2_dv_timings dv_timings; u32 fmt_code; @@ -227,7 +238,7 @@ static int adv_smbus_read_i2c_block_data(struct i2c_client *client, return ret; } -static inline void adv7511_edid_rd(struct v4l2_subdev *sd, u16 len, u8 *buf) +static void adv7511_edid_rd(struct v4l2_subdev *sd, uint16_t len, uint8_t *buf) { struct adv7511_state *state = get_adv7511_state(sd); int i; @@ -242,6 +253,34 @@ static inline void adv7511_edid_rd(struct v4l2_subdev *sd, u16 len, u8 *buf) v4l2_err(sd, "%s: i2c read error\n", __func__); } +static inline int adv7511_cec_read(struct v4l2_subdev *sd, u8 reg) +{ + struct adv7511_state *state = get_adv7511_state(sd); + + return i2c_smbus_read_byte_data(state->i2c_cec, reg); +} + +static int adv7511_cec_write(struct v4l2_subdev *sd, u8 reg, u8 val) +{ + struct adv7511_state *state = get_adv7511_state(sd); + int ret; + int i; + + for (i = 0; i < 3; i++) { + ret = i2c_smbus_write_byte_data(state->i2c_cec, reg, val); + if (ret == 0) + return 0; + } + v4l2_err(sd, "%s: I2C Write Problem\n", __func__); + return ret; +} + +static inline int adv7511_cec_write_and_or(struct v4l2_subdev *sd, u8 reg, u8 mask, + u8 val) +{ + return adv7511_cec_write(sd, reg, (adv7511_cec_read(sd, reg) & mask) | val); +} + static int adv7511_pktmem_rd(struct v4l2_subdev *sd, u8 reg) { struct adv7511_state *state = get_adv7511_state(sd); @@ -343,28 +382,20 @@ static void adv7511_csc_rgb_full2limit(struct v4l2_subdev *sd, bool enable) } } -static void adv7511_set_IT_content_AVI_InfoFrame(struct v4l2_subdev *sd) +static void adv7511_set_rgb_quantization_mode(struct v4l2_subdev *sd, struct v4l2_ctrl *ctrl) { struct adv7511_state *state = get_adv7511_state(sd); - if (state->dv_timings.bt.flags & V4L2_DV_FL_IS_CE_VIDEO) { - /* CE format, not IT */ - adv7511_wr_and_or(sd, 0x57, 0x7f, 0x00); - } else { - /* IT format */ - adv7511_wr_and_or(sd, 0x57, 0x7f, 0x80); + + /* Only makes sense for RGB formats */ + if (state->fmt_code != MEDIA_BUS_FMT_RGB888_1X24) { + /* so just keep quantization */ + adv7511_csc_rgb_full2limit(sd, false); + return; } -} -static int adv7511_set_rgb_quantization_mode(struct v4l2_subdev *sd, struct v4l2_ctrl *ctrl) -{ switch (ctrl->val) { - default: - return -EINVAL; - break; - case V4L2_DV_RGB_RANGE_AUTO: { + case V4L2_DV_RGB_RANGE_AUTO: /* automatic */ - struct adv7511_state *state = get_adv7511_state(sd); - if (state->dv_timings.bt.flags & V4L2_DV_FL_IS_CE_VIDEO) { /* CE format, RGB limited range (16-235) */ adv7511_csc_rgb_full2limit(sd, true); @@ -372,7 +403,6 @@ static int adv7511_set_rgb_quantization_mode(struct v4l2_subdev *sd, struct v4l2 /* not CE format, RGB full range (0-255) */ adv7511_csc_rgb_full2limit(sd, false); } - } break; case V4L2_DV_RGB_RANGE_LIMITED: /* RGB limited range (16-235) */ @@ -383,7 +413,6 @@ static int adv7511_set_rgb_quantization_mode(struct v4l2_subdev *sd, struct v4l2 adv7511_csc_rgb_full2limit(sd, false); break; } - return 0; } /* ------------------------------ CTRL OPS ------------------------------ */ @@ -400,8 +429,10 @@ static int adv7511_s_ctrl(struct v4l2_ctrl *ctrl) adv7511_wr_and_or(sd, 0xaf, 0xfd, ctrl->val == V4L2_DV_TX_MODE_HDMI ? 0x02 : 0x00); return 0; } - if (state->rgb_quantization_range_ctrl == ctrl) - return adv7511_set_rgb_quantization_mode(sd, ctrl); + if (state->rgb_quantization_range_ctrl == ctrl) { + adv7511_set_rgb_quantization_mode(sd, ctrl); + return 0; + } if (state->content_type_ctrl == ctrl) { u8 itc, cn; @@ -425,16 +456,28 @@ static const struct v4l2_ctrl_ops adv7511_ctrl_ops = { #ifdef CONFIG_VIDEO_ADV_DEBUG static void adv7511_inv_register(struct v4l2_subdev *sd) { + struct adv7511_state *state = get_adv7511_state(sd); + v4l2_info(sd, "0x000-0x0ff: Main Map\n"); + if (state->i2c_cec) + v4l2_info(sd, "0x100-0x1ff: CEC Map\n"); } static int adv7511_g_register(struct v4l2_subdev *sd, struct v4l2_dbg_register *reg) { + struct adv7511_state *state = get_adv7511_state(sd); + reg->size = 1; switch (reg->reg >> 8) { case 0: reg->val = adv7511_rd(sd, reg->reg & 0xff); break; + case 1: + if (state->i2c_cec) { + reg->val = adv7511_cec_read(sd, reg->reg & 0xff); + break; + } + /* fall through */ default: v4l2_info(sd, "Register %03llx not supported\n", reg->reg); adv7511_inv_register(sd); @@ -445,10 +488,18 @@ static int adv7511_g_register(struct v4l2_subdev *sd, struct v4l2_dbg_register * static int adv7511_s_register(struct v4l2_subdev *sd, const struct v4l2_dbg_register *reg) { + struct adv7511_state *state = get_adv7511_state(sd); + switch (reg->reg >> 8) { case 0: adv7511_wr(sd, reg->reg & 0xff, reg->val & 0xff); break; + case 1: + if (state->i2c_cec) { + adv7511_cec_write(sd, reg->reg & 0xff, reg->val & 0xff); + break; + } + /* fall through */ default: v4l2_info(sd, "Register %03llx not supported\n", reg->reg); adv7511_inv_register(sd); @@ -536,6 +587,7 @@ static int adv7511_log_status(struct v4l2_subdev *sd) { struct adv7511_state *state = get_adv7511_state(sd); struct adv7511_state_edid *edid = &state->edid; + int i; static const char * const states[] = { "in reset", @@ -605,7 +657,23 @@ static int adv7511_log_status(struct v4l2_subdev *sd) else v4l2_info(sd, "no timings set\n"); v4l2_info(sd, "i2c edid addr: 0x%x\n", state->i2c_edid_addr); + + if (state->i2c_cec == NULL) + return 0; + v4l2_info(sd, "i2c cec addr: 0x%x\n", state->i2c_cec_addr); + + v4l2_info(sd, "CEC: %s\n", state->cec_enabled_adap ? + "enabled" : "disabled"); + if (state->cec_enabled_adap) { + for (i = 0; i < ADV7511_MAX_ADDRS; i++) { + bool is_valid = state->cec_valid_addrs & (1 << i); + + if (is_valid) + v4l2_info(sd, "CEC Logical Address: 0x%x\n", + state->cec_addr[i]); + } + } v4l2_info(sd, "i2c pktmem addr: 0x%x\n", state->i2c_pktmem_addr); return 0; } @@ -663,15 +731,197 @@ static int adv7511_s_power(struct v4l2_subdev *sd, int on) return true; } +#if IS_ENABLED(CONFIG_VIDEO_ADV7511_CEC) +static int adv7511_cec_adap_enable(struct cec_adapter *adap, bool enable) +{ + struct adv7511_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + + if (state->i2c_cec == NULL) + return -EIO; + + if (!state->cec_enabled_adap && enable) { + /* power up cec section */ + adv7511_cec_write_and_or(sd, 0x4e, 0xfc, 0x01); + /* legacy mode and clear all rx buffers */ + adv7511_cec_write(sd, 0x4a, 0x07); + adv7511_cec_write(sd, 0x4a, 0); + adv7511_cec_write_and_or(sd, 0x11, 0xfe, 0); /* initially disable tx */ + /* enabled irqs: */ + /* tx: ready */ + /* tx: arbitration lost */ + /* tx: retry timeout */ + /* rx: ready 1 */ + if (state->enabled_irq) + adv7511_wr_and_or(sd, 0x95, 0xc0, 0x39); + } else if (state->cec_enabled_adap && !enable) { + if (state->enabled_irq) + adv7511_wr_and_or(sd, 0x95, 0xc0, 0x00); + /* disable address mask 1-3 */ + adv7511_cec_write_and_or(sd, 0x4b, 0x8f, 0x00); + /* power down cec section */ + adv7511_cec_write_and_or(sd, 0x4e, 0xfc, 0x00); + state->cec_valid_addrs = 0; + } + state->cec_enabled_adap = enable; + return 0; +} + +static int adv7511_cec_adap_log_addr(struct cec_adapter *adap, u8 addr) +{ + struct adv7511_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + unsigned int i, free_idx = ADV7511_MAX_ADDRS; + + if (!state->cec_enabled_adap) + return addr == CEC_LOG_ADDR_INVALID ? 0 : -EIO; + + if (addr == CEC_LOG_ADDR_INVALID) { + adv7511_cec_write_and_or(sd, 0x4b, 0x8f, 0); + state->cec_valid_addrs = 0; + return 0; + } + + for (i = 0; i < ADV7511_MAX_ADDRS; i++) { + bool is_valid = state->cec_valid_addrs & (1 << i); + + if (free_idx == ADV7511_MAX_ADDRS && !is_valid) + free_idx = i; + if (is_valid && state->cec_addr[i] == addr) + return 0; + } + if (i == ADV7511_MAX_ADDRS) { + i = free_idx; + if (i == ADV7511_MAX_ADDRS) + return -ENXIO; + } + state->cec_addr[i] = addr; + state->cec_valid_addrs |= 1 << i; + + switch (i) { + case 0: + /* enable address mask 0 */ + adv7511_cec_write_and_or(sd, 0x4b, 0xef, 0x10); + /* set address for mask 0 */ + adv7511_cec_write_and_or(sd, 0x4c, 0xf0, addr); + break; + case 1: + /* enable address mask 1 */ + adv7511_cec_write_and_or(sd, 0x4b, 0xdf, 0x20); + /* set address for mask 1 */ + adv7511_cec_write_and_or(sd, 0x4c, 0x0f, addr << 4); + break; + case 2: + /* enable address mask 2 */ + adv7511_cec_write_and_or(sd, 0x4b, 0xbf, 0x40); + /* set address for mask 1 */ + adv7511_cec_write_and_or(sd, 0x4d, 0xf0, addr); + break; + } + return 0; +} + +static int adv7511_cec_adap_transmit(struct cec_adapter *adap, u8 attempts, + u32 signal_free_time, struct cec_msg *msg) +{ + struct adv7511_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + u8 len = msg->len; + unsigned int i; + + v4l2_dbg(1, debug, sd, "%s: len %d\n", __func__, len); + + if (len > 16) { + v4l2_err(sd, "%s: len exceeded 16 (%d)\n", __func__, len); + return -EINVAL; + } + + /* + * The number of retries is the number of attempts - 1, but retry + * at least once. It's not clear if a value of 0 is allowed, so + * let's do at least one retry. + */ + adv7511_cec_write_and_or(sd, 0x12, ~0x70, max(1, attempts - 1) << 4); + + /* blocking, clear cec tx irq status */ + adv7511_wr_and_or(sd, 0x97, 0xc7, 0x38); + + /* write data */ + for (i = 0; i < len; i++) + adv7511_cec_write(sd, i, msg->msg[i]); + + /* set length (data + header) */ + adv7511_cec_write(sd, 0x10, len); + /* start transmit, enable tx */ + adv7511_cec_write(sd, 0x11, 0x01); + return 0; +} + +static void adv_cec_tx_raw_status(struct v4l2_subdev *sd, u8 tx_raw_status) +{ + struct adv7511_state *state = get_adv7511_state(sd); + + if ((adv7511_cec_read(sd, 0x11) & 0x01) == 0) { + v4l2_dbg(1, debug, sd, "%s: tx raw: tx disabled\n", __func__); + return; + } + + if (tx_raw_status & 0x10) { + v4l2_dbg(1, debug, sd, + "%s: tx raw: arbitration lost\n", __func__); + cec_transmit_done(state->cec_adap, CEC_TX_STATUS_ARB_LOST, + 1, 0, 0, 0); + return; + } + if (tx_raw_status & 0x08) { + u8 status; + u8 nack_cnt; + u8 low_drive_cnt; + + v4l2_dbg(1, debug, sd, "%s: tx raw: retry failed\n", __func__); + /* + * We set this status bit since this hardware performs + * retransmissions. + */ + status = CEC_TX_STATUS_MAX_RETRIES; + nack_cnt = adv7511_cec_read(sd, 0x14) & 0xf; + if (nack_cnt) + status |= CEC_TX_STATUS_NACK; + low_drive_cnt = adv7511_cec_read(sd, 0x14) >> 4; + if (low_drive_cnt) + status |= CEC_TX_STATUS_LOW_DRIVE; + cec_transmit_done(state->cec_adap, status, + 0, nack_cnt, low_drive_cnt, 0); + return; + } + if (tx_raw_status & 0x20) { + v4l2_dbg(1, debug, sd, "%s: tx raw: ready ok\n", __func__); + cec_transmit_done(state->cec_adap, CEC_TX_STATUS_OK, 0, 0, 0, 0); + return; + } +} + +static const struct cec_adap_ops adv7511_cec_adap_ops = { + .adap_enable = adv7511_cec_adap_enable, + .adap_log_addr = adv7511_cec_adap_log_addr, + .adap_transmit = adv7511_cec_adap_transmit, +}; +#endif + /* Enable interrupts */ static void adv7511_set_isr(struct v4l2_subdev *sd, bool enable) { + struct adv7511_state *state = get_adv7511_state(sd); u8 irqs = MASK_ADV7511_HPD_INT | MASK_ADV7511_MSEN_INT; u8 irqs_rd; int retries = 100; v4l2_dbg(2, debug, sd, "%s: %s\n", __func__, enable ? "enable" : "disable"); + if (state->enabled_irq == enable) + return; + state->enabled_irq = enable; + /* The datasheet says that the EDID ready interrupt should be disabled if there is no hotplug. */ if (!enable) @@ -679,6 +929,9 @@ static void adv7511_set_isr(struct v4l2_subdev *sd, bool enable) else if (adv7511_have_hotplug(sd)) irqs |= MASK_ADV7511_EDID_RDY_INT; + adv7511_wr_and_or(sd, 0x95, 0xc0, + (state->cec_enabled_adap && enable) ? 0x39 : 0x00); + /* * This i2c write can fail (approx. 1 in 1000 writes). But it * is essential that this register is correct, so retry it @@ -701,20 +954,53 @@ static void adv7511_set_isr(struct v4l2_subdev *sd, bool enable) static int adv7511_isr(struct v4l2_subdev *sd, u32 status, bool *handled) { u8 irq_status; + u8 cec_irq; /* disable interrupts to prevent a race condition */ adv7511_set_isr(sd, false); irq_status = adv7511_rd(sd, 0x96); + cec_irq = adv7511_rd(sd, 0x97); /* clear detected interrupts */ adv7511_wr(sd, 0x96, irq_status); + adv7511_wr(sd, 0x97, cec_irq); - v4l2_dbg(1, debug, sd, "%s: irq 0x%x\n", __func__, irq_status); + v4l2_dbg(1, debug, sd, "%s: irq 0x%x, cec-irq 0x%x\n", __func__, + irq_status, cec_irq); if (irq_status & (MASK_ADV7511_HPD_INT | MASK_ADV7511_MSEN_INT)) adv7511_check_monitor_present_status(sd); if (irq_status & MASK_ADV7511_EDID_RDY_INT) adv7511_check_edid_status(sd); +#if IS_ENABLED(CONFIG_VIDEO_ADV7511_CEC) + if (cec_irq & 0x38) + adv_cec_tx_raw_status(sd, cec_irq); + + if (cec_irq & 1) { + struct adv7511_state *state = get_adv7511_state(sd); + struct cec_msg msg; + + msg.len = adv7511_cec_read(sd, 0x25) & 0x1f; + + v4l2_dbg(1, debug, sd, "%s: cec msg len %d\n", __func__, + msg.len); + + if (msg.len > 16) + msg.len = 16; + + if (msg.len) { + u8 i; + + for (i = 0; i < msg.len; i++) + msg.msg[i] = adv7511_cec_read(sd, i + 0x15); + + adv7511_cec_write(sd, 0x4a, 1); /* toggle to re-enable rx 1 */ + adv7511_cec_write(sd, 0x4a, 0); + cec_received_msg(state->cec_adap, &msg); + } + } +#endif + /* enable interrupts */ adv7511_set_isr(sd, true); @@ -771,12 +1057,14 @@ static int adv7511_s_dv_timings(struct v4l2_subdev *sd, /* save timings */ state->dv_timings = *timings; + /* set h/vsync polarities */ + adv7511_wr_and_or(sd, 0x17, 0x9f, + ((timings->bt.polarities & V4L2_DV_VSYNC_POS_POL) ? 0 : 0x40) | + ((timings->bt.polarities & V4L2_DV_HSYNC_POS_POL) ? 0 : 0x20)); + /* update quantization range based on new dv_timings */ adv7511_set_rgb_quantization_mode(sd, state->rgb_quantization_range_ctrl); - /* update AVI infoframe */ - adv7511_set_IT_content_AVI_InfoFrame(sd); - return 0; } @@ -956,8 +1244,6 @@ static int adv7511_enum_mbus_code(struct v4l2_subdev *sd, static void adv7511_fill_format(struct adv7511_state *state, struct v4l2_mbus_framefmt *format) { - memset(format, 0, sizeof(*format)); - format->width = state->dv_timings.bt.width; format->height = state->dv_timings.bt.height; format->field = V4L2_FIELD_NONE; @@ -972,6 +1258,7 @@ static int adv7511_get_fmt(struct v4l2_subdev *sd, if (format->pad != 0) return -EINVAL; + memset(&format->format, 0, sizeof(format->format)); adv7511_fill_format(state, &format->format); if (format->which == V4L2_SUBDEV_FORMAT_TRY) { @@ -1132,6 +1419,7 @@ static int adv7511_set_fmt(struct v4l2_subdev *sd, adv7511_wr_and_or(sd, 0x57, 0x83, (ec << 4) | (q << 2) | (itc << 7)); adv7511_wr_and_or(sd, 0x59, 0x0f, (yq << 6) | (cn << 4)); adv7511_wr_and_or(sd, 0x4a, 0xff, 1); + adv7511_set_rgb_quantization_mode(sd, state->rgb_quantization_range_ctrl); return 0; } @@ -1183,6 +1471,8 @@ static void adv7511_notify_no_edid(struct v4l2_subdev *sd) /* We failed to read the EDID, so send an event for this. */ ed.present = false; ed.segment = adv7511_rd(sd, 0xc4); + ed.phys_addr = CEC_PHYS_ADDR_INVALID; + cec_s_phys_addr(state->cec_adap, ed.phys_addr, false); v4l2_subdev_notify(sd, ADV7511_EDID_DETECT, (void *)&ed); v4l2_ctrl_s_ctrl(state->have_edid0_ctrl, 0x0); } @@ -1406,13 +1696,16 @@ static bool adv7511_check_edid_status(struct v4l2_subdev *sd) v4l2_dbg(1, debug, sd, "%s: edid complete with %d segment(s)\n", __func__, state->edid.segments); state->edid.complete = true; - + ed.phys_addr = cec_get_edid_phys_addr(state->edid.data, + state->edid.segments * 256, + NULL); /* report when we have all segments but report only for segment 0 */ ed.present = true; ed.segment = 0; state->edid_detect_counter++; + cec_s_phys_addr(state->cec_adap, ed.phys_addr, false); v4l2_subdev_notify(sd, ADV7511_EDID_DETECT, (void *)&ed); return ed.present; } @@ -1420,17 +1713,43 @@ static bool adv7511_check_edid_status(struct v4l2_subdev *sd) return false; } +static int adv7511_registered(struct v4l2_subdev *sd) +{ + struct adv7511_state *state = get_adv7511_state(sd); + int err; + + err = cec_register_adapter(state->cec_adap); + if (err) + cec_delete_adapter(state->cec_adap); + return err; +} + +static void adv7511_unregistered(struct v4l2_subdev *sd) +{ + struct adv7511_state *state = get_adv7511_state(sd); + + cec_unregister_adapter(state->cec_adap); +} + +static const struct v4l2_subdev_internal_ops adv7511_int_ops = { + .registered = adv7511_registered, + .unregistered = adv7511_unregistered, +}; + /* ----------------------------------------------------------------------- */ /* Setup ADV7511 */ static void adv7511_init_setup(struct v4l2_subdev *sd) { struct adv7511_state *state = get_adv7511_state(sd); struct adv7511_state_edid *edid = &state->edid; + u32 cec_clk = state->pdata.cec_clk; + u8 ratio; v4l2_dbg(1, debug, sd, "%s\n", __func__); /* clear all interrupts */ adv7511_wr(sd, 0x96, 0xff); + adv7511_wr(sd, 0x97, 0xff); /* * Stop HPD from resetting a lot of registers. * It might leave the chip in a partly un-initialized state, @@ -1442,6 +1761,25 @@ static void adv7511_init_setup(struct v4l2_subdev *sd) adv7511_set_isr(sd, false); adv7511_s_stream(sd, false); adv7511_s_audio_stream(sd, false); + + if (state->i2c_cec == NULL) + return; + + v4l2_dbg(1, debug, sd, "%s: cec_clk %d\n", __func__, cec_clk); + + /* cec soft reset */ + adv7511_cec_write(sd, 0x50, 0x01); + adv7511_cec_write(sd, 0x50, 0x00); + + /* legacy mode */ + adv7511_cec_write(sd, 0x4a, 0x00); + + if (cec_clk % 750000 != 0) + v4l2_err(sd, "%s: cec_clk %d, not multiple of 750 Khz\n", + __func__, cec_clk); + + ratio = (cec_clk / 750000) - 1; + adv7511_cec_write(sd, 0x4e, ratio << 2); } static int adv7511_probe(struct i2c_client *client, const struct i2c_device_id *id) @@ -1476,6 +1814,7 @@ static int adv7511_probe(struct i2c_client *client, const struct i2c_device_id * client->addr << 1); v4l2_i2c_subdev_init(sd, client, &adv7511_ops); + sd->internal_ops = &adv7511_int_ops; hdl = &state->hdl; v4l2_ctrl_handler_init(hdl, 10); @@ -1516,26 +1855,47 @@ static int adv7511_probe(struct i2c_client *client, const struct i2c_device_id * chip_id[0] = adv7511_rd(sd, 0xf5); chip_id[1] = adv7511_rd(sd, 0xf6); if (chip_id[0] != 0x75 || chip_id[1] != 0x11) { - v4l2_err(sd, "chip_id != 0x7511, read 0x%02x%02x\n", chip_id[0], chip_id[1]); + v4l2_err(sd, "chip_id != 0x7511, read 0x%02x%02x\n", chip_id[0], + chip_id[1]); err = -EIO; goto err_entity; } - state->i2c_edid = i2c_new_dummy(client->adapter, state->i2c_edid_addr >> 1); + state->i2c_edid = i2c_new_dummy(client->adapter, + state->i2c_edid_addr >> 1); if (state->i2c_edid == NULL) { v4l2_err(sd, "failed to register edid i2c client\n"); err = -ENOMEM; goto err_entity; } + adv7511_wr(sd, 0xe1, state->i2c_cec_addr); + if (state->pdata.cec_clk < 3000000 || + state->pdata.cec_clk > 100000000) { + v4l2_err(sd, "%s: cec_clk %u outside range, disabling cec\n", + __func__, state->pdata.cec_clk); + state->pdata.cec_clk = 0; + } + + if (state->pdata.cec_clk) { + state->i2c_cec = i2c_new_dummy(client->adapter, + state->i2c_cec_addr >> 1); + if (state->i2c_cec == NULL) { + v4l2_err(sd, "failed to register cec i2c client\n"); + goto err_unreg_edid; + } + adv7511_wr(sd, 0xe2, 0x00); /* power up cec section */ + } else { + adv7511_wr(sd, 0xe2, 0x01); /* power down cec section */ + } + state->i2c_pktmem = i2c_new_dummy(client->adapter, state->i2c_pktmem_addr >> 1); if (state->i2c_pktmem == NULL) { v4l2_err(sd, "failed to register pktmem i2c client\n"); err = -ENOMEM; - goto err_unreg_edid; + goto err_unreg_cec; } - adv7511_wr(sd, 0xe2, 0x01); /* power down cec section */ state->work_queue = create_singlethread_workqueue(sd->name); if (state->work_queue == NULL) { v4l2_err(sd, "could not create workqueue\n"); @@ -1546,6 +1906,19 @@ static int adv7511_probe(struct i2c_client *client, const struct i2c_device_id * INIT_DELAYED_WORK(&state->edid_handler, adv7511_edid_handler); adv7511_init_setup(sd); + +#if IS_ENABLED(CONFIG_VIDEO_ADV7511_CEC) + state->cec_adap = cec_allocate_adapter(&adv7511_cec_adap_ops, + state, dev_name(&client->dev), CEC_CAP_TRANSMIT | + CEC_CAP_LOG_ADDRS | CEC_CAP_PASSTHROUGH | CEC_CAP_RC, + ADV7511_MAX_ADDRS, &client->dev); + err = PTR_ERR_OR_ZERO(state->cec_adap); + if (err) { + destroy_workqueue(state->work_queue); + goto err_unreg_pktmem; + } +#endif + adv7511_set_isr(sd, true); adv7511_check_monitor_present_status(sd); @@ -1555,6 +1928,9 @@ static int adv7511_probe(struct i2c_client *client, const struct i2c_device_id * err_unreg_pktmem: i2c_unregister_device(state->i2c_pktmem); +err_unreg_cec: + if (state->i2c_cec) + i2c_unregister_device(state->i2c_cec); err_unreg_edid: i2c_unregister_device(state->i2c_edid); err_entity: @@ -1576,9 +1952,12 @@ static int adv7511_remove(struct i2c_client *client) v4l2_dbg(1, debug, sd, "%s removed @ 0x%x (%s)\n", client->name, client->addr << 1, client->adapter->name); + adv7511_set_isr(sd, false); adv7511_init_setup(sd); cancel_delayed_work(&state->edid_handler); i2c_unregister_device(state->i2c_edid); + if (state->i2c_cec) + i2c_unregister_device(state->i2c_cec); i2c_unregister_device(state->i2c_pktmem); destroy_workqueue(state->work_queue); v4l2_device_unregister_subdev(sd); diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c index beb2841ceae5..4003831de712 100644 --- a/drivers/media/i2c/adv7604.c +++ b/drivers/media/i2c/adv7604.c @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -80,6 +81,8 @@ MODULE_LICENSE("GPL"); #define ADV76XX_OP_SWAP_CB_CR (1 << 0) +#define ADV76XX_MAX_ADDRS (3) + enum adv76xx_type { ADV7604, ADV7611, @@ -164,6 +167,7 @@ struct adv76xx_state { struct adv76xx_platform_data pdata; struct gpio_desc *hpd_gpio[4]; + struct gpio_desc *reset_gpio; struct v4l2_subdev sd; struct media_pad pads[ADV76XX_PAD_MAX]; @@ -184,10 +188,15 @@ struct adv76xx_state { u16 spa_port_a[2]; struct v4l2_fract aspect_ratio; u32 rgb_quantization_range; - struct workqueue_struct *work_queues; struct delayed_work delayed_work_enable_hotplug; bool restart_stdi_once; + /* CEC */ + struct cec_adapter *cec_adap; + u8 cec_addr[ADV76XX_MAX_ADDRS]; + u8 cec_valid_addrs; + bool cec_enabled_adap; + /* i2c clients */ struct i2c_client *i2c_clients[ADV76XX_PAGE_MAX]; @@ -381,7 +390,8 @@ static inline int io_write(struct v4l2_subdev *sd, u8 reg, u8 val) return regmap_write(state->regmap[ADV76XX_PAGE_IO], reg, val); } -static inline int io_write_clr_set(struct v4l2_subdev *sd, u8 reg, u8 mask, u8 val) +static inline int io_write_clr_set(struct v4l2_subdev *sd, u8 reg, u8 mask, + u8 val) { return io_write(sd, reg, (io_read(sd, reg) & ~mask) | val); } @@ -414,6 +424,12 @@ static inline int cec_write(struct v4l2_subdev *sd, u8 reg, u8 val) return regmap_write(state->regmap[ADV76XX_PAGE_CEC], reg, val); } +static inline int cec_write_clr_set(struct v4l2_subdev *sd, u8 reg, u8 mask, + u8 val) +{ + return cec_write(sd, reg, (cec_read(sd, reg) & ~mask) | val); +} + static inline int infoframe_read(struct v4l2_subdev *sd, u8 reg) { struct adv76xx_state *state = to_state(sd); @@ -779,11 +795,31 @@ static const struct v4l2_dv_timings_cap adv76xx_timings_cap_digital = { V4L2_DV_BT_CAP_CUSTOM) }; -static inline const struct v4l2_dv_timings_cap * -adv76xx_get_dv_timings_cap(struct v4l2_subdev *sd) +/* + * Return the DV timings capabilities for the requested sink pad. As a special + * case, pad value -1 returns the capabilities for the currently selected input. + */ +static const struct v4l2_dv_timings_cap * +adv76xx_get_dv_timings_cap(struct v4l2_subdev *sd, int pad) { - return is_digital_input(sd) ? &adv76xx_timings_cap_digital : - &adv7604_timings_cap_analog; + if (pad == -1) { + struct adv76xx_state *state = to_state(sd); + + pad = state->selected_input; + } + + switch (pad) { + case ADV76XX_PAD_HDMI_PORT_A: + case ADV7604_PAD_HDMI_PORT_B: + case ADV7604_PAD_HDMI_PORT_C: + case ADV7604_PAD_HDMI_PORT_D: + return &adv76xx_timings_cap_digital; + + case ADV7604_PAD_VGA_RGB: + case ADV7604_PAD_VGA_COMP: + default: + return &adv7604_timings_cap_analog; + } } @@ -872,9 +908,9 @@ static int adv76xx_s_detect_tx_5v_ctrl(struct v4l2_subdev *sd) { struct adv76xx_state *state = to_state(sd); const struct adv76xx_chip_info *info = state->info; + u16 cable_det = info->read_cable_det(sd); - return v4l2_ctrl_s_ctrl(state->detect_tx_5v_ctrl, - info->read_cable_det(sd)); + return v4l2_ctrl_s_ctrl(state->detect_tx_5v_ctrl, cable_det); } static int find_and_set_predefined_video_timings(struct v4l2_subdev *sd, @@ -1066,6 +1102,10 @@ static void set_rgb_quantization_range(struct v4l2_subdev *sd) struct adv76xx_state *state = to_state(sd); bool rgb_output = io_read(sd, 0x02) & 0x02; bool hdmi_signal = hdmi_read(sd, 0x05) & 0x80; + u8 y = HDMI_COLORSPACE_RGB; + + if (hdmi_signal && (io_read(sd, 0x60) & 1)) + y = infoframe_read(sd, 0x01) >> 5; v4l2_dbg(2, debug, sd, "%s: RGB quantization range: %d, RGB out: %d, HDMI: %d\n", __func__, state->rgb_quantization_range, @@ -1073,6 +1113,7 @@ static void set_rgb_quantization_range(struct v4l2_subdev *sd) adv76xx_set_gain(sd, true, 0x0, 0x0, 0x0); adv76xx_set_offset(sd, true, 0x0, 0x0, 0x0); + io_write_clr_set(sd, 0x02, 0x04, rgb_output ? 0 : 4); switch (state->rgb_quantization_range) { case V4L2_DV_RGB_RANGE_AUTO: @@ -1122,6 +1163,9 @@ static void set_rgb_quantization_range(struct v4l2_subdev *sd) break; } + if (y != HDMI_COLORSPACE_RGB) + break; + /* RGB limited range (16-235) */ io_write_clr_set(sd, 0x02, 0xf0, 0x00); @@ -1133,6 +1177,9 @@ static void set_rgb_quantization_range(struct v4l2_subdev *sd) break; } + if (y != HDMI_COLORSPACE_RGB) + break; + /* RGB full range (0-255) */ io_write_clr_set(sd, 0x02, 0xf0, 0x10); @@ -1329,7 +1376,7 @@ static int stdi2dv_timings(struct v4l2_subdev *sd, const struct v4l2_bt_timings *bt = &v4l2_dv_timings_presets[i].bt; if (!v4l2_valid_dv_timings(&v4l2_dv_timings_presets[i], - adv76xx_get_dv_timings_cap(sd), + adv76xx_get_dv_timings_cap(sd, -1), adv76xx_check_dv_timings, NULL)) continue; if (vtotal(bt) != stdi->lcf + 1) @@ -1430,18 +1477,22 @@ static int adv76xx_enum_dv_timings(struct v4l2_subdev *sd, return -EINVAL; return v4l2_enum_dv_timings_cap(timings, - adv76xx_get_dv_timings_cap(sd), adv76xx_check_dv_timings, NULL); + adv76xx_get_dv_timings_cap(sd, timings->pad), + adv76xx_check_dv_timings, NULL); } static int adv76xx_dv_timings_cap(struct v4l2_subdev *sd, struct v4l2_dv_timings_cap *cap) { struct adv76xx_state *state = to_state(sd); + unsigned int pad = cap->pad; if (cap->pad >= state->source_pad) return -EINVAL; - *cap = *adv76xx_get_dv_timings_cap(sd); + *cap = *adv76xx_get_dv_timings_cap(sd, pad); + cap->pad = pad; + return 0; } @@ -1450,9 +1501,9 @@ static int adv76xx_dv_timings_cap(struct v4l2_subdev *sd, static void adv76xx_fill_optional_dv_timings_fields(struct v4l2_subdev *sd, struct v4l2_dv_timings *timings) { - v4l2_find_dv_timings_cap(timings, adv76xx_get_dv_timings_cap(sd), - is_digital_input(sd) ? 250000 : 1000000, - adv76xx_check_dv_timings, NULL); + v4l2_find_dv_timings_cap(timings, adv76xx_get_dv_timings_cap(sd, -1), + is_digital_input(sd) ? 250000 : 1000000, + adv76xx_check_dv_timings, NULL); } static unsigned int adv7604_read_hdmi_pixelclock(struct v4l2_subdev *sd) @@ -1620,7 +1671,7 @@ static int adv76xx_s_dv_timings(struct v4l2_subdev *sd, bt = &timings->bt; - if (!v4l2_valid_dv_timings(timings, adv76xx_get_dv_timings_cap(sd), + if (!v4l2_valid_dv_timings(timings, adv76xx_get_dv_timings_cap(sd, -1), adv76xx_check_dv_timings, NULL)) return -ERANGE; @@ -1825,6 +1876,7 @@ static void adv76xx_setup_format(struct adv76xx_state *state) io_write_clr_set(sd, 0x04, 0xe0, adv76xx_op_ch_sel(state)); io_write_clr_set(sd, 0x05, 0x01, state->format->swap_cb_cr ? ADV76XX_OP_SWAP_CB_CR : 0); + set_rgb_quantization_range(sd); } static int adv76xx_get_format(struct v4l2_subdev *sd, @@ -1900,6 +1952,210 @@ static int adv76xx_set_format(struct v4l2_subdev *sd, return 0; } +#if IS_ENABLED(CONFIG_VIDEO_ADV7604_CEC) +static void adv76xx_cec_tx_raw_status(struct v4l2_subdev *sd, u8 tx_raw_status) +{ + struct adv76xx_state *state = to_state(sd); + + if ((cec_read(sd, 0x11) & 0x01) == 0) { + v4l2_dbg(1, debug, sd, "%s: tx raw: tx disabled\n", __func__); + return; + } + + if (tx_raw_status & 0x02) { + v4l2_dbg(1, debug, sd, "%s: tx raw: arbitration lost\n", + __func__); + cec_transmit_done(state->cec_adap, CEC_TX_STATUS_ARB_LOST, + 1, 0, 0, 0); + } + if (tx_raw_status & 0x04) { + u8 status; + u8 nack_cnt; + u8 low_drive_cnt; + + v4l2_dbg(1, debug, sd, "%s: tx raw: retry failed\n", __func__); + /* + * We set this status bit since this hardware performs + * retransmissions. + */ + status = CEC_TX_STATUS_MAX_RETRIES; + nack_cnt = cec_read(sd, 0x14) & 0xf; + if (nack_cnt) + status |= CEC_TX_STATUS_NACK; + low_drive_cnt = cec_read(sd, 0x14) >> 4; + if (low_drive_cnt) + status |= CEC_TX_STATUS_LOW_DRIVE; + cec_transmit_done(state->cec_adap, status, + 0, nack_cnt, low_drive_cnt, 0); + return; + } + if (tx_raw_status & 0x01) { + v4l2_dbg(1, debug, sd, "%s: tx raw: ready ok\n", __func__); + cec_transmit_done(state->cec_adap, CEC_TX_STATUS_OK, 0, 0, 0, 0); + return; + } +} + +static void adv76xx_cec_isr(struct v4l2_subdev *sd, bool *handled) +{ + struct adv76xx_state *state = to_state(sd); + u8 cec_irq; + + /* cec controller */ + cec_irq = io_read(sd, 0x4d) & 0x0f; + if (!cec_irq) + return; + + v4l2_dbg(1, debug, sd, "%s: cec: irq 0x%x\n", __func__, cec_irq); + adv76xx_cec_tx_raw_status(sd, cec_irq); + if (cec_irq & 0x08) { + struct cec_msg msg; + + msg.len = cec_read(sd, 0x25) & 0x1f; + if (msg.len > 16) + msg.len = 16; + + if (msg.len) { + u8 i; + + for (i = 0; i < msg.len; i++) + msg.msg[i] = cec_read(sd, i + 0x15); + cec_write(sd, 0x26, 0x01); /* re-enable rx */ + cec_received_msg(state->cec_adap, &msg); + } + } + + /* note: the bit order is swapped between 0x4d and 0x4e */ + cec_irq = ((cec_irq & 0x08) >> 3) | ((cec_irq & 0x04) >> 1) | + ((cec_irq & 0x02) << 1) | ((cec_irq & 0x01) << 3); + io_write(sd, 0x4e, cec_irq); + + if (handled) + *handled = true; +} + +static int adv76xx_cec_adap_enable(struct cec_adapter *adap, bool enable) +{ + struct adv76xx_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + + if (!state->cec_enabled_adap && enable) { + cec_write_clr_set(sd, 0x2a, 0x01, 0x01); /* power up cec */ + cec_write(sd, 0x2c, 0x01); /* cec soft reset */ + cec_write_clr_set(sd, 0x11, 0x01, 0); /* initially disable tx */ + /* enabled irqs: */ + /* tx: ready */ + /* tx: arbitration lost */ + /* tx: retry timeout */ + /* rx: ready */ + io_write_clr_set(sd, 0x50, 0x0f, 0x0f); + cec_write(sd, 0x26, 0x01); /* enable rx */ + } else if (state->cec_enabled_adap && !enable) { + /* disable cec interrupts */ + io_write_clr_set(sd, 0x50, 0x0f, 0x00); + /* disable address mask 1-3 */ + cec_write_clr_set(sd, 0x27, 0x70, 0x00); + /* power down cec section */ + cec_write_clr_set(sd, 0x2a, 0x01, 0x00); + state->cec_valid_addrs = 0; + } + state->cec_enabled_adap = enable; + adv76xx_s_detect_tx_5v_ctrl(sd); + return 0; +} + +static int adv76xx_cec_adap_log_addr(struct cec_adapter *adap, u8 addr) +{ + struct adv76xx_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + unsigned int i, free_idx = ADV76XX_MAX_ADDRS; + + if (!state->cec_enabled_adap) + return addr == CEC_LOG_ADDR_INVALID ? 0 : -EIO; + + if (addr == CEC_LOG_ADDR_INVALID) { + cec_write_clr_set(sd, 0x27, 0x70, 0); + state->cec_valid_addrs = 0; + return 0; + } + + for (i = 0; i < ADV76XX_MAX_ADDRS; i++) { + bool is_valid = state->cec_valid_addrs & (1 << i); + + if (free_idx == ADV76XX_MAX_ADDRS && !is_valid) + free_idx = i; + if (is_valid && state->cec_addr[i] == addr) + return 0; + } + if (i == ADV76XX_MAX_ADDRS) { + i = free_idx; + if (i == ADV76XX_MAX_ADDRS) + return -ENXIO; + } + state->cec_addr[i] = addr; + state->cec_valid_addrs |= 1 << i; + + switch (i) { + case 0: + /* enable address mask 0 */ + cec_write_clr_set(sd, 0x27, 0x10, 0x10); + /* set address for mask 0 */ + cec_write_clr_set(sd, 0x28, 0x0f, addr); + break; + case 1: + /* enable address mask 1 */ + cec_write_clr_set(sd, 0x27, 0x20, 0x20); + /* set address for mask 1 */ + cec_write_clr_set(sd, 0x28, 0xf0, addr << 4); + break; + case 2: + /* enable address mask 2 */ + cec_write_clr_set(sd, 0x27, 0x40, 0x40); + /* set address for mask 1 */ + cec_write_clr_set(sd, 0x29, 0x0f, addr); + break; + } + return 0; +} + +static int adv76xx_cec_adap_transmit(struct cec_adapter *adap, u8 attempts, + u32 signal_free_time, struct cec_msg *msg) +{ + struct adv76xx_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + u8 len = msg->len; + unsigned int i; + + /* + * The number of retries is the number of attempts - 1, but retry + * at least once. It's not clear if a value of 0 is allowed, so + * let's do at least one retry. + */ + cec_write_clr_set(sd, 0x12, 0x70, max(1, attempts - 1) << 4); + + if (len > 16) { + v4l2_err(sd, "%s: len exceeded 16 (%d)\n", __func__, len); + return -EINVAL; + } + + /* write data */ + for (i = 0; i < len; i++) + cec_write(sd, i, msg->msg[i]); + + /* set length (data + header) */ + cec_write(sd, 0x10, len); + /* start transmit, enable tx */ + cec_write(sd, 0x11, 0x01); + return 0; +} + +static const struct cec_adap_ops adv76xx_cec_adap_ops = { + .adap_enable = adv76xx_cec_adap_enable, + .adap_log_addr = adv76xx_cec_adap_log_addr, + .adap_transmit = adv76xx_cec_adap_transmit, +}; +#endif + static int adv76xx_isr(struct v4l2_subdev *sd, u32 status, bool *handled) { struct adv76xx_state *state = to_state(sd); @@ -1945,6 +2201,11 @@ static int adv76xx_isr(struct v4l2_subdev *sd, u32 status, bool *handled) *handled = true; } +#if IS_ENABLED(CONFIG_VIDEO_ADV7604_CEC) + /* cec */ + adv76xx_cec_isr(sd, handled); +#endif + /* tx 5v detect */ tx_5v = irq_reg_0x70 & info->cable_det_mask; if (tx_5v) { @@ -1994,39 +2255,12 @@ static int adv76xx_get_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid) return 0; } -static int get_edid_spa_location(const u8 *edid) -{ - u8 d; - - if ((edid[0x7e] != 1) || - (edid[0x80] != 0x02) || - (edid[0x81] != 0x03)) { - return -1; - } - - /* search Vendor Specific Data Block (tag 3) */ - d = edid[0x82] & 0x7f; - if (d > 4) { - int i = 0x84; - int end = 0x80 + d; - - do { - u8 tag = edid[i] >> 5; - u8 len = edid[i] & 0x1f; - - if ((tag == 3) && (len >= 5)) - return i + 4; - i += len + 1; - } while (i < end); - } - return -1; -} - static int adv76xx_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid) { struct adv76xx_state *state = to_state(sd); const struct adv76xx_chip_info *info = state->info; - int spa_loc; + unsigned int spa_loc; + u16 pa; int err; int i; @@ -2057,6 +2291,10 @@ static int adv76xx_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid) edid->blocks = 2; return -E2BIG; } + pa = cec_get_edid_phys_addr(edid->edid, edid->blocks * 128, &spa_loc); + err = cec_phys_addr_validate(pa, &pa, NULL); + if (err) + return err; v4l2_dbg(2, debug, sd, "%s: write EDID pad %d, edid.present = 0x%x\n", __func__, edid->pad, state->edid.present); @@ -2066,9 +2304,12 @@ static int adv76xx_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid) adv76xx_set_hpd(state, 0); rep_write_clr_set(sd, info->edid_enable_reg, 0x0f, 0x00); - spa_loc = get_edid_spa_location(edid->edid); - if (spa_loc < 0) - spa_loc = 0xc0; /* Default value [REF_02, p. 116] */ + /* + * Return an error if no location of the source physical address + * was found. + */ + if (spa_loc == 0) + return -EINVAL; switch (edid->pad) { case ADV76XX_PAD_HDMI_PORT_A: @@ -2128,10 +2369,10 @@ static int adv76xx_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid) v4l2_err(sd, "error enabling edid (0x%x)\n", state->edid.present); return -EIO; } + cec_s_phys_addr(state->cec_adap, pa, false); /* enable hotplug after 100 ms */ - queue_delayed_work(state->work_queues, - &state->delayed_work_enable_hotplug, HZ / 10); + schedule_delayed_work(&state->delayed_work_enable_hotplug, HZ / 10); return 0; } @@ -2252,8 +2493,19 @@ static int adv76xx_log_status(struct v4l2_subdev *sd) ((edid_enabled & 0x02) ? "Yes" : "No"), ((edid_enabled & 0x04) ? "Yes" : "No"), ((edid_enabled & 0x08) ? "Yes" : "No")); - v4l2_info(sd, "CEC: %s\n", !!(cec_read(sd, 0x2a) & 0x01) ? + v4l2_info(sd, "CEC: %s\n", state->cec_enabled_adap ? "enabled" : "disabled"); + if (state->cec_enabled_adap) { + int i; + + for (i = 0; i < ADV76XX_MAX_ADDRS; i++) { + bool is_valid = state->cec_valid_addrs & (1 << i); + + if (is_valid) + v4l2_info(sd, "CEC Logical Address: 0x%x\n", + state->cec_addr[i]); + } + } v4l2_info(sd, "-----Signal status-----\n"); cable_det = info->read_cable_det(sd); @@ -2299,11 +2551,10 @@ static int adv76xx_log_status(struct v4l2_subdev *sd) rgb_quantization_range_txt[state->rgb_quantization_range]); v4l2_info(sd, "Input color space: %s\n", input_color_space_txt[reg_io_0x02 >> 4]); - v4l2_info(sd, "Output color space: %s %s, saturator %s, alt-gamma %s\n", + v4l2_info(sd, "Output color space: %s %s, alt-gamma %s\n", (reg_io_0x02 & 0x02) ? "RGB" : "YCbCr", - (reg_io_0x02 & 0x04) ? "(16-235)" : "(0-255)", (((reg_io_0x02 >> 2) & 0x01) ^ (reg_io_0x02 & 0x01)) ? - "enabled" : "disabled", + "(16-235)" : "(0-255)", (reg_io_0x02 & 0x08) ? "enabled" : "disabled"); v4l2_info(sd, "Color space conversion: %s\n", csc_coeff_sel_rb[cp_read(sd, info->cp_csc) >> 4]); @@ -2363,6 +2614,24 @@ static int adv76xx_subscribe_event(struct v4l2_subdev *sd, } } +static int adv76xx_registered(struct v4l2_subdev *sd) +{ + struct adv76xx_state *state = to_state(sd); + int err; + + err = cec_register_adapter(state->cec_adap); + if (err) + cec_delete_adapter(state->cec_adap); + return err; +} + +static void adv76xx_unregistered(struct v4l2_subdev *sd) +{ + struct adv76xx_state *state = to_state(sd); + + cec_unregister_adapter(state->cec_adap); +} + /* ----------------------------------------------------------------------- */ static const struct v4l2_ctrl_ops adv76xx_ctrl_ops = { @@ -2406,6 +2675,11 @@ static const struct v4l2_subdev_ops adv76xx_ops = { .pad = &adv76xx_pad_ops, }; +static const struct v4l2_subdev_internal_ops adv76xx_int_ops = { + .registered = adv76xx_registered, + .unregistered = adv76xx_unregistered, +}; + /* -------------------------- custom ctrls ---------------------------------- */ static const struct v4l2_ctrl_config adv7604_ctrl_analog_sampling_phase = { @@ -2468,10 +2742,7 @@ static int adv76xx_core_init(struct v4l2_subdev *sd) cp_write(sd, 0xcf, 0x01); /* Power down macrovision */ /* video format */ - io_write_clr_set(sd, 0x02, 0x0f, - pdata->alt_gamma << 3 | - pdata->op_656_range << 2 | - pdata->alt_data_sat << 0); + io_write_clr_set(sd, 0x02, 0x0f, pdata->alt_gamma << 3); io_write_clr_set(sd, 0x05, 0x0e, pdata->blank_data << 3 | pdata->insert_av_codes << 2 | pdata->replicate_av_codes << 1); @@ -2821,10 +3092,8 @@ static int adv76xx_parse_dt(struct adv76xx_state *state) if (flags & V4L2_MBUS_PCLK_SAMPLE_RISING) state->pdata.inv_llc_pol = 1; - if (bus_cfg.bus_type == V4L2_MBUS_BT656) { + if (bus_cfg.bus_type == V4L2_MBUS_BT656) state->pdata.insert_av_codes = 1; - state->pdata.op_656_range = 1; - } /* Disable the interrupt for now as no DT-based board uses it. */ state->pdata.int1_config = ADV76XX_INT1_CONFIG_DISABLED; @@ -2847,7 +3116,6 @@ static int adv76xx_parse_dt(struct adv76xx_state *state) state->pdata.disable_pwrdnb = 0; state->pdata.disable_cable_det_rst = 0; state->pdata.blank_data = 1; - state->pdata.alt_data_sat = 1; state->pdata.op_format_mode_sel = ADV7604_OP_FORMAT_MODE0; state->pdata.bus_order = ADV7604_BUS_ORDER_RGB; @@ -2996,6 +3264,19 @@ static int configure_regmaps(struct adv76xx_state *state) return 0; } +static void adv76xx_reset(struct adv76xx_state *state) +{ + if (state->reset_gpio) { + /* ADV76XX can be reset by a low reset pulse of minimum 5 ms. */ + gpiod_set_value_cansleep(state->reset_gpio, 0); + usleep_range(5000, 10000); + gpiod_set_value_cansleep(state->reset_gpio, 1); + /* It is recommended to wait 5 ms after the low pulse before */ + /* an I2C write is performed to the ADV76XX. */ + usleep_range(5000, 10000); + } +} + static int adv76xx_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -3059,6 +3340,12 @@ static int adv76xx_probe(struct i2c_client *client, if (state->hpd_gpio[i]) v4l_info(client, "Handling HPD %u GPIO\n", i); } + state->reset_gpio = devm_gpiod_get_optional(&client->dev, "reset", + GPIOD_OUT_HIGH); + if (IS_ERR(state->reset_gpio)) + return PTR_ERR(state->reset_gpio); + + adv76xx_reset(state); state->timings = cea640x480; state->format = adv76xx_format_info(state, MEDIA_BUS_FMT_YUYV8_2X8); @@ -3069,6 +3356,7 @@ static int adv76xx_probe(struct i2c_client *client, id->name, i2c_adapter_id(client->adapter), client->addr); sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS; + sd->internal_ops = &adv76xx_int_ops; /* Configure IO Regmap region */ err = configure_regmap(state, ADV76XX_PAGE_IO); @@ -3182,14 +3470,6 @@ static int adv76xx_probe(struct i2c_client *client, } } - /* work queues */ - state->work_queues = create_singlethread_workqueue(client->name); - if (!state->work_queues) { - v4l2_err(sd, "Could not create work queue\n"); - err = -ENOMEM; - goto err_i2c; - } - INIT_DELAYED_WORK(&state->delayed_work_enable_hotplug, adv76xx_delayed_work_enable_hotplug); @@ -3212,6 +3492,18 @@ static int adv76xx_probe(struct i2c_client *client, err = adv76xx_core_init(sd); if (err) goto err_entity; + +#if IS_ENABLED(CONFIG_VIDEO_ADV7604_CEC) + state->cec_adap = cec_allocate_adapter(&adv76xx_cec_adap_ops, + state, dev_name(&client->dev), + CEC_CAP_TRANSMIT | CEC_CAP_LOG_ADDRS | + CEC_CAP_PASSTHROUGH | CEC_CAP_RC, ADV76XX_MAX_ADDRS, + &client->dev); + err = PTR_ERR_OR_ZERO(state->cec_adap); + if (err) + goto err_entity; +#endif + v4l2_info(sd, "%s found @ 0x%x (%s)\n", client->name, client->addr << 1, client->adapter->name); @@ -3225,7 +3517,6 @@ err_entity: media_entity_cleanup(&sd->entity); err_work_queues: cancel_delayed_work(&state->delayed_work_enable_hotplug); - destroy_workqueue(state->work_queues); err_i2c: adv76xx_unregister_clients(state); err_hdl: @@ -3240,8 +3531,14 @@ static int adv76xx_remove(struct i2c_client *client) struct v4l2_subdev *sd = i2c_get_clientdata(client); struct adv76xx_state *state = to_state(sd); + /* disable interrupts */ + io_write(sd, 0x40, 0); + io_write(sd, 0x41, 0); + io_write(sd, 0x46, 0); + io_write(sd, 0x6e, 0); + io_write(sd, 0x73, 0); + cancel_delayed_work(&state->delayed_work_enable_hotplug); - destroy_workqueue(state->work_queues); v4l2_async_unregister_subdev(sd); media_entity_cleanup(&sd->entity); adv76xx_unregister_clients(to_state(sd)); diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c index ecaacb0a6fa1..8c2a52e280af 100644 --- a/drivers/media/i2c/adv7842.c +++ b/drivers/media/i2c/adv7842.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -79,6 +80,8 @@ MODULE_LICENSE("GPL"); #define ADV7842_OP_SWAP_CB_CR (1 << 0) +#define ADV7842_MAX_ADDRS (3) + /* ********************************************************************** * @@ -118,7 +121,6 @@ struct adv7842_state { struct v4l2_fract aspect_ratio; u32 rgb_quantization_range; bool is_cea_format; - struct workqueue_struct *work_queues; struct delayed_work delayed_work_enable_hotplug; bool restart_stdi_once; bool hdmi_port_a; @@ -142,6 +144,11 @@ struct adv7842_state { struct v4l2_ctrl *free_run_color_ctrl_manual; struct v4l2_ctrl *free_run_color_ctrl; struct v4l2_ctrl *rgb_quantization_range_ctrl; + + struct cec_adapter *cec_adap; + u8 cec_addr[ADV7842_MAX_ADDRS]; + u8 cec_valid_addrs; + bool cec_enabled_adap; }; /* Unsupported timings. This device cannot support 720p30. */ @@ -418,9 +425,9 @@ static inline int cec_write(struct v4l2_subdev *sd, u8 reg, u8 val) return adv_smbus_write_byte_data(state->i2c_cec, reg, val); } -static inline int cec_write_and_or(struct v4l2_subdev *sd, u8 reg, u8 mask, u8 val) +static inline int cec_write_clr_set(struct v4l2_subdev *sd, u8 reg, u8 mask, u8 val) { - return cec_write(sd, reg, (cec_read(sd, reg) & mask) | val); + return cec_write(sd, reg, (cec_read(sd, reg) & ~mask) | val); } static inline int infoframe_read(struct v4l2_subdev *sd, u8 reg) @@ -696,6 +703,18 @@ adv7842_get_dv_timings_cap(struct v4l2_subdev *sd) /* ----------------------------------------------------------------------- */ +static u16 adv7842_read_cable_det(struct v4l2_subdev *sd) +{ + u8 reg = io_read(sd, 0x6f); + u16 val = 0; + + if (reg & 0x02) + val |= 1; /* port A */ + if (reg & 0x01) + val |= 2; /* port B */ + return val; +} + static void adv7842_delayed_work_enable_hotplug(struct work_struct *work) { struct delayed_work *dwork = to_delayed_work(work); @@ -756,56 +775,23 @@ static int edid_write_vga_segment(struct v4l2_subdev *sd) } /* enable hotplug after 200 ms */ - queue_delayed_work(state->work_queues, - &state->delayed_work_enable_hotplug, HZ / 5); + schedule_delayed_work(&state->delayed_work_enable_hotplug, HZ / 5); return 0; } -static int edid_spa_location(const u8 *edid) -{ - u8 d; - - /* - * TODO, improve and update for other CEA extensions - * currently only for 1 segment (256 bytes), - * i.e. 1 extension block and CEA revision 3. - */ - if ((edid[0x7e] != 1) || - (edid[0x80] != 0x02) || - (edid[0x81] != 0x03)) { - return -EINVAL; - } - /* - * search Vendor Specific Data Block (tag 3) - */ - d = edid[0x82] & 0x7f; - if (d > 4) { - int i = 0x84; - int end = 0x80 + d; - do { - u8 tag = edid[i]>>5; - u8 len = edid[i] & 0x1f; - - if ((tag == 3) && (len >= 5)) - return i + 4; - i += len + 1; - } while (i < end); - } - return -EINVAL; -} - static int edid_write_hdmi_segment(struct v4l2_subdev *sd, u8 port) { struct i2c_client *client = v4l2_get_subdevdata(sd); struct adv7842_state *state = to_state(sd); - const u8 *val = state->hdmi_edid.edid; - int spa_loc = edid_spa_location(val); + const u8 *edid = state->hdmi_edid.edid; + int spa_loc; + u16 pa; int err = 0; int i; - v4l2_dbg(2, debug, sd, "%s: write EDID on port %c (spa at 0x%x)\n", - __func__, (port == ADV7842_EDID_PORT_A) ? 'A' : 'B', spa_loc); + v4l2_dbg(2, debug, sd, "%s: write EDID on port %c\n", + __func__, (port == ADV7842_EDID_PORT_A) ? 'A' : 'B'); /* HPA disable on port A and B */ io_write_and_or(sd, 0x20, 0xcf, 0x00); @@ -816,24 +802,33 @@ static int edid_write_hdmi_segment(struct v4l2_subdev *sd, u8 port) if (!state->hdmi_edid.present) return 0; + pa = cec_get_edid_phys_addr(edid, 256, &spa_loc); + err = cec_phys_addr_validate(pa, &pa, NULL); + if (err) + return err; + + /* + * Return an error if no location of the source physical address + * was found. + */ + if (spa_loc == 0) + return -EINVAL; + /* edid segment pointer '0' for HDMI ports */ rep_write_and_or(sd, 0x77, 0xef, 0x00); for (i = 0; !err && i < 256; i += I2C_SMBUS_BLOCK_MAX) err = adv_smbus_write_i2c_block_data(state->i2c_edid, i, - I2C_SMBUS_BLOCK_MAX, val + i); + I2C_SMBUS_BLOCK_MAX, edid + i); if (err) return err; - if (spa_loc < 0) - spa_loc = 0xc0; /* Default value [REF_02, p. 199] */ - if (port == ADV7842_EDID_PORT_A) { - rep_write(sd, 0x72, val[spa_loc]); - rep_write(sd, 0x73, val[spa_loc + 1]); + rep_write(sd, 0x72, edid[spa_loc]); + rep_write(sd, 0x73, edid[spa_loc + 1]); } else { - rep_write(sd, 0x74, val[spa_loc]); - rep_write(sd, 0x75, val[spa_loc + 1]); + rep_write(sd, 0x74, edid[spa_loc]); + rep_write(sd, 0x75, edid[spa_loc + 1]); } rep_write(sd, 0x76, spa_loc & 0xff); rep_write_and_or(sd, 0x77, 0xbf, (spa_loc >> 2) & 0x40); @@ -853,10 +848,10 @@ static int edid_write_hdmi_segment(struct v4l2_subdev *sd, u8 port) (port == ADV7842_EDID_PORT_A) ? 'A' : 'B'); return -EIO; } + cec_s_phys_addr(state->cec_adap, pa, false); /* enable hotplug after 200 ms */ - queue_delayed_work(state->work_queues, - &state->delayed_work_enable_hotplug, HZ / 5); + schedule_delayed_work(&state->delayed_work_enable_hotplug, HZ / 5); return 0; } @@ -983,20 +978,11 @@ static int adv7842_s_register(struct v4l2_subdev *sd, static int adv7842_s_detect_tx_5v_ctrl(struct v4l2_subdev *sd) { struct adv7842_state *state = to_state(sd); - int prev = v4l2_ctrl_g_ctrl(state->detect_tx_5v_ctrl); - u8 reg_io_6f = io_read(sd, 0x6f); - int val = 0; + u16 cable_det = adv7842_read_cable_det(sd); - if (reg_io_6f & 0x02) - val |= 1; /* port A */ - if (reg_io_6f & 0x01) - val |= 2; /* port B */ - - v4l2_dbg(1, debug, sd, "%s: 0x%x -> 0x%x\n", __func__, prev, val); + v4l2_dbg(1, debug, sd, "%s: 0x%x\n", __func__, cable_det); - if (val != prev) - return v4l2_ctrl_s_ctrl(state->detect_tx_5v_ctrl, val); - return 0; + return v4l2_ctrl_s_ctrl(state->detect_tx_5v_ctrl, cable_det); } static int find_and_set_predefined_video_timings(struct v4l2_subdev *sd, @@ -1198,6 +1184,10 @@ static void set_rgb_quantization_range(struct v4l2_subdev *sd) struct adv7842_state *state = to_state(sd); bool rgb_output = io_read(sd, 0x02) & 0x02; bool hdmi_signal = hdmi_read(sd, 0x05) & 0x80; + u8 y = HDMI_COLORSPACE_RGB; + + if (hdmi_signal && (io_read(sd, 0x60) & 1)) + y = infoframe_read(sd, 0x01) >> 5; v4l2_dbg(2, debug, sd, "%s: RGB quantization range: %d, RGB out: %d, HDMI: %d\n", __func__, state->rgb_quantization_range, @@ -1205,6 +1195,7 @@ static void set_rgb_quantization_range(struct v4l2_subdev *sd) adv7842_set_gain(sd, true, 0x0, 0x0, 0x0); adv7842_set_offset(sd, true, 0x0, 0x0, 0x0); + io_write_clr_set(sd, 0x02, 0x04, rgb_output ? 0 : 4); switch (state->rgb_quantization_range) { case V4L2_DV_RGB_RANGE_AUTO: @@ -1254,6 +1245,9 @@ static void set_rgb_quantization_range(struct v4l2_subdev *sd) break; } + if (y != HDMI_COLORSPACE_RGB) + break; + /* RGB limited range (16-235) */ io_write_and_or(sd, 0x02, 0x0f, 0x00); @@ -1265,6 +1259,9 @@ static void set_rgb_quantization_range(struct v4l2_subdev *sd) break; } + if (y != HDMI_COLORSPACE_RGB) + break; + /* RGB full range (0-255) */ io_write_and_or(sd, 0x02, 0x0f, 0x10); @@ -2072,6 +2069,7 @@ static void adv7842_setup_format(struct adv7842_state *state) io_write_clr_set(sd, 0x04, 0xe0, adv7842_op_ch_sel(state)); io_write_clr_set(sd, 0x05, 0x01, state->format->swap_cb_cr ? ADV7842_OP_SWAP_CB_CR : 0); + set_rgb_quantization_range(sd); } static int adv7842_get_format(struct v4l2_subdev *sd, @@ -2170,6 +2168,207 @@ static void adv7842_irq_enable(struct v4l2_subdev *sd, bool enable) } } +#if IS_ENABLED(CONFIG_VIDEO_ADV7842_CEC) +static void adv7842_cec_tx_raw_status(struct v4l2_subdev *sd, u8 tx_raw_status) +{ + struct adv7842_state *state = to_state(sd); + + if ((cec_read(sd, 0x11) & 0x01) == 0) { + v4l2_dbg(1, debug, sd, "%s: tx raw: tx disabled\n", __func__); + return; + } + + if (tx_raw_status & 0x02) { + v4l2_dbg(1, debug, sd, "%s: tx raw: arbitration lost\n", + __func__); + cec_transmit_done(state->cec_adap, CEC_TX_STATUS_ARB_LOST, + 1, 0, 0, 0); + return; + } + if (tx_raw_status & 0x04) { + u8 status; + u8 nack_cnt; + u8 low_drive_cnt; + + v4l2_dbg(1, debug, sd, "%s: tx raw: retry failed\n", __func__); + /* + * We set this status bit since this hardware performs + * retransmissions. + */ + status = CEC_TX_STATUS_MAX_RETRIES; + nack_cnt = cec_read(sd, 0x14) & 0xf; + if (nack_cnt) + status |= CEC_TX_STATUS_NACK; + low_drive_cnt = cec_read(sd, 0x14) >> 4; + if (low_drive_cnt) + status |= CEC_TX_STATUS_LOW_DRIVE; + cec_transmit_done(state->cec_adap, status, + 0, nack_cnt, low_drive_cnt, 0); + return; + } + if (tx_raw_status & 0x01) { + v4l2_dbg(1, debug, sd, "%s: tx raw: ready ok\n", __func__); + cec_transmit_done(state->cec_adap, CEC_TX_STATUS_OK, 0, 0, 0, 0); + return; + } +} + +static void adv7842_cec_isr(struct v4l2_subdev *sd, bool *handled) +{ + u8 cec_irq; + + /* cec controller */ + cec_irq = io_read(sd, 0x93) & 0x0f; + if (!cec_irq) + return; + + v4l2_dbg(1, debug, sd, "%s: cec: irq 0x%x\n", __func__, cec_irq); + adv7842_cec_tx_raw_status(sd, cec_irq); + if (cec_irq & 0x08) { + struct adv7842_state *state = to_state(sd); + struct cec_msg msg; + + msg.len = cec_read(sd, 0x25) & 0x1f; + if (msg.len > 16) + msg.len = 16; + + if (msg.len) { + u8 i; + + for (i = 0; i < msg.len; i++) + msg.msg[i] = cec_read(sd, i + 0x15); + cec_write(sd, 0x26, 0x01); /* re-enable rx */ + cec_received_msg(state->cec_adap, &msg); + } + } + + io_write(sd, 0x94, cec_irq); + + if (handled) + *handled = true; +} + +static int adv7842_cec_adap_enable(struct cec_adapter *adap, bool enable) +{ + struct adv7842_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + + if (!state->cec_enabled_adap && enable) { + cec_write_clr_set(sd, 0x2a, 0x01, 0x01); /* power up cec */ + cec_write(sd, 0x2c, 0x01); /* cec soft reset */ + cec_write_clr_set(sd, 0x11, 0x01, 0); /* initially disable tx */ + /* enabled irqs: */ + /* tx: ready */ + /* tx: arbitration lost */ + /* tx: retry timeout */ + /* rx: ready */ + io_write_clr_set(sd, 0x96, 0x0f, 0x0f); + cec_write(sd, 0x26, 0x01); /* enable rx */ + } else if (state->cec_enabled_adap && !enable) { + /* disable cec interrupts */ + io_write_clr_set(sd, 0x96, 0x0f, 0x00); + /* disable address mask 1-3 */ + cec_write_clr_set(sd, 0x27, 0x70, 0x00); + /* power down cec section */ + cec_write_clr_set(sd, 0x2a, 0x01, 0x00); + state->cec_valid_addrs = 0; + } + state->cec_enabled_adap = enable; + return 0; +} + +static int adv7842_cec_adap_log_addr(struct cec_adapter *adap, u8 addr) +{ + struct adv7842_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + unsigned int i, free_idx = ADV7842_MAX_ADDRS; + + if (!state->cec_enabled_adap) + return addr == CEC_LOG_ADDR_INVALID ? 0 : -EIO; + + if (addr == CEC_LOG_ADDR_INVALID) { + cec_write_clr_set(sd, 0x27, 0x70, 0); + state->cec_valid_addrs = 0; + return 0; + } + + for (i = 0; i < ADV7842_MAX_ADDRS; i++) { + bool is_valid = state->cec_valid_addrs & (1 << i); + + if (free_idx == ADV7842_MAX_ADDRS && !is_valid) + free_idx = i; + if (is_valid && state->cec_addr[i] == addr) + return 0; + } + if (i == ADV7842_MAX_ADDRS) { + i = free_idx; + if (i == ADV7842_MAX_ADDRS) + return -ENXIO; + } + state->cec_addr[i] = addr; + state->cec_valid_addrs |= 1 << i; + + switch (i) { + case 0: + /* enable address mask 0 */ + cec_write_clr_set(sd, 0x27, 0x10, 0x10); + /* set address for mask 0 */ + cec_write_clr_set(sd, 0x28, 0x0f, addr); + break; + case 1: + /* enable address mask 1 */ + cec_write_clr_set(sd, 0x27, 0x20, 0x20); + /* set address for mask 1 */ + cec_write_clr_set(sd, 0x28, 0xf0, addr << 4); + break; + case 2: + /* enable address mask 2 */ + cec_write_clr_set(sd, 0x27, 0x40, 0x40); + /* set address for mask 1 */ + cec_write_clr_set(sd, 0x29, 0x0f, addr); + break; + } + return 0; +} + +static int adv7842_cec_adap_transmit(struct cec_adapter *adap, u8 attempts, + u32 signal_free_time, struct cec_msg *msg) +{ + struct adv7842_state *state = adap->priv; + struct v4l2_subdev *sd = &state->sd; + u8 len = msg->len; + unsigned int i; + + /* + * The number of retries is the number of attempts - 1, but retry + * at least once. It's not clear if a value of 0 is allowed, so + * let's do at least one retry. + */ + cec_write_clr_set(sd, 0x12, 0x70, max(1, attempts - 1) << 4); + + if (len > 16) { + v4l2_err(sd, "%s: len exceeded 16 (%d)\n", __func__, len); + return -EINVAL; + } + + /* write data */ + for (i = 0; i < len; i++) + cec_write(sd, i, msg->msg[i]); + + /* set length (data + header) */ + cec_write(sd, 0x10, len); + /* start transmit, enable tx */ + cec_write(sd, 0x11, 0x01); + return 0; +} + +static const struct cec_adap_ops adv7842_cec_adap_ops = { + .adap_enable = adv7842_cec_adap_enable, + .adap_log_addr = adv7842_cec_adap_log_addr, + .adap_transmit = adv7842_cec_adap_transmit, +}; +#endif + static int adv7842_isr(struct v4l2_subdev *sd, u32 status, bool *handled) { struct adv7842_state *state = to_state(sd); @@ -2241,6 +2440,11 @@ static int adv7842_isr(struct v4l2_subdev *sd, u32 status, bool *handled) *handled = true; } +#if IS_ENABLED(CONFIG_VIDEO_ADV7842_CEC) + /* cec */ + adv7842_cec_isr(sd, handled); +#endif + /* tx 5v detect */ if (irq_status[2] & 0x3) { v4l2_dbg(1, debug, sd, "%s: irq tx_5v\n", __func__); @@ -2321,10 +2525,12 @@ static int adv7842_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *e) case ADV7842_EDID_PORT_A: case ADV7842_EDID_PORT_B: memset(&state->hdmi_edid.edid, 0, 256); - if (e->blocks) + if (e->blocks) { state->hdmi_edid.present |= 0x04 << e->pad; - else + } else { state->hdmi_edid.present &= ~(0x04 << e->pad); + adv7842_s_detect_tx_5v_ctrl(sd); + } memcpy(&state->hdmi_edid.edid, e->edid, 128 * e->blocks); err = edid_write_hdmi_segment(sd, e->pad); break; @@ -2397,6 +2603,8 @@ static void adv7842_log_infoframes(struct v4l2_subdev *sd) log_infoframe(sd, &cri[i]); } +#if 0 +/* Let's keep it here for now, as it could be useful for debug */ static const char * const prim_mode_txt[] = { "SDP", "Component", @@ -2415,6 +2623,7 @@ static const char * const prim_mode_txt[] = { "Reserved", "Reserved", }; +#endif static int adv7842_sdp_log_status(struct v4l2_subdev *sd) { @@ -2509,8 +2718,19 @@ static int adv7842_cp_log_status(struct v4l2_subdev *sd) v4l2_info(sd, "HPD A %s, B %s\n", reg_io_0x21 & 0x02 ? "enabled" : "disabled", reg_io_0x21 & 0x01 ? "enabled" : "disabled"); - v4l2_info(sd, "CEC %s\n", !!(cec_read(sd, 0x2a) & 0x01) ? + v4l2_info(sd, "CEC: %s\n", state->cec_enabled_adap ? "enabled" : "disabled"); + if (state->cec_enabled_adap) { + int i; + + for (i = 0; i < ADV7842_MAX_ADDRS; i++) { + bool is_valid = state->cec_valid_addrs & (1 << i); + + if (is_valid) + v4l2_info(sd, "CEC Logical Address: 0x%x\n", + state->cec_addr[i]); + } + } v4l2_info(sd, "-----Signal status-----\n"); if (state->hdmi_port_a) { @@ -2569,11 +2789,11 @@ static int adv7842_cp_log_status(struct v4l2_subdev *sd) rgb_quantization_range_txt[state->rgb_quantization_range]); v4l2_info(sd, "Input color space: %s\n", input_color_space_txt[reg_io_0x02 >> 4]); - v4l2_info(sd, "Output color space: %s %s, saturator %s\n", + v4l2_info(sd, "Output color space: %s %s, alt-gamma %s\n", (reg_io_0x02 & 0x02) ? "RGB" : "YCbCr", - (reg_io_0x02 & 0x04) ? "(16-235)" : "(0-255)", - ((reg_io_0x02 & 0x04) ^ (reg_io_0x02 & 0x01)) ? - "enabled" : "disabled"); + (((reg_io_0x02 >> 2) & 0x01) ^ (reg_io_0x02 & 0x01)) ? + "(16-235)" : "(0-255)", + (reg_io_0x02 & 0x08) ? "enabled" : "disabled"); v4l2_info(sd, "Color space conversion: %s\n", csc_coeff_sel_rb[cp_read(sd, 0xf4) >> 4]); @@ -2777,11 +2997,7 @@ static int adv7842_core_init(struct v4l2_subdev *sd) io_write(sd, 0x15, 0x80); /* Power up pads */ /* video format */ - io_write(sd, 0x02, - 0xf0 | - pdata->alt_gamma << 3 | - pdata->op_656_range << 2 | - pdata->alt_data_sat << 0); + io_write(sd, 0x02, 0xf0 | pdata->alt_gamma << 3); io_write_and_or(sd, 0x05, 0xf0, pdata->blank_data << 3 | pdata->insert_av_codes << 2 | pdata->replicate_av_codes << 1); @@ -3031,6 +3247,24 @@ static int adv7842_subscribe_event(struct v4l2_subdev *sd, } } +static int adv7842_registered(struct v4l2_subdev *sd) +{ + struct adv7842_state *state = to_state(sd); + int err; + + err = cec_register_adapter(state->cec_adap); + if (err) + cec_delete_adapter(state->cec_adap); + return err; +} + +static void adv7842_unregistered(struct v4l2_subdev *sd) +{ + struct adv7842_state *state = to_state(sd); + + cec_unregister_adapter(state->cec_adap); +} + /* ----------------------------------------------------------------------- */ static const struct v4l2_ctrl_ops adv7842_ctrl_ops = { @@ -3077,6 +3311,11 @@ static const struct v4l2_subdev_ops adv7842_ops = { .pad = &adv7842_pad_ops, }; +static const struct v4l2_subdev_internal_ops adv7842_int_ops = { + .registered = adv7842_registered, + .unregistered = adv7842_unregistered, +}; + /* -------------------------- custom ctrls ---------------------------------- */ static const struct v4l2_ctrl_config adv7842_ctrl_analog_sampling_phase = { @@ -3241,6 +3480,7 @@ static int adv7842_probe(struct i2c_client *client, sd = &state->sd; v4l2_i2c_subdev_init(sd, client, &adv7842_ops); sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS; + sd->internal_ops = &adv7842_int_ops; state->mode = pdata->mode; state->hdmi_port_a = pdata->input == ADV7842_SELECT_HDMI_PORT_A; @@ -3311,13 +3551,6 @@ static int adv7842_probe(struct i2c_client *client, goto err_i2c; } - /* work queues */ - state->work_queues = create_singlethread_workqueue(client->name); - if (!state->work_queues) { - v4l2_err(sd, "Could not create work queue\n"); - err = -ENOMEM; - goto err_i2c; - } INIT_DELAYED_WORK(&state->delayed_work_enable_hotplug, adv7842_delayed_work_enable_hotplug); @@ -3331,6 +3564,17 @@ static int adv7842_probe(struct i2c_client *client, if (err) goto err_entity; +#if IS_ENABLED(CONFIG_VIDEO_ADV7842_CEC) + state->cec_adap = cec_allocate_adapter(&adv7842_cec_adap_ops, + state, dev_name(&client->dev), + CEC_CAP_TRANSMIT | CEC_CAP_LOG_ADDRS | + CEC_CAP_PASSTHROUGH | CEC_CAP_RC, ADV7842_MAX_ADDRS, + &client->dev); + err = PTR_ERR_OR_ZERO(state->cec_adap); + if (err) + goto err_entity; +#endif + v4l2_info(sd, "%s found @ 0x%x (%s)\n", client->name, client->addr << 1, client->adapter->name); return 0; @@ -3339,7 +3583,6 @@ err_entity: media_entity_cleanup(&sd->entity); err_work_queues: cancel_delayed_work(&state->delayed_work_enable_hotplug); - destroy_workqueue(state->work_queues); err_i2c: adv7842_unregister_clients(sd); err_hdl: @@ -3355,9 +3598,7 @@ static int adv7842_remove(struct i2c_client *client) struct adv7842_state *state = to_state(sd); adv7842_irq_enable(sd, false); - cancel_delayed_work(&state->delayed_work_enable_hotplug); - destroy_workqueue(state->work_queues); v4l2_device_unregister_subdev(sd); media_entity_cleanup(&sd->entity); adv7842_unregister_clients(sd); diff --git a/drivers/media/i2c/cs53l32a.c b/drivers/media/i2c/cs53l32a.c index b7e87e38642a..e4b3cf49dd38 100644 --- a/drivers/media/i2c/cs53l32a.c +++ b/drivers/media/i2c/cs53l32a.c @@ -121,13 +121,6 @@ static const struct v4l2_ctrl_ops cs53l32a_ctrl_ops = { static const struct v4l2_subdev_core_ops cs53l32a_core_ops = { .log_status = cs53l32a_log_status, - .g_ext_ctrls = v4l2_subdev_g_ext_ctrls, - .try_ext_ctrls = v4l2_subdev_try_ext_ctrls, - .s_ext_ctrls = v4l2_subdev_s_ext_ctrls, - .g_ctrl = v4l2_subdev_g_ctrl, - .s_ctrl = v4l2_subdev_s_ctrl, - .queryctrl = v4l2_subdev_queryctrl, - .querymenu = v4l2_subdev_querymenu, }; static const struct v4l2_subdev_audio_ops cs53l32a_audio_ops = { diff --git a/drivers/media/i2c/cx25840/cx25840-core.c b/drivers/media/i2c/cx25840/cx25840-core.c index 07a3e7173144..142ae28803bb 100644 --- a/drivers/media/i2c/cx25840/cx25840-core.c +++ b/drivers/media/i2c/cx25840/cx25840-core.c @@ -5042,13 +5042,6 @@ static const struct v4l2_ctrl_ops cx25840_ctrl_ops = { static const struct v4l2_subdev_core_ops cx25840_core_ops = { .log_status = cx25840_log_status, - .g_ctrl = v4l2_subdev_g_ctrl, - .s_ctrl = v4l2_subdev_s_ctrl, - .s_ext_ctrls = v4l2_subdev_s_ext_ctrls, - .try_ext_ctrls = v4l2_subdev_try_ext_ctrls, - .g_ext_ctrls = v4l2_subdev_g_ext_ctrls, - .queryctrl = v4l2_subdev_queryctrl, - .querymenu = v4l2_subdev_querymenu, .reset = cx25840_reset, .load_fw = cx25840_load_fw, .s_io_pin_config = common_s_io_pin_config, diff --git a/drivers/media/i2c/msp3400-driver.c b/drivers/media/i2c/msp3400-driver.c index e016626ebf89..503b7c4f0a9b 100644 --- a/drivers/media/i2c/msp3400-driver.c +++ b/drivers/media/i2c/msp3400-driver.c @@ -642,13 +642,6 @@ static const struct v4l2_ctrl_ops msp_ctrl_ops = { static const struct v4l2_subdev_core_ops msp_core_ops = { .log_status = msp_log_status, - .g_ext_ctrls = v4l2_subdev_g_ext_ctrls, - .try_ext_ctrls = v4l2_subdev_try_ext_ctrls, - .s_ext_ctrls = v4l2_subdev_s_ext_ctrls, - .g_ctrl = v4l2_subdev_g_ctrl, - .s_ctrl = v4l2_subdev_s_ctrl, - .queryctrl = v4l2_subdev_queryctrl, - .querymenu = v4l2_subdev_querymenu, }; static const struct v4l2_subdev_video_ops msp_video_ops = { diff --git a/drivers/media/i2c/mt9t001.c b/drivers/media/i2c/mt9t001.c index 702d562f8e39..842017fa4aab 100644 --- a/drivers/media/i2c/mt9t001.c +++ b/drivers/media/i2c/mt9t001.c @@ -233,10 +233,21 @@ static int __mt9t001_set_power(struct mt9t001 *mt9t001, bool on) ret = mt9t001_reset(mt9t001); if (ret < 0) { dev_err(&client->dev, "Failed to reset the camera\n"); - return ret; + goto e_power; } - return v4l2_ctrl_handler_setup(&mt9t001->ctrls); + ret = v4l2_ctrl_handler_setup(&mt9t001->ctrls); + if (ret < 0) { + dev_err(&client->dev, "Failed to set up control handlers\n"); + goto e_power; + } + + return 0; + +e_power: + mt9t001_power_off(mt9t001); + + return ret; } /* ----------------------------------------------------------------------------- @@ -834,7 +845,7 @@ static struct v4l2_subdev_ops mt9t001_subdev_ops = { .pad = &mt9t001_subdev_pad_ops, }; -static struct v4l2_subdev_internal_ops mt9t001_subdev_internal_ops = { +static const struct v4l2_subdev_internal_ops mt9t001_subdev_internal_ops = { .registered = mt9t001_registered, .open = mt9t001_open, .close = mt9t001_close, diff --git a/drivers/media/i2c/mt9v032.c b/drivers/media/i2c/mt9v032.c index 501b37039449..58eb62f1ba21 100644 --- a/drivers/media/i2c/mt9v032.c +++ b/drivers/media/i2c/mt9v032.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -133,9 +132,16 @@ #define MT9V032_TEST_PATTERN_GRAY_DIAGONAL (3 << 11) #define MT9V032_TEST_PATTERN_ENABLE (1 << 13) #define MT9V032_TEST_PATTERN_FLIP (1 << 14) +#define MT9V032_AEGC_DESIRED_BIN 0xa5 +#define MT9V032_AEC_UPDATE_FREQUENCY 0xa6 +#define MT9V032_AEC_LPF 0xa8 +#define MT9V032_AGC_UPDATE_FREQUENCY 0xa9 +#define MT9V032_AGC_LPF 0xaa #define MT9V032_AEC_AGC_ENABLE 0xaf #define MT9V032_AEC_ENABLE (1 << 0) #define MT9V032_AGC_ENABLE (1 << 1) +#define MT9V034_AEC_MAX_SHUTTER_WIDTH 0xad +#define MT9V032_AEC_MAX_SHUTTER_WIDTH 0xbd #define MT9V032_THERMAL_INFO 0xc1 enum mt9v032_model { @@ -162,6 +168,8 @@ struct mt9v032_model_data { unsigned int min_shutter; unsigned int max_shutter; unsigned int pclk_reg; + unsigned int aec_max_shutter_reg; + const struct v4l2_ctrl_config * const aec_max_shutter_v4l2_ctrl; }; struct mt9v032_model_info { @@ -175,63 +183,6 @@ static const struct mt9v032_model_version mt9v032_versions[] = { { MT9V034_CHIP_ID_REV1, "MT9V024/MT9V034 rev1" }, }; -static const struct mt9v032_model_data mt9v032_model_data[] = { - { - /* MT9V022, MT9V032 revisions 1/2/3 */ - .min_row_time = 660, - .min_hblank = MT9V032_HORIZONTAL_BLANKING_MIN, - .min_vblank = MT9V032_VERTICAL_BLANKING_MIN, - .max_vblank = MT9V032_VERTICAL_BLANKING_MAX, - .min_shutter = MT9V032_TOTAL_SHUTTER_WIDTH_MIN, - .max_shutter = MT9V032_TOTAL_SHUTTER_WIDTH_MAX, - .pclk_reg = MT9V032_PIXEL_CLOCK, - }, { - /* MT9V024, MT9V034 */ - .min_row_time = 690, - .min_hblank = MT9V034_HORIZONTAL_BLANKING_MIN, - .min_vblank = MT9V034_VERTICAL_BLANKING_MIN, - .max_vblank = MT9V034_VERTICAL_BLANKING_MAX, - .min_shutter = MT9V034_TOTAL_SHUTTER_WIDTH_MIN, - .max_shutter = MT9V034_TOTAL_SHUTTER_WIDTH_MAX, - .pclk_reg = MT9V034_PIXEL_CLOCK, - }, -}; - -static const struct mt9v032_model_info mt9v032_models[] = { - [MT9V032_MODEL_V022_COLOR] = { - .data = &mt9v032_model_data[0], - .color = true, - }, - [MT9V032_MODEL_V022_MONO] = { - .data = &mt9v032_model_data[0], - .color = false, - }, - [MT9V032_MODEL_V024_COLOR] = { - .data = &mt9v032_model_data[1], - .color = true, - }, - [MT9V032_MODEL_V024_MONO] = { - .data = &mt9v032_model_data[1], - .color = false, - }, - [MT9V032_MODEL_V032_COLOR] = { - .data = &mt9v032_model_data[0], - .color = true, - }, - [MT9V032_MODEL_V032_MONO] = { - .data = &mt9v032_model_data[0], - .color = false, - }, - [MT9V032_MODEL_V034_COLOR] = { - .data = &mt9v032_model_data[1], - .color = true, - }, - [MT9V032_MODEL_V034_MONO] = { - .data = &mt9v032_model_data[1], - .color = false, - }, -}; - struct mt9v032 { struct v4l2_subdev subdev; struct media_pad pad; @@ -349,7 +300,8 @@ static int mt9v032_power_on(struct mt9v032 *mt9v032) if (ret < 0) return ret; - return regmap_write(map, MT9V032_CHIP_CONTROL, 0); + return regmap_write(map, MT9V032_CHIP_CONTROL, + MT9V032_CHIP_CONTROL_MASTER_MODE); } static void mt9v032_power_off(struct mt9v032 *mt9v032) @@ -421,8 +373,7 @@ __mt9v032_get_pad_crop(struct mt9v032 *mt9v032, struct v4l2_subdev_pad_config *c static int mt9v032_s_stream(struct v4l2_subdev *subdev, int enable) { - const u16 mode = MT9V032_CHIP_CONTROL_MASTER_MODE - | MT9V032_CHIP_CONTROL_DOUT_ENABLE + const u16 mode = MT9V032_CHIP_CONTROL_DOUT_ENABLE | MT9V032_CHIP_CONTROL_SEQUENTIAL; struct mt9v032 *mt9v032 = to_mt9v032(subdev); struct v4l2_rect *crop = &mt9v032->crop; @@ -647,6 +598,34 @@ static int mt9v032_set_selection(struct v4l2_subdev *subdev, */ #define V4L2_CID_TEST_PATTERN_COLOR (V4L2_CID_USER_BASE | 0x1001) +/* + * Value between 1 and 64 to set the desired bin. This is effectively a measure + * of how bright the image is supposed to be. Both AGC and AEC try to reach + * this. + */ +#define V4L2_CID_AEGC_DESIRED_BIN (V4L2_CID_USER_BASE | 0x1002) +/* + * LPF is the low pass filter capability of the chip. Both AEC and AGC have + * this setting. This limits the speed in which AGC/AEC adjust their settings. + * Possible values are 0-2. 0 means no LPF. For 1 and 2 this equation is used: + * + * if |(calculated new exp - current exp)| > (current exp / 4) + * next exp = calculated new exp + * else + * next exp = current exp + ((calculated new exp - current exp) / 2^LPF) + */ +#define V4L2_CID_AEC_LPF (V4L2_CID_USER_BASE | 0x1003) +#define V4L2_CID_AGC_LPF (V4L2_CID_USER_BASE | 0x1004) +/* + * Value between 0 and 15. This is the number of frames being skipped before + * updating the auto exposure/gain. + */ +#define V4L2_CID_AEC_UPDATE_INTERVAL (V4L2_CID_USER_BASE | 0x1005) +#define V4L2_CID_AGC_UPDATE_INTERVAL (V4L2_CID_USER_BASE | 0x1006) +/* + * Maximum shutter width used for AEC. + */ +#define V4L2_CID_AEC_MAX_SHUTTER_WIDTH (V4L2_CID_USER_BASE | 0x1007) static int mt9v032_s_ctrl(struct v4l2_ctrl *ctrl) { @@ -716,6 +695,28 @@ static int mt9v032_s_ctrl(struct v4l2_ctrl *ctrl) break; } return regmap_write(map, MT9V032_TEST_PATTERN, data); + + case V4L2_CID_AEGC_DESIRED_BIN: + return regmap_write(map, MT9V032_AEGC_DESIRED_BIN, ctrl->val); + + case V4L2_CID_AEC_LPF: + return regmap_write(map, MT9V032_AEC_LPF, ctrl->val); + + case V4L2_CID_AGC_LPF: + return regmap_write(map, MT9V032_AGC_LPF, ctrl->val); + + case V4L2_CID_AEC_UPDATE_INTERVAL: + return regmap_write(map, MT9V032_AEC_UPDATE_FREQUENCY, + ctrl->val); + + case V4L2_CID_AGC_UPDATE_INTERVAL: + return regmap_write(map, MT9V032_AGC_UPDATE_FREQUENCY, + ctrl->val); + + case V4L2_CID_AEC_MAX_SHUTTER_WIDTH: + return regmap_write(map, + mt9v032->model->data->aec_max_shutter_reg, + ctrl->val); } return 0; @@ -745,6 +746,84 @@ static const struct v4l2_ctrl_config mt9v032_test_pattern_color = { .flags = 0, }; +static const struct v4l2_ctrl_config mt9v032_aegc_controls[] = { + { + .ops = &mt9v032_ctrl_ops, + .id = V4L2_CID_AEGC_DESIRED_BIN, + .type = V4L2_CTRL_TYPE_INTEGER, + .name = "AEC/AGC Desired Bin", + .min = 1, + .max = 64, + .step = 1, + .def = 58, + .flags = 0, + }, { + .ops = &mt9v032_ctrl_ops, + .id = V4L2_CID_AEC_LPF, + .type = V4L2_CTRL_TYPE_INTEGER, + .name = "AEC Low Pass Filter", + .min = 0, + .max = 2, + .step = 1, + .def = 0, + .flags = 0, + }, { + .ops = &mt9v032_ctrl_ops, + .id = V4L2_CID_AGC_LPF, + .type = V4L2_CTRL_TYPE_INTEGER, + .name = "AGC Low Pass Filter", + .min = 0, + .max = 2, + .step = 1, + .def = 2, + .flags = 0, + }, { + .ops = &mt9v032_ctrl_ops, + .id = V4L2_CID_AEC_UPDATE_INTERVAL, + .type = V4L2_CTRL_TYPE_INTEGER, + .name = "AEC Update Interval", + .min = 0, + .max = 16, + .step = 1, + .def = 2, + .flags = 0, + }, { + .ops = &mt9v032_ctrl_ops, + .id = V4L2_CID_AGC_UPDATE_INTERVAL, + .type = V4L2_CTRL_TYPE_INTEGER, + .name = "AGC Update Interval", + .min = 0, + .max = 16, + .step = 1, + .def = 2, + .flags = 0, + } +}; + +static const struct v4l2_ctrl_config mt9v032_aec_max_shutter_width = { + .ops = &mt9v032_ctrl_ops, + .id = V4L2_CID_AEC_MAX_SHUTTER_WIDTH, + .type = V4L2_CTRL_TYPE_INTEGER, + .name = "AEC Max Shutter Width", + .min = 1, + .max = 2047, + .step = 1, + .def = 480, + .flags = 0, +}; + +static const struct v4l2_ctrl_config mt9v034_aec_max_shutter_width = { + .ops = &mt9v032_ctrl_ops, + .id = V4L2_CID_AEC_MAX_SHUTTER_WIDTH, + .type = V4L2_CTRL_TYPE_INTEGER, + .name = "AEC Max Shutter Width", + .min = 1, + .max = 32765, + .step = 1, + .def = 480, + .flags = 0, +}; + /* ----------------------------------------------------------------------------- * V4L2 subdev core operations */ @@ -953,13 +1032,6 @@ static int mt9v032_probe(struct i2c_client *client, unsigned int i; int ret; - if (!i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_WORD_DATA)) { - dev_warn(&client->adapter->dev, - "I2C-Adapter doesn't support I2C_FUNC_SMBUS_WORD\n"); - return -EIO; - } - mt9v032 = devm_kzalloc(&client->dev, sizeof(*mt9v032), GFP_KERNEL); if (!mt9v032) return -ENOMEM; @@ -986,7 +1058,8 @@ static int mt9v032_probe(struct i2c_client *client, mt9v032->pdata = pdata; mt9v032->model = (const void *)did->driver_data; - v4l2_ctrl_handler_init(&mt9v032->ctrls, 10); + v4l2_ctrl_handler_init(&mt9v032->ctrls, 11 + + ARRAY_SIZE(mt9v032_aegc_controls)); v4l2_ctrl_new_std(&mt9v032->ctrls, &mt9v032_ctrl_ops, V4L2_CID_AUTOGAIN, 0, 1, 1, 1); @@ -1015,6 +1088,13 @@ static int mt9v032_probe(struct i2c_client *client, mt9v032->test_pattern_color = v4l2_ctrl_new_custom(&mt9v032->ctrls, &mt9v032_test_pattern_color, NULL); + v4l2_ctrl_new_custom(&mt9v032->ctrls, + mt9v032->model->data->aec_max_shutter_v4l2_ctrl, + NULL); + for (i = 0; i < ARRAY_SIZE(mt9v032_aegc_controls); ++i) + v4l2_ctrl_new_custom(&mt9v032->ctrls, &mt9v032_aegc_controls[i], + NULL); + v4l2_ctrl_cluster(2, &mt9v032->test_pattern); mt9v032->pixel_rate = @@ -1103,6 +1183,67 @@ static int mt9v032_remove(struct i2c_client *client) return 0; } +static const struct mt9v032_model_data mt9v032_model_data[] = { + { + /* MT9V022, MT9V032 revisions 1/2/3 */ + .min_row_time = 660, + .min_hblank = MT9V032_HORIZONTAL_BLANKING_MIN, + .min_vblank = MT9V032_VERTICAL_BLANKING_MIN, + .max_vblank = MT9V032_VERTICAL_BLANKING_MAX, + .min_shutter = MT9V032_TOTAL_SHUTTER_WIDTH_MIN, + .max_shutter = MT9V032_TOTAL_SHUTTER_WIDTH_MAX, + .pclk_reg = MT9V032_PIXEL_CLOCK, + .aec_max_shutter_reg = MT9V032_AEC_MAX_SHUTTER_WIDTH, + .aec_max_shutter_v4l2_ctrl = &mt9v032_aec_max_shutter_width, + }, { + /* MT9V024, MT9V034 */ + .min_row_time = 690, + .min_hblank = MT9V034_HORIZONTAL_BLANKING_MIN, + .min_vblank = MT9V034_VERTICAL_BLANKING_MIN, + .max_vblank = MT9V034_VERTICAL_BLANKING_MAX, + .min_shutter = MT9V034_TOTAL_SHUTTER_WIDTH_MIN, + .max_shutter = MT9V034_TOTAL_SHUTTER_WIDTH_MAX, + .pclk_reg = MT9V034_PIXEL_CLOCK, + .aec_max_shutter_reg = MT9V034_AEC_MAX_SHUTTER_WIDTH, + .aec_max_shutter_v4l2_ctrl = &mt9v034_aec_max_shutter_width, + }, +}; + +static const struct mt9v032_model_info mt9v032_models[] = { + [MT9V032_MODEL_V022_COLOR] = { + .data = &mt9v032_model_data[0], + .color = true, + }, + [MT9V032_MODEL_V022_MONO] = { + .data = &mt9v032_model_data[0], + .color = false, + }, + [MT9V032_MODEL_V024_COLOR] = { + .data = &mt9v032_model_data[1], + .color = true, + }, + [MT9V032_MODEL_V024_MONO] = { + .data = &mt9v032_model_data[1], + .color = false, + }, + [MT9V032_MODEL_V032_COLOR] = { + .data = &mt9v032_model_data[0], + .color = true, + }, + [MT9V032_MODEL_V032_MONO] = { + .data = &mt9v032_model_data[0], + .color = false, + }, + [MT9V032_MODEL_V034_COLOR] = { + .data = &mt9v032_model_data[1], + .color = true, + }, + [MT9V032_MODEL_V034_MONO] = { + .data = &mt9v032_model_data[1], + .color = false, + }, +}; + static const struct i2c_device_id mt9v032_id[] = { { "mt9v022", (kernel_ulong_t)&mt9v032_models[MT9V032_MODEL_V022_COLOR] }, { "mt9v022m", (kernel_ulong_t)&mt9v032_models[MT9V032_MODEL_V022_MONO] }, diff --git a/drivers/media/i2c/saa7115.c b/drivers/media/i2c/saa7115.c index bd3526bdd539..58062b41c923 100644 --- a/drivers/media/i2c/saa7115.c +++ b/drivers/media/i2c/saa7115.c @@ -1585,13 +1585,6 @@ static const struct v4l2_ctrl_ops saa711x_ctrl_ops = { static const struct v4l2_subdev_core_ops saa711x_core_ops = { .log_status = saa711x_log_status, - .g_ext_ctrls = v4l2_subdev_g_ext_ctrls, - .try_ext_ctrls = v4l2_subdev_try_ext_ctrls, - .s_ext_ctrls = v4l2_subdev_s_ext_ctrls, - .g_ctrl = v4l2_subdev_g_ctrl, - .s_ctrl = v4l2_subdev_s_ctrl, - .queryctrl = v4l2_subdev_queryctrl, - .querymenu = v4l2_subdev_querymenu, .reset = saa711x_reset, .s_gpio = saa711x_s_gpio, #ifdef CONFIG_VIDEO_ADV_DEBUG diff --git a/drivers/media/i2c/smiapp/smiapp-core.c b/drivers/media/i2c/smiapp/smiapp-core.c index 3dfe387abf6e..d08ab6c8357c 100644 --- a/drivers/media/i2c/smiapp/smiapp-core.c +++ b/drivers/media/i2c/smiapp/smiapp-core.c @@ -3044,10 +3044,8 @@ static struct smiapp_platform_data *smiapp_get_pdata(struct device *dev) pdata->op_sys_clock = devm_kcalloc( dev, bus_cfg->nr_of_link_frequencies + 1 /* guardian */, sizeof(*pdata->op_sys_clock), GFP_KERNEL); - if (!pdata->op_sys_clock) { - rval = -ENOMEM; + if (!pdata->op_sys_clock) goto out_err; - } for (i = 0; i < bus_cfg->nr_of_link_frequencies; i++) { pdata->op_sys_clock[i] = bus_cfg->link_frequencies[i]; diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c index 6cf6d06737a5..1e3a0dd2238c 100644 --- a/drivers/media/i2c/tc358743.c +++ b/drivers/media/i2c/tc358743.c @@ -89,8 +89,6 @@ struct tc358743_state { struct v4l2_ctrl *audio_sampling_rate_ctrl; struct v4l2_ctrl *audio_present_ctrl; - /* work queues */ - struct workqueue_struct *work_queues; struct delayed_work delayed_work_enable_hotplug; /* edid */ @@ -425,8 +423,7 @@ static void tc358743_enable_edid(struct v4l2_subdev *sd) /* Enable hotplug after 100 ms. DDC access to EDID is also enabled when * hotplug is enabled. See register DDC_CTL */ - queue_delayed_work(state->work_queues, - &state->delayed_work_enable_hotplug, HZ / 10); + schedule_delayed_work(&state->delayed_work_enable_hotplug, HZ / 10); tc358743_enable_interrupts(sd, true); tc358743_s_ctrl_detect_tx_5v(sd); @@ -1884,14 +1881,6 @@ static int tc358743_probe(struct i2c_client *client, goto err_hdl; } - /* work queues */ - state->work_queues = create_singlethread_workqueue(client->name); - if (!state->work_queues) { - v4l2_err(sd, "Could not create work queue\n"); - err = -ENOMEM; - goto err_hdl; - } - state->pad.flags = MEDIA_PAD_FL_SOURCE; err = media_entity_pads_init(&sd->entity, 1, &state->pad); if (err < 0) @@ -1940,7 +1929,6 @@ static int tc358743_probe(struct i2c_client *client, err_work_queues: cancel_delayed_work(&state->delayed_work_enable_hotplug); - destroy_workqueue(state->work_queues); mutex_destroy(&state->confctl_mutex); err_hdl: media_entity_cleanup(&sd->entity); @@ -1954,7 +1942,6 @@ static int tc358743_remove(struct i2c_client *client) struct tc358743_state *state = to_state(sd); cancel_delayed_work(&state->delayed_work_enable_hotplug); - destroy_workqueue(state->work_queues); v4l2_async_unregister_subdev(sd); v4l2_device_unregister_subdev(sd); mutex_destroy(&state->confctl_mutex); diff --git a/drivers/media/i2c/tvaudio.c b/drivers/media/i2c/tvaudio.c index fece2a4339a1..42d1e26e581c 100644 --- a/drivers/media/i2c/tvaudio.c +++ b/drivers/media/i2c/tvaudio.c @@ -1855,13 +1855,6 @@ static const struct v4l2_ctrl_ops tvaudio_ctrl_ops = { static const struct v4l2_subdev_core_ops tvaudio_core_ops = { .log_status = tvaudio_log_status, - .g_ext_ctrls = v4l2_subdev_g_ext_ctrls, - .try_ext_ctrls = v4l2_subdev_try_ext_ctrls, - .s_ext_ctrls = v4l2_subdev_s_ext_ctrls, - .g_ctrl = v4l2_subdev_g_ctrl, - .s_ctrl = v4l2_subdev_s_ctrl, - .queryctrl = v4l2_subdev_queryctrl, - .querymenu = v4l2_subdev_querymenu, }; static const struct v4l2_subdev_tuner_ops tvaudio_tuner_ops = { diff --git a/drivers/media/i2c/wm8775.c b/drivers/media/i2c/wm8775.c index 6e00f145b948..5581f4db02af 100644 --- a/drivers/media/i2c/wm8775.c +++ b/drivers/media/i2c/wm8775.c @@ -178,13 +178,6 @@ static const struct v4l2_ctrl_ops wm8775_ctrl_ops = { static const struct v4l2_subdev_core_ops wm8775_core_ops = { .log_status = wm8775_log_status, - .g_ext_ctrls = v4l2_subdev_g_ext_ctrls, - .try_ext_ctrls = v4l2_subdev_try_ext_ctrls, - .s_ext_ctrls = v4l2_subdev_s_ext_ctrls, - .g_ctrl = v4l2_subdev_g_ctrl, - .s_ctrl = v4l2_subdev_s_ctrl, - .queryctrl = v4l2_subdev_queryctrl, - .querymenu = v4l2_subdev_querymenu, }; static const struct v4l2_subdev_tuner_ops wm8775_tuner_ops = { diff --git a/drivers/media/media-device.c b/drivers/media/media-device.c index a1cd50f331f1..1795abeda658 100644 --- a/drivers/media/media-device.c +++ b/drivers/media/media-device.c @@ -423,7 +423,7 @@ static long media_device_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { struct media_devnode *devnode = media_devnode_data(filp); - struct media_device *dev = to_media_device(devnode); + struct media_device *dev = devnode->media_dev; long ret; mutex_lock(&dev->graph_mutex); @@ -495,7 +495,7 @@ static long media_device_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { struct media_devnode *devnode = media_devnode_data(filp); - struct media_device *dev = to_media_device(devnode); + struct media_device *dev = devnode->media_dev; long ret; switch (cmd) { @@ -531,7 +531,8 @@ static const struct media_file_operations media_device_fops = { static ssize_t show_model(struct device *cd, struct device_attribute *attr, char *buf) { - struct media_device *mdev = to_media_device(to_media_devnode(cd)); + struct media_devnode *devnode = to_media_devnode(cd); + struct media_device *mdev = devnode->media_dev; return sprintf(buf, "%.*s\n", (int)sizeof(mdev->model), mdev->model); } @@ -704,23 +705,35 @@ EXPORT_SYMBOL_GPL(media_device_cleanup); int __must_check __media_device_register(struct media_device *mdev, struct module *owner) { + struct media_devnode *devnode; int ret; + devnode = kzalloc(sizeof(*devnode), GFP_KERNEL); + if (!devnode) + return -ENOMEM; + /* Register the device node. */ - mdev->devnode.fops = &media_device_fops; - mdev->devnode.parent = mdev->dev; - mdev->devnode.release = media_device_release; + mdev->devnode = devnode; + devnode->fops = &media_device_fops; + devnode->parent = mdev->dev; + devnode->release = media_device_release; /* Set version 0 to indicate user-space that the graph is static */ mdev->topology_version = 0; - ret = media_devnode_register(&mdev->devnode, owner); - if (ret < 0) + ret = media_devnode_register(mdev, devnode, owner); + if (ret < 0) { + /* devnode free is handled in media_devnode_*() */ + mdev->devnode = NULL; return ret; + } - ret = device_create_file(&mdev->devnode.dev, &dev_attr_model); + ret = device_create_file(&devnode->dev, &dev_attr_model); if (ret < 0) { - media_devnode_unregister(&mdev->devnode); + /* devnode free is handled in media_devnode_*() */ + mdev->devnode = NULL; + media_devnode_unregister_prepare(devnode); + media_devnode_unregister(devnode); return ret; } @@ -771,11 +784,14 @@ void media_device_unregister(struct media_device *mdev) mutex_lock(&mdev->graph_mutex); /* Check if mdev was ever registered at all */ - if (!media_devnode_is_registered(&mdev->devnode)) { + if (!media_devnode_is_registered(mdev->devnode)) { mutex_unlock(&mdev->graph_mutex); return; } + /* Clear the devnode register bit to avoid races with media dev open */ + media_devnode_unregister_prepare(mdev->devnode); + /* Remove all entities from the media device */ list_for_each_entry_safe(entity, next, &mdev->entities, graph_obj.list) __media_device_unregister_entity(entity); @@ -794,9 +810,12 @@ void media_device_unregister(struct media_device *mdev) mutex_unlock(&mdev->graph_mutex); - device_remove_file(&mdev->devnode.dev, &dev_attr_model); - dev_dbg(mdev->dev, "Media device unregistering\n"); - media_devnode_unregister(&mdev->devnode); + dev_dbg(mdev->dev, "Media device unregistered\n"); + + device_remove_file(&mdev->devnode->dev, &dev_attr_model); + media_devnode_unregister(mdev->devnode); + /* devnode free is handled in media_devnode_*() */ + mdev->devnode = NULL; } EXPORT_SYMBOL_GPL(media_device_unregister); diff --git a/drivers/media/media-devnode.c b/drivers/media/media-devnode.c index b66dc9d0766b..f2772ba6f611 100644 --- a/drivers/media/media-devnode.c +++ b/drivers/media/media-devnode.c @@ -44,6 +44,7 @@ #include #include +#include #define MEDIA_NUM_DEVICES 256 #define MEDIA_NAME "media" @@ -59,21 +60,19 @@ static DECLARE_BITMAP(media_devnode_nums, MEDIA_NUM_DEVICES); /* Called when the last user of the media device exits. */ static void media_devnode_release(struct device *cd) { - struct media_devnode *mdev = to_media_devnode(cd); + struct media_devnode *devnode = to_media_devnode(cd); mutex_lock(&media_devnode_lock); - - /* Delete the cdev on this minor as well */ - cdev_del(&mdev->cdev); - /* Mark device node number as free */ - clear_bit(mdev->minor, media_devnode_nums); - + clear_bit(devnode->minor, media_devnode_nums); mutex_unlock(&media_devnode_lock); /* Release media_devnode and perform other cleanups as needed. */ - if (mdev->release) - mdev->release(mdev); + if (devnode->release) + devnode->release(devnode); + + kfree(devnode); + pr_debug("%s: Media Devnode Deallocated\n", __func__); } static struct bus_type media_bus_type = { @@ -83,37 +82,37 @@ static struct bus_type media_bus_type = { static ssize_t media_read(struct file *filp, char __user *buf, size_t sz, loff_t *off) { - struct media_devnode *mdev = media_devnode_data(filp); + struct media_devnode *devnode = media_devnode_data(filp); - if (!mdev->fops->read) + if (!devnode->fops->read) return -EINVAL; - if (!media_devnode_is_registered(mdev)) + if (!media_devnode_is_registered(devnode)) return -EIO; - return mdev->fops->read(filp, buf, sz, off); + return devnode->fops->read(filp, buf, sz, off); } static ssize_t media_write(struct file *filp, const char __user *buf, size_t sz, loff_t *off) { - struct media_devnode *mdev = media_devnode_data(filp); + struct media_devnode *devnode = media_devnode_data(filp); - if (!mdev->fops->write) + if (!devnode->fops->write) return -EINVAL; - if (!media_devnode_is_registered(mdev)) + if (!media_devnode_is_registered(devnode)) return -EIO; - return mdev->fops->write(filp, buf, sz, off); + return devnode->fops->write(filp, buf, sz, off); } static unsigned int media_poll(struct file *filp, struct poll_table_struct *poll) { - struct media_devnode *mdev = media_devnode_data(filp); + struct media_devnode *devnode = media_devnode_data(filp); - if (!media_devnode_is_registered(mdev)) + if (!media_devnode_is_registered(devnode)) return POLLERR | POLLHUP; - if (!mdev->fops->poll) + if (!devnode->fops->poll) return DEFAULT_POLLMASK; - return mdev->fops->poll(filp, poll); + return devnode->fops->poll(filp, poll); } static long @@ -121,12 +120,12 @@ __media_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, long (*ioctl_func)(struct file *filp, unsigned int cmd, unsigned long arg)) { - struct media_devnode *mdev = media_devnode_data(filp); + struct media_devnode *devnode = media_devnode_data(filp); if (!ioctl_func) return -ENOTTY; - if (!media_devnode_is_registered(mdev)) + if (!media_devnode_is_registered(devnode)) return -EIO; return ioctl_func(filp, cmd, arg); @@ -134,9 +133,9 @@ __media_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, static long media_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { - struct media_devnode *mdev = media_devnode_data(filp); + struct media_devnode *devnode = media_devnode_data(filp); - return __media_ioctl(filp, cmd, arg, mdev->fops->ioctl); + return __media_ioctl(filp, cmd, arg, devnode->fops->ioctl); } #ifdef CONFIG_COMPAT @@ -144,9 +143,9 @@ static long media_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) static long media_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { - struct media_devnode *mdev = media_devnode_data(filp); + struct media_devnode *devnode = media_devnode_data(filp); - return __media_ioctl(filp, cmd, arg, mdev->fops->compat_ioctl); + return __media_ioctl(filp, cmd, arg, devnode->fops->compat_ioctl); } #endif /* CONFIG_COMPAT */ @@ -154,7 +153,7 @@ static long media_compat_ioctl(struct file *filp, unsigned int cmd, /* Override for the open function */ static int media_open(struct inode *inode, struct file *filp) { - struct media_devnode *mdev; + struct media_devnode *devnode; int ret; /* Check if the media device is available. This needs to be done with @@ -164,23 +163,23 @@ static int media_open(struct inode *inode, struct file *filp) * a crash. */ mutex_lock(&media_devnode_lock); - mdev = container_of(inode->i_cdev, struct media_devnode, cdev); + devnode = container_of(inode->i_cdev, struct media_devnode, cdev); /* return ENXIO if the media device has been removed already or if it is not registered anymore. */ - if (!media_devnode_is_registered(mdev)) { + if (!media_devnode_is_registered(devnode)) { mutex_unlock(&media_devnode_lock); return -ENXIO; } /* and increase the device refcount */ - get_device(&mdev->dev); + get_device(&devnode->dev); mutex_unlock(&media_devnode_lock); - filp->private_data = mdev; + filp->private_data = devnode; - if (mdev->fops->open) { - ret = mdev->fops->open(filp); + if (devnode->fops->open) { + ret = devnode->fops->open(filp); if (ret) { - put_device(&mdev->dev); + put_device(&devnode->dev); filp->private_data = NULL; return ret; } @@ -192,16 +191,18 @@ static int media_open(struct inode *inode, struct file *filp) /* Override for the release function */ static int media_release(struct inode *inode, struct file *filp) { - struct media_devnode *mdev = media_devnode_data(filp); + struct media_devnode *devnode = media_devnode_data(filp); - if (mdev->fops->release) - mdev->fops->release(filp); + if (devnode->fops->release) + devnode->fops->release(filp); filp->private_data = NULL; /* decrease the refcount unconditionally since the release() return value is ignored. */ - put_device(&mdev->dev); + put_device(&devnode->dev); + + pr_debug("%s: Media Release\n", __func__); return 0; } @@ -219,7 +220,8 @@ static const struct file_operations media_devnode_fops = { .llseek = no_llseek, }; -int __must_check media_devnode_register(struct media_devnode *mdev, +int __must_check media_devnode_register(struct media_device *mdev, + struct media_devnode *devnode, struct module *owner) { int minor; @@ -231,61 +233,80 @@ int __must_check media_devnode_register(struct media_devnode *mdev, if (minor == MEDIA_NUM_DEVICES) { mutex_unlock(&media_devnode_lock); pr_err("could not get a free minor\n"); + kfree(devnode); return -ENFILE; } set_bit(minor, media_devnode_nums); mutex_unlock(&media_devnode_lock); - mdev->minor = minor; + devnode->minor = minor; + devnode->media_dev = mdev; + + /* Part 1: Initialize dev now to use dev.kobj for cdev.kobj.parent */ + devnode->dev.bus = &media_bus_type; + devnode->dev.devt = MKDEV(MAJOR(media_dev_t), devnode->minor); + devnode->dev.release = media_devnode_release; + if (devnode->parent) + devnode->dev.parent = devnode->parent; + dev_set_name(&devnode->dev, "media%d", devnode->minor); + device_initialize(&devnode->dev); /* Part 2: Initialize and register the character device */ - cdev_init(&mdev->cdev, &media_devnode_fops); - mdev->cdev.owner = owner; + cdev_init(&devnode->cdev, &media_devnode_fops); + devnode->cdev.owner = owner; + devnode->cdev.kobj.parent = &devnode->dev.kobj; - ret = cdev_add(&mdev->cdev, MKDEV(MAJOR(media_dev_t), mdev->minor), 1); + ret = cdev_add(&devnode->cdev, MKDEV(MAJOR(media_dev_t), devnode->minor), 1); if (ret < 0) { pr_err("%s: cdev_add failed\n", __func__); - goto error; + goto cdev_add_error; } - /* Part 3: Register the media device */ - mdev->dev.bus = &media_bus_type; - mdev->dev.devt = MKDEV(MAJOR(media_dev_t), mdev->minor); - mdev->dev.release = media_devnode_release; - if (mdev->parent) - mdev->dev.parent = mdev->parent; - dev_set_name(&mdev->dev, "media%d", mdev->minor); - ret = device_register(&mdev->dev); + /* Part 3: Add the media device */ + ret = device_add(&devnode->dev); if (ret < 0) { - pr_err("%s: device_register failed\n", __func__); - goto error; + pr_err("%s: device_add failed\n", __func__); + goto device_add_error; } /* Part 4: Activate this minor. The char device can now be used. */ - set_bit(MEDIA_FLAG_REGISTERED, &mdev->flags); + set_bit(MEDIA_FLAG_REGISTERED, &devnode->flags); return 0; -error: +device_add_error: + cdev_del(&devnode->cdev); +cdev_add_error: mutex_lock(&media_devnode_lock); - cdev_del(&mdev->cdev); - clear_bit(mdev->minor, media_devnode_nums); + clear_bit(devnode->minor, media_devnode_nums); + devnode->media_dev = NULL; mutex_unlock(&media_devnode_lock); + put_device(&devnode->dev); return ret; } -void media_devnode_unregister(struct media_devnode *mdev) +void media_devnode_unregister_prepare(struct media_devnode *devnode) { - /* Check if mdev was ever registered at all */ - if (!media_devnode_is_registered(mdev)) + /* Check if devnode was ever registered at all */ + if (!media_devnode_is_registered(devnode)) return; mutex_lock(&media_devnode_lock); - clear_bit(MEDIA_FLAG_REGISTERED, &mdev->flags); + clear_bit(MEDIA_FLAG_REGISTERED, &devnode->flags); + mutex_unlock(&media_devnode_lock); +} + +void media_devnode_unregister(struct media_devnode *devnode) +{ + mutex_lock(&media_devnode_lock); + /* Delete the cdev on this minor as well */ + cdev_del(&devnode->cdev); mutex_unlock(&media_devnode_lock); - device_unregister(&mdev->dev); + device_del(&devnode->dev); + devnode->media_dev = NULL; + put_device(&devnode->dev); } /* diff --git a/drivers/media/pci/bt8xx/dst_ca.c b/drivers/media/pci/bt8xx/dst_ca.c index da8b414fd824..8681b9143a35 100644 --- a/drivers/media/pci/bt8xx/dst_ca.c +++ b/drivers/media/pci/bt8xx/dst_ca.c @@ -655,7 +655,6 @@ static long dst_ca_ioctl(struct file *file, unsigned int cmd, unsigned long ioct static int dst_ca_open(struct inode *inode, struct file *file) { dprintk(verbose, DST_CA_DEBUG, 1, " Device opened [%p] ", file); - try_module_get(THIS_MODULE); return 0; } @@ -663,7 +662,6 @@ static int dst_ca_open(struct inode *inode, struct file *file) static int dst_ca_release(struct inode *inode, struct file *file) { dprintk(verbose, DST_CA_DEBUG, 1, " Device closed."); - module_put(THIS_MODULE); return 0; } diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c index 8d6f04fc8013..476f7f0dcf81 100644 --- a/drivers/media/pci/cobalt/cobalt-driver.c +++ b/drivers/media/pci/cobalt/cobalt-driver.c @@ -492,7 +492,6 @@ static int cobalt_subdevs_init(struct cobalt *cobalt) .ain_sel = ADV7604_AIN7_8_9_NC_SYNC_3_1, .bus_order = ADV7604_BUS_ORDER_BRG, .blank_data = 1, - .op_656_range = 1, .op_format_mode_sel = ADV7604_OP_FORMAT_MODE0, .int1_config = ADV76XX_INT1_CONFIG_ACTIVE_HIGH, .dr_str_data = ADV76XX_DR_STR_HIGH, @@ -571,7 +570,6 @@ static int cobalt_subdevs_hsma_init(struct cobalt *cobalt) .bus_order = ADV7842_BUS_ORDER_RBG, .op_format_mode_sel = ADV7842_OP_FORMAT_MODE0, .blank_data = 1, - .op_656_range = 1, .dr_str_data = 3, .dr_str_clk = 3, .dr_str_sync = 3, @@ -691,17 +689,10 @@ static int cobalt_probe(struct pci_dev *pci_dev, cobalt->pci_dev = pci_dev; cobalt->instance = i; - cobalt->alloc_ctx = vb2_dma_sg_init_ctx(&pci_dev->dev); - if (IS_ERR(cobalt->alloc_ctx)) { - kfree(cobalt); - return -ENOMEM; - } - retval = v4l2_device_register(&pci_dev->dev, &cobalt->v4l2_dev); if (retval) { pr_err("cobalt: v4l2_device_register of card %d failed\n", cobalt->instance); - vb2_dma_sg_cleanup_ctx(cobalt->alloc_ctx); kfree(cobalt); return retval; } @@ -782,7 +773,6 @@ err: cobalt_err("error %d on initialization\n", retval); v4l2_device_unregister(&cobalt->v4l2_dev); - vb2_dma_sg_cleanup_ctx(cobalt->alloc_ctx); kfree(cobalt); return retval; } @@ -818,7 +808,6 @@ static void cobalt_remove(struct pci_dev *pci_dev) cobalt_info("removed cobalt card\n"); v4l2_device_unregister(v4l2_dev); - vb2_dma_sg_cleanup_ctx(cobalt->alloc_ctx); kfree(cobalt); } diff --git a/drivers/media/pci/cobalt/cobalt-driver.h b/drivers/media/pci/cobalt/cobalt-driver.h index b2f08e4a68bf..ed00dc9d9399 100644 --- a/drivers/media/pci/cobalt/cobalt-driver.h +++ b/drivers/media/pci/cobalt/cobalt-driver.h @@ -262,7 +262,6 @@ struct cobalt { int instance; struct pci_dev *pci_dev; struct v4l2_device v4l2_dev; - void *alloc_ctx; void __iomem *bar0, *bar1; diff --git a/drivers/media/pci/cobalt/cobalt-v4l2.c b/drivers/media/pci/cobalt/cobalt-v4l2.c index c0ba458f6cf3..d05672fe9ff9 100644 --- a/drivers/media/pci/cobalt/cobalt-v4l2.c +++ b/drivers/media/pci/cobalt/cobalt-v4l2.c @@ -45,7 +45,7 @@ static const struct v4l2_dv_timings cea1080p60 = V4L2_DV_BT_CEA_1920X1080P60; static int cobalt_queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cobalt_stream *s = q->drv_priv; unsigned size = s->stride * s->height; @@ -54,7 +54,6 @@ static int cobalt_queue_setup(struct vb2_queue *q, *num_buffers = 3; if (*num_buffers > NR_BUFS) *num_buffers = NR_BUFS; - alloc_ctxs[0] = s->cobalt->alloc_ctx; if (*num_planes) return sizes[0] < size ? -EINVAL : 0; *num_planes = 1; @@ -1224,6 +1223,7 @@ static int cobalt_node_register(struct cobalt *cobalt, int node) q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->min_buffers_needed = 2; q->lock = &s->lock; + q->dev = &cobalt->pci_dev->dev; vdev->queue = q; video_set_drvdata(vdev, s); diff --git a/drivers/media/pci/cx18/cx18-alsa-mixer.c b/drivers/media/pci/cx18/cx18-alsa-mixer.c index 341bddc00b77..284275270f1b 100644 --- a/drivers/media/pci/cx18/cx18-alsa-mixer.c +++ b/drivers/media/pci/cx18/cx18-alsa-mixer.c @@ -93,7 +93,7 @@ static int snd_cx18_mixer_tv_vol_get(struct snd_kcontrol *kctl, vctrl.value = dB_to_cx18_av_vol(uctl->value.integer.value[0]); snd_cx18_lock(cxsc); - ret = v4l2_subdev_call(cx->sd_av, core, g_ctrl, &vctrl); + ret = v4l2_g_ctrl(cx->sd_av->ctrl_handler, &vctrl); snd_cx18_unlock(cxsc); if (!ret) @@ -115,14 +115,14 @@ static int snd_cx18_mixer_tv_vol_put(struct snd_kcontrol *kctl, snd_cx18_lock(cxsc); /* Fetch current state */ - ret = v4l2_subdev_call(cx->sd_av, core, g_ctrl, &vctrl); + ret = v4l2_g_ctrl(cx->sd_av->ctrl_handler, &vctrl); if (ret || (cx18_av_vol_to_dB(vctrl.value) != uctl->value.integer.value[0])) { /* Set, if needed */ vctrl.value = dB_to_cx18_av_vol(uctl->value.integer.value[0]); - ret = v4l2_subdev_call(cx->sd_av, core, s_ctrl, &vctrl); + ret = v4l2_s_ctrl(cx->sd_av->ctrl_handler, &vctrl); if (!ret) ret = 1; /* Indicate control was changed w/o error */ } diff --git a/drivers/media/pci/cx18/cx18-driver.c b/drivers/media/pci/cx18/cx18-driver.c index 260e462d91b4..2f23b26b16c0 100644 --- a/drivers/media/pci/cx18/cx18-driver.c +++ b/drivers/media/pci/cx18/cx18-driver.c @@ -560,7 +560,7 @@ static void cx18_process_options(struct cx18 *cx) cx->stream_buf_size[CX18_ENC_STREAM_TYPE_MPG] = enc_mpg_bufsize; cx->stream_buf_size[CX18_ENC_STREAM_TYPE_IDX] = enc_idx_bufsize; cx->stream_buf_size[CX18_ENC_STREAM_TYPE_YUV] = enc_yuv_bufsize; - cx->stream_buf_size[CX18_ENC_STREAM_TYPE_VBI] = vbi_active_samples * 36; + cx->stream_buf_size[CX18_ENC_STREAM_TYPE_VBI] = VBI_ACTIVE_SAMPLES * 36; cx->stream_buf_size[CX18_ENC_STREAM_TYPE_PCM] = enc_pcm_bufsize; cx->stream_buf_size[CX18_ENC_STREAM_TYPE_RAD] = 0; /* control no data */ diff --git a/drivers/media/pci/cx18/cx18-driver.h b/drivers/media/pci/cx18/cx18-driver.h index 47ce80fa73b9..ef308a10e870 100644 --- a/drivers/media/pci/cx18/cx18-driver.h +++ b/drivers/media/pci/cx18/cx18-driver.h @@ -492,9 +492,9 @@ struct cx18_card; * (1/15.625 kHz) * 2 * 13.5 MHz = 1728 samples/line = * 4 bytes SAV + 280 bytes anc data + 4 bytes SAV + 1440 active samples */ -static const u32 vbi_active_samples = 1444; /* 4 byte SAV + 720 Y + 720 U/V */ -static const u32 vbi_hblank_samples_60Hz = 272; /* 4 byte EAV + 268 anc/fill */ -static const u32 vbi_hblank_samples_50Hz = 284; /* 4 byte EAV + 280 anc/fill */ +#define VBI_ACTIVE_SAMPLES 1444 /* 4 byte SAV + 720 Y + 720 U/V */ +#define VBI_HBLANK_SAMPLES_60HZ 272 /* 4 byte EAV + 268 anc/fill */ +#define VBI_HBLANK_SAMPLES_50HZ 284 /* 4 byte EAV + 280 anc/fill */ #define CX18_VBI_FRAMES 32 diff --git a/drivers/media/pci/cx18/cx18-ioctl.c b/drivers/media/pci/cx18/cx18-ioctl.c index eeb741c7db1b..fecca2a63891 100644 --- a/drivers/media/pci/cx18/cx18-ioctl.c +++ b/drivers/media/pci/cx18/cx18-ioctl.c @@ -177,7 +177,7 @@ static int cx18_g_fmt_vbi_cap(struct file *file, void *fh, vbifmt->sampling_rate = 27000000; vbifmt->offset = 248; /* FIXME - slightly wrong for both 50 & 60 Hz */ - vbifmt->samples_per_line = vbi_active_samples - 4; + vbifmt->samples_per_line = VBI_ACTIVE_SAMPLES - 4; vbifmt->sample_format = V4L2_PIX_FMT_GREY; vbifmt->start[0] = cx->vbi.start[0]; vbifmt->start[1] = cx->vbi.start[1]; diff --git a/drivers/media/pci/cx18/cx18-streams.c b/drivers/media/pci/cx18/cx18-streams.c index c9860845264f..f3802ec1b383 100644 --- a/drivers/media/pci/cx18/cx18-streams.c +++ b/drivers/media/pci/cx18/cx18-streams.c @@ -605,9 +605,9 @@ static void cx18_vbi_setup(struct cx18_stream *s) /* Lines per field */ data[1] = (lines / 2) | ((lines / 2) << 16); /* bytes per line */ - data[2] = (raw ? vbi_active_samples - : (cx->is_60hz ? vbi_hblank_samples_60Hz - : vbi_hblank_samples_50Hz)); + data[2] = (raw ? VBI_ACTIVE_SAMPLES + : (cx->is_60hz ? VBI_HBLANK_SAMPLES_60HZ + : VBI_HBLANK_SAMPLES_50HZ)); /* Every X number of frames a VBI interrupt arrives (frames as in 25 or 30 fps) */ data[3] = 1; @@ -761,7 +761,7 @@ static void cx18_stream_configure_mdls(struct cx18_stream *s) s->bufs_per_mdl = 1; if (cx18_raw_vbi(s->cx)) { s->mdl_size = (s->cx->is_60hz ? 12 : 18) - * 2 * vbi_active_samples; + * 2 * VBI_ACTIVE_SAMPLES; } else { /* * See comment in cx18_vbi_setup() below about the @@ -769,8 +769,8 @@ static void cx18_stream_configure_mdls(struct cx18_stream *s) * the lines on which EAV RP codes toggle. */ s->mdl_size = s->cx->is_60hz - ? (21 - 4 + 1) * 2 * vbi_hblank_samples_60Hz - : (23 - 2 + 1) * 2 * vbi_hblank_samples_50Hz; + ? (21 - 4 + 1) * 2 * VBI_HBLANK_SAMPLES_60HZ + : (23 - 2 + 1) * 2 * VBI_HBLANK_SAMPLES_50HZ; } break; default: diff --git a/drivers/media/pci/cx18/cx18-vbi.c b/drivers/media/pci/cx18/cx18-vbi.c index add99642f1e2..43360cbcf24b 100644 --- a/drivers/media/pci/cx18/cx18-vbi.c +++ b/drivers/media/pci/cx18/cx18-vbi.c @@ -108,7 +108,7 @@ static void copy_vbi_data(struct cx18 *cx, int lines, u32 pts_stamp) /* FIXME - this function ignores the input size. */ static u32 compress_raw_buf(struct cx18 *cx, u8 *buf, u32 size, u32 hdr_size) { - u32 line_size = vbi_active_samples; + u32 line_size = VBI_ACTIVE_SAMPLES; u32 lines = cx->vbi.count * 2; u8 *q = buf; u8 *p; @@ -145,8 +145,8 @@ static u32 compress_sliced_buf(struct cx18 *cx, u8 *buf, u32 size, struct v4l2_decode_vbi_line vbi; int i; u32 line = 0; - u32 line_size = cx->is_60hz ? vbi_hblank_samples_60Hz - : vbi_hblank_samples_50Hz; + u32 line_size = cx->is_60hz ? VBI_HBLANK_SAMPLES_60HZ + : VBI_HBLANK_SAMPLES_50HZ; /* find the first valid line */ for (i = hdr_size, buf += hdr_size; i < size; i++, buf++) { diff --git a/drivers/media/pci/cx23885/cx23885-417.c b/drivers/media/pci/cx23885/cx23885-417.c index bd333875a1f7..efec2d1a7afd 100644 --- a/drivers/media/pci/cx23885/cx23885-417.c +++ b/drivers/media/pci/cx23885/cx23885-417.c @@ -1140,7 +1140,7 @@ static int cx23885_initialize_codec(struct cx23885_dev *dev, int startencoder) static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx23885_dev *dev = q->drv_priv; @@ -1148,7 +1148,6 @@ static int queue_setup(struct vb2_queue *q, dev->ts1.ts_packet_count = mpeglines; *num_planes = 1; sizes[0] = mpeglinesize * mpeglines; - alloc_ctxs[0] = dev->alloc_ctx; *num_buffers = mpegbufs; return 0; } diff --git a/drivers/media/pci/cx23885/cx23885-cards.c b/drivers/media/pci/cx23885/cx23885-cards.c index 310ee769aed4..4abf50f2694f 100644 --- a/drivers/media/pci/cx23885/cx23885-cards.c +++ b/drivers/media/pci/cx23885/cx23885-cards.c @@ -765,6 +765,11 @@ struct cx23885_board cx23885_boards[] = { .amux = CX25840_AUDIO7, } }, }, + [CX23885_BOARD_HAUPPAUGE_QUADHD_DVB] = { + .name = "Hauppauge WinTV-QuadHD-DVB", + .portb = CX23885_MPEG_DVB, + .portc = CX23885_MPEG_DVB, + }, }; const unsigned int cx23885_bcount = ARRAY_SIZE(cx23885_boards); @@ -1060,6 +1065,14 @@ struct cx23885_subid cx23885_subids[] = { .subvendor = 0x1576, .subdevice = 0x0460, .card = CX23885_BOARD_VIEWCAST_460E, + }, { + .subvendor = 0x0070, + .subdevice = 0x6a28, + .card = CX23885_BOARD_HAUPPAUGE_QUADHD_DVB, /* Tuner Pair 1 */ + }, { + .subvendor = 0x0070, + .subdevice = 0x6b28, + .card = CX23885_BOARD_HAUPPAUGE_QUADHD_DVB, /* Tuner Pair 2 */ }, }; const unsigned int cx23885_idcount = ARRAY_SIZE(cx23885_subids); @@ -1257,6 +1270,14 @@ static void hauppauge_eeprom(struct cx23885_dev *dev, u8 *eeprom_data) case 150329: /* WinTV-HVR5525 (PCIe, DVB-S/S2, DVB-T/T2/C) */ break; + case 166100: + /* WinTV-QuadHD (DVB) Tuner Pair 1 (PCIe, IR, half height, + DVB-T/T2/C, DVB-T/T2/C */ + break; + case 166101: + /* WinTV-QuadHD (DVB) Tuner Pair 2 (PCIe, IR, half height, + DVB-T/T2/C, DVB-T/T2/C */ + break; default: printk(KERN_WARNING "%s: warning: " "unknown hauppauge model #%d\n", @@ -1729,20 +1750,22 @@ void cx23885_gpio_setup(struct cx23885_dev *dev) cx23885_gpio_set(dev, GPIO_2); break; case CX23885_BOARD_HAUPPAUGE_HVR5525: + case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB: /* - * GPIO-00 IR_WIDE - * GPIO-02 wake# - * GPIO-03 VAUX Pres. - * GPIO-07 PROG# - * GPIO-08 SAT_RESN - * GPIO-09 TER_RESN - * GPIO-10 B2_SENSE - * GPIO-11 B1_SENSE - * GPIO-15 IR_LED_STATUS - * GPIO-19 IR_NARROW - * GPIO-20 Blauster1 - * ALTGPIO VAUX_SWITCH - * AUX_PLL_CLK : Blaster2 + * HVR5525 GPIO Details: + * GPIO-00 IR_WIDE + * GPIO-02 wake# + * GPIO-03 VAUX Pres. + * GPIO-07 PROG# + * GPIO-08 SAT_RESN + * GPIO-09 TER_RESN + * GPIO-10 B2_SENSE + * GPIO-11 B1_SENSE + * GPIO-15 IR_LED_STATUS + * GPIO-19 IR_NARROW + * GPIO-20 Blauster1 + * ALTGPIO VAUX_SWITCH + * AUX_PLL_CLK : Blaster2 */ /* Put the parts into reset and back */ cx23885_gpio_enable(dev, GPIO_8 | GPIO_9, 1); @@ -1802,6 +1825,7 @@ int cx23885_ir_init(struct cx23885_dev *dev) case CX23885_BOARD_HAUPPAUGE_HVR1255: case CX23885_BOARD_HAUPPAUGE_HVR1255_22111: case CX23885_BOARD_HAUPPAUGE_HVR1210: + case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB: /* FIXME: Implement me */ break; case CX23885_BOARD_HAUPPAUGE_HVR1270: @@ -2000,6 +2024,7 @@ void cx23885_card_setup(struct cx23885_dev *dev) case CX23885_BOARD_HAUPPAUGE_STARBURST: case CX23885_BOARD_HAUPPAUGE_IMPACTVCBE: case CX23885_BOARD_HAUPPAUGE_HVR5525: + case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB: if (dev->i2c_bus[0].i2c_rc == 0) hauppauge_eeprom(dev, eeprom+0xc0); break; @@ -2145,6 +2170,14 @@ void cx23885_card_setup(struct cx23885_dev *dev) ts2->ts_clk_en_val = 0x1; /* Enable TS_CLK */ ts2->src_sel_val = CX23885_SRC_SEL_PARALLEL_MPEG_VIDEO; break; + case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB: + ts1->gen_ctrl_val = 0xc; /* Serial bus + punctured clock */ + ts1->ts_clk_en_val = 0x1; /* Enable TS_CLK */ + ts1->src_sel_val = CX23885_SRC_SEL_PARALLEL_MPEG_VIDEO; + ts2->gen_ctrl_val = 0xc; /* Serial bus + punctured clock */ + ts2->ts_clk_en_val = 0x1; /* Enable TS_CLK */ + ts2->src_sel_val = CX23885_SRC_SEL_PARALLEL_MPEG_VIDEO; + break; case CX23885_BOARD_HAUPPAUGE_HVR1250: case CX23885_BOARD_HAUPPAUGE_HVR1500: case CX23885_BOARD_HAUPPAUGE_HVR1500Q: diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c index 813c217b5e1a..c86b1093ab99 100644 --- a/drivers/media/pci/cx23885/cx23885-core.c +++ b/drivers/media/pci/cx23885/cx23885-core.c @@ -2005,14 +2005,9 @@ static int cx23885_initdev(struct pci_dev *pci_dev, err = pci_set_dma_mask(pci_dev, 0xffffffff); if (err) { printk("%s/0: Oops: no 32bit PCI DMA ???\n", dev->name); - goto fail_context; + goto fail_ctrl; } - dev->alloc_ctx = vb2_dma_sg_init_ctx(&pci_dev->dev); - if (IS_ERR(dev->alloc_ctx)) { - err = PTR_ERR(dev->alloc_ctx); - goto fail_context; - } err = request_irq(pci_dev->irq, cx23885_irq, IRQF_SHARED, dev->name, dev); if (err < 0) { @@ -2041,8 +2036,6 @@ static int cx23885_initdev(struct pci_dev *pci_dev, return 0; fail_irq: - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); -fail_context: cx23885_dev_unregister(dev); fail_ctrl: v4l2_ctrl_handler_free(hdl); @@ -2068,7 +2061,6 @@ static void cx23885_finidev(struct pci_dev *pci_dev) pci_disable_device(pci_dev); cx23885_dev_unregister(dev); - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); v4l2_ctrl_handler_free(&dev->ctrl_handler); v4l2_device_unregister(v4l2_dev); kfree(dev); diff --git a/drivers/media/pci/cx23885/cx23885-dvb.c b/drivers/media/pci/cx23885/cx23885-dvb.c index f041b6931ba8..e5748a93c479 100644 --- a/drivers/media/pci/cx23885/cx23885-dvb.c +++ b/drivers/media/pci/cx23885/cx23885-dvb.c @@ -94,7 +94,7 @@ DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx23885_tsport *port = q->drv_priv; @@ -102,7 +102,6 @@ static int queue_setup(struct vb2_queue *q, port->ts_packet_count = 32; *num_planes = 1; sizes[0] = port->ts_packet_size * port->ts_packet_count; - alloc_ctxs[0] = port->dev->alloc_ctx; *num_buffers = 32; return 0; } @@ -2269,9 +2268,107 @@ static int dvb_register(struct cx23885_tsport *port) } break; } + case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB: + switch (port->nr) { + /* port b - Terrestrial/cable */ + case 1: + /* attach frontend */ + memset(&si2168_config, 0, sizeof(si2168_config)); + si2168_config.i2c_adapter = &adapter; + si2168_config.fe = &fe0->dvb.frontend; + si2168_config.ts_mode = SI2168_TS_SERIAL; + memset(&info, 0, sizeof(struct i2c_board_info)); + strlcpy(info.type, "si2168", I2C_NAME_SIZE); + info.addr = 0x64; + info.platform_data = &si2168_config; + request_module("%s", info.type); + client_demod = i2c_new_device(&dev->i2c_bus[0].i2c_adap, &info); + if (!client_demod || !client_demod->dev.driver) + goto frontend_detach; + if (!try_module_get(client_demod->dev.driver->owner)) { + i2c_unregister_device(client_demod); + goto frontend_detach; + } + port->i2c_client_demod = client_demod; + + /* attach tuner */ + memset(&si2157_config, 0, sizeof(si2157_config)); + si2157_config.fe = fe0->dvb.frontend; + si2157_config.if_port = 1; + memset(&info, 0, sizeof(struct i2c_board_info)); + strlcpy(info.type, "si2157", I2C_NAME_SIZE); + info.addr = 0x60; + info.platform_data = &si2157_config; + request_module("%s", info.type); + client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info); + if (!client_tuner || !client_tuner->dev.driver) { + module_put(client_demod->dev.driver->owner); + i2c_unregister_device(client_demod); + port->i2c_client_demod = NULL; + goto frontend_detach; + } + if (!try_module_get(client_tuner->dev.driver->owner)) { + i2c_unregister_device(client_tuner); + module_put(client_demod->dev.driver->owner); + i2c_unregister_device(client_demod); + port->i2c_client_demod = NULL; + goto frontend_detach; + } + port->i2c_client_tuner = client_tuner; + break; + + /* port c - terrestrial/cable */ + case 2: + /* attach frontend */ + memset(&si2168_config, 0, sizeof(si2168_config)); + si2168_config.i2c_adapter = &adapter; + si2168_config.fe = &fe0->dvb.frontend; + si2168_config.ts_mode = SI2168_TS_SERIAL; + memset(&info, 0, sizeof(struct i2c_board_info)); + strlcpy(info.type, "si2168", I2C_NAME_SIZE); + info.addr = 0x66; + info.platform_data = &si2168_config; + request_module("%s", info.type); + client_demod = i2c_new_device(&dev->i2c_bus[0].i2c_adap, &info); + if (!client_demod || !client_demod->dev.driver) + goto frontend_detach; + if (!try_module_get(client_demod->dev.driver->owner)) { + i2c_unregister_device(client_demod); + goto frontend_detach; + } + port->i2c_client_demod = client_demod; + + /* attach tuner */ + memset(&si2157_config, 0, sizeof(si2157_config)); + si2157_config.fe = fe0->dvb.frontend; + si2157_config.if_port = 1; + memset(&info, 0, sizeof(struct i2c_board_info)); + strlcpy(info.type, "si2157", I2C_NAME_SIZE); + info.addr = 0x62; + info.platform_data = &si2157_config; + request_module("%s", info.type); + client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info); + if (!client_tuner || !client_tuner->dev.driver) { + module_put(client_demod->dev.driver->owner); + i2c_unregister_device(client_demod); + port->i2c_client_demod = NULL; + goto frontend_detach; + } + if (!try_module_get(client_tuner->dev.driver->owner)) { + i2c_unregister_device(client_tuner); + module_put(client_demod->dev.driver->owner); + i2c_unregister_device(client_demod); + port->i2c_client_demod = NULL; + goto frontend_detach; + } + port->i2c_client_tuner = client_tuner; + break; + } + break; + default: printk(KERN_INFO "%s: The frontend of your DVB/ATSC card " - " isn't supported yet\n", + " isn't supported yet\n", dev->name); break; } @@ -2397,6 +2494,7 @@ int cx23885_dvb_register(struct cx23885_tsport *port) q->mem_ops = &vb2_dma_sg_memops; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &dev->lock; + q->dev = &dev->pci->dev; err = vb2_queue_init(q); if (err < 0) diff --git a/drivers/media/pci/cx23885/cx23885-vbi.c b/drivers/media/pci/cx23885/cx23885-vbi.c index 39750ebcc04c..75e7fa7b1121 100644 --- a/drivers/media/pci/cx23885/cx23885-vbi.c +++ b/drivers/media/pci/cx23885/cx23885-vbi.c @@ -122,7 +122,7 @@ static int cx23885_start_vbi_dma(struct cx23885_dev *dev, static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx23885_dev *dev = q->drv_priv; unsigned lines = VBI_PAL_LINE_COUNT; @@ -131,7 +131,6 @@ static int queue_setup(struct vb2_queue *q, lines = VBI_NTSC_LINE_COUNT; *num_planes = 1; sizes[0] = lines * VBI_LINE_LENGTH * 2; - alloc_ctxs[0] = dev->alloc_ctx; return 0; } diff --git a/drivers/media/pci/cx23885/cx23885-video.c b/drivers/media/pci/cx23885/cx23885-video.c index e1d7d0847167..6d735222a958 100644 --- a/drivers/media/pci/cx23885/cx23885-video.c +++ b/drivers/media/pci/cx23885/cx23885-video.c @@ -335,13 +335,12 @@ static int cx23885_start_video_dma(struct cx23885_dev *dev, static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx23885_dev *dev = q->drv_priv; *num_planes = 1; sizes[0] = (dev->fmt->depth * dev->width * dev->height) >> 3; - alloc_ctxs[0] = dev->alloc_ctx; return 0; } @@ -1268,6 +1267,7 @@ int cx23885_video_register(struct cx23885_dev *dev) q->mem_ops = &vb2_dma_sg_memops; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &dev->lock; + q->dev = &dev->pci->dev; err = vb2_queue_init(q); if (err < 0) @@ -1284,6 +1284,7 @@ int cx23885_video_register(struct cx23885_dev *dev) q->mem_ops = &vb2_dma_sg_memops; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &dev->lock; + q->dev = &dev->pci->dev; err = vb2_queue_init(q); if (err < 0) diff --git a/drivers/media/pci/cx23885/cx23885.h b/drivers/media/pci/cx23885/cx23885.h index b1a5409408c7..24a0a6c5b501 100644 --- a/drivers/media/pci/cx23885/cx23885.h +++ b/drivers/media/pci/cx23885/cx23885.h @@ -103,6 +103,7 @@ #define CX23885_BOARD_HAUPPAUGE_STARBURST 53 #define CX23885_BOARD_VIEWCAST_260E 54 #define CX23885_BOARD_VIEWCAST_460E 55 +#define CX23885_BOARD_HAUPPAUGE_QUADHD_DVB 56 #define GPIO_0 0x00000001 #define GPIO_1 0x00000002 @@ -430,7 +431,6 @@ struct cx23885_dev { struct vb2_queue vb2_vidq; struct cx23885_dmaqueue vbiq; struct vb2_queue vb2_vbiq; - void *alloc_ctx; spinlock_t slock; diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c index b602eba2b601..df189b16af12 100644 --- a/drivers/media/pci/cx25821/cx25821-alsa.c +++ b/drivers/media/pci/cx25821/cx25821-alsa.c @@ -693,7 +693,7 @@ static int snd_cx25821_pcm(struct cx25821_audio_dev *chip, int device, * Only boards with eeprom and byte 1 at eeprom=1 have it */ -static const struct pci_device_id cx25821_audio_pci_tbl[] = { +static const struct pci_device_id __maybe_unused cx25821_audio_pci_tbl[] = { {0x14f1, 0x0920, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, {0,} }; diff --git a/drivers/media/pci/cx25821/cx25821-core.c b/drivers/media/pci/cx25821/cx25821-core.c index 0042803a9de7..9a5f912ca859 100644 --- a/drivers/media/pci/cx25821/cx25821-core.c +++ b/drivers/media/pci/cx25821/cx25821-core.c @@ -1301,15 +1301,10 @@ static int cx25821_initdev(struct pci_dev *pci_dev, goto fail_unregister_device; } - dev->alloc_ctx = vb2_dma_sg_init_ctx(&pci_dev->dev); - if (IS_ERR(dev->alloc_ctx)) { - err = PTR_ERR(dev->alloc_ctx); - goto fail_unregister_pci; - } err = cx25821_dev_setup(dev); if (err) - goto fail_free_ctx; + goto fail_unregister_pci; /* print pci info */ pci_read_config_byte(pci_dev, PCI_CLASS_REVISION, &dev->pci_rev); @@ -1340,8 +1335,6 @@ fail_irq: pr_info("cx25821_initdev() can't get IRQ !\n"); cx25821_dev_unregister(dev); -fail_free_ctx: - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); fail_unregister_pci: pci_disable_device(pci_dev); fail_unregister_device: @@ -1365,7 +1358,6 @@ static void cx25821_finidev(struct pci_dev *pci_dev) free_irq(pci_dev->irq, dev); cx25821_dev_unregister(dev); - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); v4l2_device_unregister(v4l2_dev); kfree(dev); } diff --git a/drivers/media/pci/cx25821/cx25821-video.c b/drivers/media/pci/cx25821/cx25821-video.c index c48bba9daf1f..adcd09be347d 100644 --- a/drivers/media/pci/cx25821/cx25821-video.c +++ b/drivers/media/pci/cx25821/cx25821-video.c @@ -143,13 +143,11 @@ int cx25821_video_irq(struct cx25821_dev *dev, int chan_num, u32 status) static int cx25821_queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx25821_channel *chan = q->drv_priv; unsigned size = (chan->fmt->depth * chan->width * chan->height) >> 3; - alloc_ctxs[0] = chan->dev->alloc_ctx; - if (*num_planes) return sizes[0] < size ? -EINVAL : 0; @@ -759,6 +757,7 @@ int cx25821_video_register(struct cx25821_dev *dev) q->mem_ops = &vb2_dma_sg_memops; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &dev->lock; + q->dev = &dev->pci->dev; if (!is_output) { err = vb2_queue_init(q); diff --git a/drivers/media/pci/cx25821/cx25821.h b/drivers/media/pci/cx25821/cx25821.h index a513b68be0fa..35c7375e4617 100644 --- a/drivers/media/pci/cx25821/cx25821.h +++ b/drivers/media/pci/cx25821/cx25821.h @@ -249,7 +249,6 @@ struct cx25821_dev { int hwrevision; /* used by cx25821-alsa */ struct snd_card *card; - void *alloc_ctx; u32 clk_freq; diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c index e158a1da1d41..f3f13eb0c16e 100644 --- a/drivers/media/pci/cx88/cx88-alsa.c +++ b/drivers/media/pci/cx88/cx88-alsa.c @@ -799,13 +799,9 @@ static int snd_cx88_alc_put(struct snd_kcontrol *kcontrol, { snd_cx88_card_t *chip = snd_kcontrol_chip(kcontrol); struct cx88_core *core = chip->core; - struct v4l2_control client_ctl; - - memset(&client_ctl, 0, sizeof(client_ctl)); - client_ctl.value = 0 != value->value.integer.value[0]; - client_ctl.id = V4L2_CID_AUDIO_LOUDNESS; - call_hw(core, WM8775_GID, core, s_ctrl, &client_ctl); + wm8775_s_ctrl(core, V4L2_CID_AUDIO_LOUDNESS, + value->value.integer.value[0] != 0); return 0; } diff --git a/drivers/media/pci/cx88/cx88-blackbird.c b/drivers/media/pci/cx88/cx88-blackbird.c index 3233d45d1e5b..04fe9af2a802 100644 --- a/drivers/media/pci/cx88/cx88-blackbird.c +++ b/drivers/media/pci/cx88/cx88-blackbird.c @@ -639,7 +639,7 @@ static int blackbird_stop_codec(struct cx8802_dev *dev) static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx8802_dev *dev = q->drv_priv; @@ -647,7 +647,6 @@ static int queue_setup(struct vb2_queue *q, dev->ts_packet_size = 188 * 4; dev->ts_packet_count = 32; sizes[0] = dev->ts_packet_size * dev->ts_packet_count; - alloc_ctxs[0] = dev->alloc_ctx; return 0; } @@ -1183,6 +1182,7 @@ static int cx8802_blackbird_probe(struct cx8802_driver *drv) q->mem_ops = &vb2_dma_sg_memops; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &core->lock; + q->dev = &dev->pci->dev; err = vb2_queue_init(q); if (err < 0) diff --git a/drivers/media/pci/cx88/cx88-dvb.c b/drivers/media/pci/cx88/cx88-dvb.c index 851d2a9caed3..5bb63e7a5691 100644 --- a/drivers/media/pci/cx88/cx88-dvb.c +++ b/drivers/media/pci/cx88/cx88-dvb.c @@ -84,7 +84,7 @@ DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx8802_dev *dev = q->drv_priv; @@ -92,7 +92,6 @@ static int queue_setup(struct vb2_queue *q, dev->ts_packet_size = 188 * 4; dev->ts_packet_count = dvb_buf_tscnt; sizes[0] = dev->ts_packet_size * dev->ts_packet_count; - alloc_ctxs[0] = dev->alloc_ctx; *num_buffers = dvb_buf_tscnt; return 0; } @@ -1793,6 +1792,7 @@ static int cx8802_dvb_probe(struct cx8802_driver *drv) q->mem_ops = &vb2_dma_sg_memops; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &core->lock; + q->dev = &dev->pci->dev; err = vb2_queue_init(q); if (err < 0) diff --git a/drivers/media/pci/cx88/cx88-mpeg.c b/drivers/media/pci/cx88/cx88-mpeg.c index f34c229f9b37..245357adbc25 100644 --- a/drivers/media/pci/cx88/cx88-mpeg.c +++ b/drivers/media/pci/cx88/cx88-mpeg.c @@ -726,11 +726,6 @@ static int cx8802_probe(struct pci_dev *pci_dev, if (NULL == dev) goto fail_core; dev->pci = pci_dev; - dev->alloc_ctx = vb2_dma_sg_init_ctx(&pci_dev->dev); - if (IS_ERR(dev->alloc_ctx)) { - err = PTR_ERR(dev->alloc_ctx); - goto fail_dev; - } dev->core = core; /* Maintain a reference so cx88-video can query the 8802 device. */ @@ -738,7 +733,7 @@ static int cx8802_probe(struct pci_dev *pci_dev, err = cx8802_init_common(dev); if (err != 0) - goto fail_free; + goto fail_dev; INIT_LIST_HEAD(&dev->drvlist); mutex_lock(&cx8802_mutex); @@ -749,8 +744,6 @@ static int cx8802_probe(struct pci_dev *pci_dev, request_modules(dev); return 0; - fail_free: - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); fail_dev: kfree(dev); fail_core: @@ -798,7 +791,6 @@ static void cx8802_remove(struct pci_dev *pci_dev) /* common */ cx8802_fini_common(dev); cx88_core_put(dev->core,dev->pci); - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); kfree(dev); } diff --git a/drivers/media/pci/cx88/cx88-vbi.c b/drivers/media/pci/cx88/cx88-vbi.c index ccc646d819f2..d3237cf8ffa3 100644 --- a/drivers/media/pci/cx88/cx88-vbi.c +++ b/drivers/media/pci/cx88/cx88-vbi.c @@ -109,7 +109,7 @@ int cx8800_restart_vbi_queue(struct cx8800_dev *dev, static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx8800_dev *dev = q->drv_priv; @@ -118,7 +118,6 @@ static int queue_setup(struct vb2_queue *q, sizes[0] = VBI_LINE_NTSC_COUNT * VBI_LINE_LENGTH * 2; else sizes[0] = VBI_LINE_PAL_COUNT * VBI_LINE_LENGTH * 2; - alloc_ctxs[0] = dev->alloc_ctx; return 0; } diff --git a/drivers/media/pci/cx88/cx88-video.c b/drivers/media/pci/cx88/cx88-video.c index 5f331df65fb9..5dc1e3f08d50 100644 --- a/drivers/media/pci/cx88/cx88-video.c +++ b/drivers/media/pci/cx88/cx88-video.c @@ -431,14 +431,13 @@ static int restart_video_queue(struct cx8800_dev *dev, static int queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct cx8800_dev *dev = q->drv_priv; struct cx88_core *core = dev->core; *num_planes = 1; sizes[0] = (dev->fmt->depth * core->width * core->height) >> 3; - alloc_ctxs[0] = dev->alloc_ctx; return 0; } @@ -1319,12 +1318,6 @@ static int cx8800_initdev(struct pci_dev *pci_dev, printk("%s/0: Oops: no 32bit PCI DMA ???\n",core->name); goto fail_core; } - dev->alloc_ctx = vb2_dma_sg_init_ctx(&pci_dev->dev); - if (IS_ERR(dev->alloc_ctx)) { - err = PTR_ERR(dev->alloc_ctx); - goto fail_core; - } - /* initialize driver struct */ spin_lock_init(&dev->slock); @@ -1445,6 +1438,7 @@ static int cx8800_initdev(struct pci_dev *pci_dev, q->mem_ops = &vb2_dma_sg_memops; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &core->lock; + q->dev = &dev->pci->dev; err = vb2_queue_init(q); if (err < 0) @@ -1461,6 +1455,7 @@ static int cx8800_initdev(struct pci_dev *pci_dev, q->mem_ops = &vb2_dma_sg_memops; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &core->lock; + q->dev = &dev->pci->dev; err = vb2_queue_init(q); if (err < 0) @@ -1530,7 +1525,6 @@ fail_unreg: free_irq(pci_dev->irq, dev); mutex_unlock(&core->lock); fail_core: - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); core->v4ldev = NULL; cx88_core_put(core,dev->pci); fail_free: @@ -1564,7 +1558,6 @@ static void cx8800_finidev(struct pci_dev *pci_dev) /* free memory */ cx88_core_put(core,dev->pci); - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); kfree(dev); } diff --git a/drivers/media/pci/cx88/cx88.h b/drivers/media/pci/cx88/cx88.h index 78f817ee7e41..ecd4b7bece99 100644 --- a/drivers/media/pci/cx88/cx88.h +++ b/drivers/media/pci/cx88/cx88.h @@ -485,7 +485,6 @@ struct cx8800_dev { /* pci i/o */ struct pci_dev *pci; unsigned char pci_rev,pci_lat; - void *alloc_ctx; const struct cx8800_fmt *fmt; @@ -549,7 +548,6 @@ struct cx8802_dev { /* pci i/o */ struct pci_dev *pci; unsigned char pci_rev,pci_lat; - void *alloc_ctx; /* dma queues */ struct cx88_dmaqueue mpegq; diff --git a/drivers/media/pci/ddbridge/ddbridge-core.c b/drivers/media/pci/ddbridge/ddbridge-core.c index 6e995ef8c37e..47def73b3502 100644 --- a/drivers/media/pci/ddbridge/ddbridge-core.c +++ b/drivers/media/pci/ddbridge/ddbridge-core.c @@ -1569,10 +1569,9 @@ static int ddb_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (pci_enable_device(pdev) < 0) return -ENODEV; - dev = vmalloc(sizeof(struct ddb)); + dev = vzalloc(sizeof(struct ddb)); if (dev == NULL) return -ENOMEM; - memset(dev, 0, sizeof(struct ddb)); dev->pdev = pdev; pci_set_drvdata(pdev, dev); diff --git a/drivers/media/pci/dt3155/dt3155.c b/drivers/media/pci/dt3155/dt3155.c index 568c0c8fb2dc..6a219694b225 100644 --- a/drivers/media/pci/dt3155/dt3155.c +++ b/drivers/media/pci/dt3155/dt3155.c @@ -133,7 +133,7 @@ static int wait_i2c_reg(void __iomem *addr) static int dt3155_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct dt3155_priv *pd = vb2_get_drv_priv(vq); @@ -141,7 +141,6 @@ dt3155_queue_setup(struct vb2_queue *vq, if (vq->num_buffers + *nbuffers < 2) *nbuffers = 2 - vq->num_buffers; - alloc_ctxs[0] = pd->alloc_ctx; if (*num_planes) return sizes[0] < size ? -EINVAL : 0; *num_planes = 1; @@ -544,21 +543,16 @@ static int dt3155_probe(struct pci_dev *pdev, const struct pci_device_id *id) pd->vidq.min_buffers_needed = 2; pd->vidq.gfp_flags = GFP_DMA32; pd->vidq.lock = &pd->mux; /* for locking v4l2_file_operations */ + pd->vidq.dev = &pdev->dev; pd->vdev.queue = &pd->vidq; err = vb2_queue_init(&pd->vidq); if (err < 0) goto err_v4l2_dev_unreg; - pd->alloc_ctx = vb2_dma_contig_init_ctx(&pdev->dev); - if (IS_ERR(pd->alloc_ctx)) { - dev_err(&pdev->dev, "Can't allocate buffer context"); - err = PTR_ERR(pd->alloc_ctx); - goto err_v4l2_dev_unreg; - } spin_lock_init(&pd->lock); pd->config = ACQ_MODE_EVEN; err = pci_enable_device(pdev); if (err) - goto err_free_ctx; + goto err_v4l2_dev_unreg; err = pci_request_region(pdev, 0, pci_name(pdev)); if (err) goto err_pci_disable; @@ -588,8 +582,6 @@ err_free_reg: pci_release_region(pdev, 0); err_pci_disable: pci_disable_device(pdev); -err_free_ctx: - vb2_dma_contig_cleanup_ctx(pd->alloc_ctx); err_v4l2_dev_unreg: v4l2_device_unregister(&pd->v4l2_dev); return err; @@ -608,7 +600,6 @@ static void dt3155_remove(struct pci_dev *pdev) pci_iounmap(pdev, pd->regs); pci_release_region(pdev, 0); pci_disable_device(pdev); - vb2_dma_contig_cleanup_ctx(pd->alloc_ctx); } static const struct pci_device_id pci_ids[] = { diff --git a/drivers/media/pci/dt3155/dt3155.h b/drivers/media/pci/dt3155/dt3155.h index b3531e0bc733..39442e58919d 100644 --- a/drivers/media/pci/dt3155/dt3155.h +++ b/drivers/media/pci/dt3155/dt3155.h @@ -161,7 +161,6 @@ * @vdev: video_device structure * @pdev: pointer to pci_dev structure * @vidq: vb2_queue structure - * @alloc_ctx: dma_contig allocation context * @curr_buf: pointer to curren buffer * @mux: mutex to protect the instance * @dmaq: queue for dma buffers @@ -181,7 +180,6 @@ struct dt3155_priv { struct video_device vdev; struct pci_dev *pdev; struct vb2_queue vidq; - struct vb2_alloc_ctx *alloc_ctx; struct vb2_v4l2_buffer *curr_buf; struct mutex mux; struct list_head dmaq; diff --git a/drivers/media/pci/ivtv/ivtv-alsa-mixer.c b/drivers/media/pci/ivtv/ivtv-alsa-mixer.c index 33ec05b09af3..79b24bde4a39 100644 --- a/drivers/media/pci/ivtv/ivtv-alsa-mixer.c +++ b/drivers/media/pci/ivtv/ivtv-alsa-mixer.c @@ -93,7 +93,7 @@ static int snd_ivtv_mixer_tv_vol_get(struct snd_kcontrol *kctl, vctrl.value = dB_to_cx25840_vol(uctl->value.integer.value[0]); snd_ivtv_lock(itvsc); - ret = v4l2_subdev_call(itv->sd_audio, core, g_ctrl, &vctrl); + ret = v4l2_g_ctrl(itv->sd_audio->ctrl_handler, &vctrl); snd_ivtv_unlock(itvsc); if (!ret) @@ -115,14 +115,14 @@ static int snd_ivtv_mixer_tv_vol_put(struct snd_kcontrol *kctl, snd_ivtv_lock(itvsc); /* Fetch current state */ - ret = v4l2_subdev_call(itv->sd_audio, core, g_ctrl, &vctrl); + ret = v4l2_g_ctrl(itv->sd_audio->ctrl_handler, &vctrl); if (ret || (cx25840_vol_to_dB(vctrl.value) != uctl->value.integer.value[0])) { /* Set, if needed */ vctrl.value = dB_to_cx25840_vol(uctl->value.integer.value[0]); - ret = v4l2_subdev_call(itv->sd_audio, core, s_ctrl, &vctrl); + ret = v4l2_s_ctrl(itv->sd_audio->ctrl_handler, &vctrl); if (!ret) ret = 1; /* Indicate control was changed w/o error */ } diff --git a/drivers/media/pci/netup_unidvb/Kconfig b/drivers/media/pci/netup_unidvb/Kconfig index f277b0b10c2d..0ad37714c7fd 100644 --- a/drivers/media/pci/netup_unidvb/Kconfig +++ b/drivers/media/pci/netup_unidvb/Kconfig @@ -5,8 +5,13 @@ config DVB_NETUP_UNIDVB select VIDEOBUF2_VMALLOC select DVB_HORUS3A if MEDIA_SUBDRV_AUTOSELECT select DVB_ASCOT2E if MEDIA_SUBDRV_AUTOSELECT + select DVB_HELENE if MEDIA_SUBDRV_AUTOSELECT select DVB_LNBH25 if MEDIA_SUBDRV_AUTOSELECT select DVB_CXD2841ER if MEDIA_SUBDRV_AUTOSELECT ---help--- Support for NetUP PCI express Universal DVB card. - + help + Say Y when you want to support NetUP Dual Universal DVB card + Card can receive two independent streams in following standards: + DVB-S/S2, T/T2, C/C2 + Two CI slots available for CAM modules. diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb.h b/drivers/media/pci/netup_unidvb/netup_unidvb.h index a67b28111905..39b08ecda1fc 100644 --- a/drivers/media/pci/netup_unidvb/netup_unidvb.h +++ b/drivers/media/pci/netup_unidvb/netup_unidvb.h @@ -50,6 +50,15 @@ #define NETUP_UNIDVB_IRQ_CAM0 (1 << 11) #define NETUP_UNIDVB_IRQ_CAM1 (1 << 12) +/* NetUP Universal DVB card hardware revisions and it's PCI device id's: + * 1.3 - CXD2841ER demod, ASCOT2E and HORUS3A tuners + * 1.4 - CXD2854ER demod, HELENE tuner +*/ +enum netup_hw_rev { + NETUP_HW_REV_1_3 = 0x18F6, + NETUP_HW_REV_1_4 = 0x18F7 +}; + struct netup_dma { u8 num; spinlock_t lock; @@ -119,6 +128,7 @@ struct netup_unidvb_dev { struct netup_dma dma[2]; struct netup_ci_state ci[2]; struct netup_spi *spi; + enum netup_hw_rev rev; }; int netup_i2c_register(struct netup_unidvb_dev *ndev); diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_ci.c b/drivers/media/pci/netup_unidvb/netup_unidvb_ci.c index f46ffac66ee9..f535270c2116 100644 --- a/drivers/media/pci/netup_unidvb/netup_unidvb_ci.c +++ b/drivers/media/pci/netup_unidvb/netup_unidvb_ci.c @@ -147,7 +147,7 @@ static int netup_unidvb_ci_read_attribute_mem(struct dvb_ca_en50221 *en50221, { struct netup_ci_state *state = en50221->data; struct netup_unidvb_dev *dev = state->dev; - u8 val = *((u8 __force *)state->membase8_io + addr); + u8 val = *((u8 __force *)state->membase8_config + addr); dev_dbg(&dev->pci_dev->dev, "%s(): addr=0x%x val=0x%x\n", __func__, addr, val); @@ -162,7 +162,7 @@ static int netup_unidvb_ci_write_attribute_mem(struct dvb_ca_en50221 *en50221, dev_dbg(&dev->pci_dev->dev, "%s(): addr=0x%x data=0x%x\n", __func__, addr, data); - *((u8 __force *)state->membase8_io + addr) = data; + *((u8 __force *)state->membase8_config + addr) = data; return 0; } diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c index 2b667b315913..ac547cb84de8 100644 --- a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c +++ b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c @@ -34,6 +34,7 @@ #include "cxd2841er.h" #include "horus3a.h" #include "ascot2e.h" +#include "helene.h" #include "lnbh25.h" static int spi_enable; @@ -120,7 +121,8 @@ static int netup_unidvb_tuner_ctrl(void *priv, int is_dvb_tc); static void netup_unidvb_queue_cleanup(struct netup_dma *dma); static struct cxd2841er_config demod_config = { - .i2c_addr = 0xc8 + .i2c_addr = 0xc8, + .xtal = SONY_XTAL_24000 }; static struct horus3a_config horus3a_conf = { @@ -134,6 +136,12 @@ static struct ascot2e_config ascot2e_conf = { .set_tuner_callback = netup_unidvb_tuner_ctrl }; +static struct helene_config helene_conf = { + .i2c_address = 0xc0, + .xtal = SONY_HELENE_XTAL_24000, + .set_tuner_callback = netup_unidvb_tuner_ctrl +}; + static struct lnbh25_config lnbh25_conf = { .i2c_address = 0x10, .data2_config = LNBH25_TEN | LNBH25_EXTM @@ -152,6 +160,11 @@ static int netup_unidvb_tuner_ctrl(void *priv, int is_dvb_tc) __func__, dma->num, is_dvb_tc); reg = readb(ndev->bmmio0 + GPIO_REG_IO); mask = (dma->num == 0) ? GPIO_RFA_CTL : GPIO_RFB_CTL; + + /* inverted tuner control in hw rev. 1.4 */ + if (ndev->rev == NETUP_HW_REV_1_4) + is_dvb_tc = !is_dvb_tc; + if (!is_dvb_tc) reg |= mask; else @@ -280,7 +293,7 @@ static int netup_unidvb_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, unsigned int sizes[], - void *alloc_ctxs[]) + struct device *alloc_devs[]) { struct netup_dma *dma = vb2_get_drv_priv(vq); @@ -372,7 +385,15 @@ static int netup_unidvb_queue_init(struct netup_dma *dma, static int netup_unidvb_dvb_init(struct netup_unidvb_dev *ndev, int num) { - struct vb2_dvb_frontend *fe0, *fe1, *fe2; + int fe_count = 2; + int i = 0; + struct vb2_dvb_frontend *fes[2]; + u8 fe_name[32]; + + if (ndev->rev == NETUP_HW_REV_1_3) + demod_config.xtal = SONY_XTAL_20500; + else + demod_config.xtal = SONY_XTAL_24000; if (num < 0 || num > 1) { dev_dbg(&ndev->pci_dev->dev, @@ -381,84 +402,96 @@ static int netup_unidvb_dvb_init(struct netup_unidvb_dev *ndev, } mutex_init(&ndev->frontends[num].lock); INIT_LIST_HEAD(&ndev->frontends[num].felist); - if (vb2_dvb_alloc_frontend(&ndev->frontends[num], 1) == NULL || - vb2_dvb_alloc_frontend( - &ndev->frontends[num], 2) == NULL || - vb2_dvb_alloc_frontend( - &ndev->frontends[num], 3) == NULL) { - dev_dbg(&ndev->pci_dev->dev, - "%s(): unable to allocate vb2_dvb_frontend\n", - __func__); - return -ENOMEM; + + for (i = 0; i < fe_count; i++) { + if (vb2_dvb_alloc_frontend(&ndev->frontends[num], i+1) + == NULL) { + dev_err(&ndev->pci_dev->dev, + "%s(): unable to allocate vb2_dvb_frontend\n", + __func__); + return -ENOMEM; + } } - fe0 = vb2_dvb_get_frontend(&ndev->frontends[num], 1); - fe1 = vb2_dvb_get_frontend(&ndev->frontends[num], 2); - fe2 = vb2_dvb_get_frontend(&ndev->frontends[num], 3); - if (fe0 == NULL || fe1 == NULL || fe2 == NULL) { - dev_dbg(&ndev->pci_dev->dev, - "%s(): frontends has not been allocated\n", __func__); - return -EINVAL; + + for (i = 0; i < fe_count; i++) { + fes[i] = vb2_dvb_get_frontend(&ndev->frontends[num], i+1); + if (fes[i] == NULL) { + dev_err(&ndev->pci_dev->dev, + "%s(): frontends has not been allocated\n", + __func__); + return -EINVAL; + } + } + + for (i = 0; i < fe_count; i++) { + netup_unidvb_queue_init(&ndev->dma[num], &fes[i]->dvb.dvbq); + snprintf(fe_name, sizeof(fe_name), "netup_fe%d", i); + fes[i]->dvb.name = fe_name; } - netup_unidvb_queue_init(&ndev->dma[num], &fe0->dvb.dvbq); - netup_unidvb_queue_init(&ndev->dma[num], &fe1->dvb.dvbq); - netup_unidvb_queue_init(&ndev->dma[num], &fe2->dvb.dvbq); - fe0->dvb.name = "netup_fe0"; - fe1->dvb.name = "netup_fe1"; - fe2->dvb.name = "netup_fe2"; - fe0->dvb.frontend = dvb_attach(cxd2841er_attach_s, + + fes[0]->dvb.frontend = dvb_attach(cxd2841er_attach_s, &demod_config, &ndev->i2c[num].adap); - if (fe0->dvb.frontend == NULL) { + if (fes[0]->dvb.frontend == NULL) { dev_dbg(&ndev->pci_dev->dev, "%s(): unable to attach DVB-S/S2 frontend\n", __func__); goto frontend_detach; } - horus3a_conf.set_tuner_priv = &ndev->dma[num]; - if (!dvb_attach(horus3a_attach, fe0->dvb.frontend, - &horus3a_conf, &ndev->i2c[num].adap)) { - dev_dbg(&ndev->pci_dev->dev, - "%s(): unable to attach DVB-S/S2 tuner frontend\n", - __func__); - goto frontend_detach; + + if (ndev->rev == NETUP_HW_REV_1_3) { + horus3a_conf.set_tuner_priv = &ndev->dma[num]; + if (!dvb_attach(horus3a_attach, fes[0]->dvb.frontend, + &horus3a_conf, &ndev->i2c[num].adap)) { + dev_dbg(&ndev->pci_dev->dev, + "%s(): unable to attach HORUS3A DVB-S/S2 tuner frontend\n", + __func__); + goto frontend_detach; + } + } else { + helene_conf.set_tuner_priv = &ndev->dma[num]; + if (!dvb_attach(helene_attach_s, fes[0]->dvb.frontend, + &helene_conf, &ndev->i2c[num].adap)) { + dev_err(&ndev->pci_dev->dev, + "%s(): unable to attach HELENE DVB-S/S2 tuner frontend\n", + __func__); + goto frontend_detach; + } } - if (!dvb_attach(lnbh25_attach, fe0->dvb.frontend, + + if (!dvb_attach(lnbh25_attach, fes[0]->dvb.frontend, &lnbh25_conf, &ndev->i2c[num].adap)) { dev_dbg(&ndev->pci_dev->dev, "%s(): unable to attach SEC frontend\n", __func__); goto frontend_detach; } + /* DVB-T/T2 frontend */ - fe1->dvb.frontend = dvb_attach(cxd2841er_attach_t, + fes[1]->dvb.frontend = dvb_attach(cxd2841er_attach_t_c, &demod_config, &ndev->i2c[num].adap); - if (fe1->dvb.frontend == NULL) { - dev_dbg(&ndev->pci_dev->dev, - "%s(): unable to attach DVB-T frontend\n", __func__); - goto frontend_detach; - } - fe1->dvb.frontend->id = 1; - ascot2e_conf.set_tuner_priv = &ndev->dma[num]; - if (!dvb_attach(ascot2e_attach, fe1->dvb.frontend, - &ascot2e_conf, &ndev->i2c[num].adap)) { - dev_dbg(&ndev->pci_dev->dev, - "%s(): unable to attach DVB-T tuner frontend\n", - __func__); - goto frontend_detach; - } - /* DVB-C/C2 frontend */ - fe2->dvb.frontend = dvb_attach(cxd2841er_attach_c, - &demod_config, &ndev->i2c[num].adap); - if (fe2->dvb.frontend == NULL) { + if (fes[1]->dvb.frontend == NULL) { dev_dbg(&ndev->pci_dev->dev, - "%s(): unable to attach DVB-C frontend\n", __func__); + "%s(): unable to attach Ter frontend\n", __func__); goto frontend_detach; } - fe2->dvb.frontend->id = 2; - if (!dvb_attach(ascot2e_attach, fe2->dvb.frontend, - &ascot2e_conf, &ndev->i2c[num].adap)) { - dev_dbg(&ndev->pci_dev->dev, - "%s(): unable to attach DVB-T/C tuner frontend\n", - __func__); - goto frontend_detach; + fes[1]->dvb.frontend->id = 1; + if (ndev->rev == NETUP_HW_REV_1_3) { + ascot2e_conf.set_tuner_priv = &ndev->dma[num]; + if (!dvb_attach(ascot2e_attach, fes[1]->dvb.frontend, + &ascot2e_conf, &ndev->i2c[num].adap)) { + dev_dbg(&ndev->pci_dev->dev, + "%s(): unable to attach Ter tuner frontend\n", + __func__); + goto frontend_detach; + } + } else { + helene_conf.set_tuner_priv = &ndev->dma[num]; + if (!dvb_attach(helene_attach, fes[1]->dvb.frontend, + &helene_conf, &ndev->i2c[num].adap)) { + dev_err(&ndev->pci_dev->dev, + "%s(): unable to attach HELENE Ter tuner frontend\n", + __func__); + goto frontend_detach; + } } if (vb2_dvb_register_bus(&ndev->frontends[num], @@ -730,7 +763,7 @@ static int netup_unidvb_request_mmio(struct pci_dev *pci_dev) static int netup_unidvb_request_modules(struct device *dev) { static const char * const modules[] = { - "lnbh25", "ascot2e", "horus3a", "cxd2841er", NULL + "lnbh25", "ascot2e", "horus3a", "cxd2841er", "helene", NULL }; const char * const *curr_mod = modules; int err; @@ -774,6 +807,16 @@ static int netup_unidvb_initdev(struct pci_dev *pci_dev, if (!ndev) goto dev_alloc_err; + /* detect hardware revision */ + if (pci_dev->device == NETUP_HW_REV_1_3) + ndev->rev = NETUP_HW_REV_1_3; + else + ndev->rev = NETUP_HW_REV_1_4; + + dev_info(&pci_dev->dev, + "%s(): board (0x%x) hardware revision 0x%x\n", + __func__, pci_dev->device, ndev->rev); + ndev->old_fw = old_firmware; ndev->wq = create_singlethread_workqueue(NETUP_UNIDVB_NAME); if (!ndev->wq) { @@ -932,7 +975,7 @@ wq_create_err: kfree(ndev); dev_alloc_err: dev_err(&pci_dev->dev, - "%s(): failed to initizalize device\n", __func__); + "%s(): failed to initialize device\n", __func__); return -EIO; } @@ -972,7 +1015,8 @@ static void netup_unidvb_finidev(struct pci_dev *pci_dev) static struct pci_device_id netup_unidvb_pci_tbl[] = { - { PCI_DEVICE(0x1b55, 0x18f6) }, + { PCI_DEVICE(0x1b55, 0x18f6) }, /* hw rev. 1.3 */ + { PCI_DEVICE(0x1b55, 0x18f7) }, /* hw rev. 1.4 */ { 0, } }; MODULE_DEVICE_TABLE(pci, netup_unidvb_pci_tbl); diff --git a/drivers/media/pci/saa7134/saa7134-core.c b/drivers/media/pci/saa7134/saa7134-core.c index c0e1780ec831..ffb66a9ae23e 100644 --- a/drivers/media/pci/saa7134/saa7134-core.c +++ b/drivers/media/pci/saa7134/saa7134-core.c @@ -1164,18 +1164,13 @@ static int saa7134_initdev(struct pci_dev *pci_dev, saa7134_board_init1(dev); saa7134_hwinit1(dev); - dev->alloc_ctx = vb2_dma_sg_init_ctx(&pci_dev->dev); - if (IS_ERR(dev->alloc_ctx)) { - err = PTR_ERR(dev->alloc_ctx); - goto fail3; - } /* get irq */ err = request_irq(pci_dev->irq, saa7134_irq, IRQF_SHARED, dev->name, dev); if (err < 0) { pr_err("%s: can't get IRQ %d\n", dev->name,pci_dev->irq); - goto fail4; + goto fail3; } /* wait a bit, register i2c bus */ @@ -1233,7 +1228,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, if (err < 0) { pr_info("%s: can't register video device\n", dev->name); - goto fail5; + goto fail4; } pr_info("%s: registered device %s [v4l2]\n", dev->name, video_device_node_name(dev->video_dev)); @@ -1246,7 +1241,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = video_register_device(dev->vbi_dev,VFL_TYPE_VBI, vbi_nr[dev->nr]); if (err < 0) - goto fail5; + goto fail4; pr_info("%s: registered device %s\n", dev->name, video_device_node_name(dev->vbi_dev)); @@ -1257,7 +1252,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = video_register_device(dev->radio_dev,VFL_TYPE_RADIO, radio_nr[dev->nr]); if (err < 0) - goto fail5; + goto fail4; pr_info("%s: registered device %s\n", dev->name, video_device_node_name(dev->radio_dev)); } @@ -1268,7 +1263,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = v4l2_mc_create_media_graph(dev->media_dev); if (err) { pr_err("failed to create media graph\n"); - goto fail5; + goto fail4; } #endif /* everything worked */ @@ -1287,17 +1282,15 @@ static int saa7134_initdev(struct pci_dev *pci_dev, #ifdef CONFIG_MEDIA_CONTROLLER err = media_device_register(dev->media_dev); if (err) - goto fail5; + goto fail4; #endif return 0; - fail5: + fail4: saa7134_unregister_video(dev); saa7134_i2c_unregister(dev); free_irq(pci_dev->irq, dev); - fail4: - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); fail3: saa7134_hwfini(dev); iounmap(dev->lmmio); @@ -1367,7 +1360,6 @@ static void saa7134_finidev(struct pci_dev *pci_dev) /* release resources */ free_irq(pci_dev->irq, dev); - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); iounmap(dev->lmmio); release_mem_region(pci_resource_start(pci_dev,0), pci_resource_len(pci_dev,0)); diff --git a/drivers/media/pci/saa7134/saa7134-ts.c b/drivers/media/pci/saa7134/saa7134-ts.c index 0584a2adbe99..7eaf36a41db9 100644 --- a/drivers/media/pci/saa7134/saa7134-ts.c +++ b/drivers/media/pci/saa7134/saa7134-ts.c @@ -118,7 +118,7 @@ EXPORT_SYMBOL_GPL(saa7134_ts_buffer_prepare); int saa7134_ts_queue_setup(struct vb2_queue *q, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct saa7134_dmaqueue *dmaq = q->drv_priv; struct saa7134_dev *dev = dmaq->dev; @@ -131,7 +131,6 @@ int saa7134_ts_queue_setup(struct vb2_queue *q, *nbuffers = 3; *nplanes = 1; sizes[0] = size; - alloc_ctxs[0] = dev->alloc_ctx; return 0; } EXPORT_SYMBOL_GPL(saa7134_ts_queue_setup); diff --git a/drivers/media/pci/saa7134/saa7134-vbi.c b/drivers/media/pci/saa7134/saa7134-vbi.c index e76da37c4a8a..cf9a31e0a390 100644 --- a/drivers/media/pci/saa7134/saa7134-vbi.c +++ b/drivers/media/pci/saa7134/saa7134-vbi.c @@ -140,7 +140,7 @@ static int buffer_prepare(struct vb2_buffer *vb2) static int queue_setup(struct vb2_queue *q, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct saa7134_dmaqueue *dmaq = q->drv_priv; struct saa7134_dev *dev = dmaq->dev; @@ -155,7 +155,6 @@ static int queue_setup(struct vb2_queue *q, *nbuffers = saa7134_buffer_count(size, *nbuffers); *nplanes = 1; sizes[0] = size; - alloc_ctxs[0] = dev->alloc_ctx; return 0; } diff --git a/drivers/media/pci/saa7134/saa7134-video.c b/drivers/media/pci/saa7134/saa7134-video.c index ffa39543eb65..8a6ebd087889 100644 --- a/drivers/media/pci/saa7134/saa7134-video.c +++ b/drivers/media/pci/saa7134/saa7134-video.c @@ -963,7 +963,7 @@ static int buffer_prepare(struct vb2_buffer *vb2) static int queue_setup(struct vb2_queue *q, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct saa7134_dmaqueue *dmaq = q->drv_priv; struct saa7134_dev *dev = dmaq->dev; @@ -980,7 +980,6 @@ static int queue_setup(struct vb2_queue *q, *nbuffers = saa7134_buffer_count(size, *nbuffers); *nplanes = 1; sizes[0] = size; - alloc_ctxs[0] = dev->alloc_ctx; saa7134_enable_analog_tuner(dev); @@ -2173,6 +2172,7 @@ int saa7134_video_init1(struct saa7134_dev *dev) q->buf_struct_size = sizeof(struct saa7134_buf); q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &dev->lock; + q->dev = &dev->pci->dev; ret = vb2_queue_init(q); if (ret) return ret; @@ -2191,6 +2191,7 @@ int saa7134_video_init1(struct saa7134_dev *dev) q->buf_struct_size = sizeof(struct saa7134_buf); q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &dev->lock; + q->dev = &dev->pci->dev; ret = vb2_queue_init(q); if (ret) return ret; diff --git a/drivers/media/pci/saa7134/saa7134.h b/drivers/media/pci/saa7134/saa7134.h index 69a9bbf22d4d..3849083526a7 100644 --- a/drivers/media/pci/saa7134/saa7134.h +++ b/drivers/media/pci/saa7134/saa7134.h @@ -610,7 +610,6 @@ struct saa7134_dev { /* video+ts+vbi capture */ - void *alloc_ctx; struct saa7134_dmaqueue video_q; struct vb2_queue video_vbq; struct saa7134_dmaqueue vbi_q; @@ -854,7 +853,7 @@ int saa7134_ts_buffer_init(struct vb2_buffer *vb2); int saa7134_ts_buffer_prepare(struct vb2_buffer *vb2); int saa7134_ts_queue_setup(struct vb2_queue *q, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]); + unsigned int sizes[], struct device *alloc_devs[]); int saa7134_ts_start_streaming(struct vb2_queue *vq, unsigned int count); void saa7134_ts_stop_streaming(struct vb2_queue *vq); diff --git a/drivers/media/pci/saa7164/saa7164-encoder.c b/drivers/media/pci/saa7164/saa7164-encoder.c index 1b184c39ba97..32a353d162e7 100644 --- a/drivers/media/pci/saa7164/saa7164-encoder.c +++ b/drivers/media/pci/saa7164/saa7164-encoder.c @@ -1022,8 +1022,7 @@ int saa7164_encoder_register(struct saa7164_port *port) dprintk(DBGLVL_ENC, "%s()\n", __func__); - if (port->type != SAA7164_MPEG_ENCODER) - BUG(); + BUG_ON(port->type != SAA7164_MPEG_ENCODER); /* Sanity check that the PCI configuration space is active */ if (port->hwcfg.BARLocation == 0) { @@ -1151,8 +1150,7 @@ void saa7164_encoder_unregister(struct saa7164_port *port) dprintk(DBGLVL_ENC, "%s(port=%d)\n", __func__, port->nr); - if (port->type != SAA7164_MPEG_ENCODER) - BUG(); + BUG_ON(port->type != SAA7164_MPEG_ENCODER); if (port->v4l_device) { if (port->v4l_device->minor != -1) diff --git a/drivers/media/pci/saa7164/saa7164.h b/drivers/media/pci/saa7164/saa7164.h index 8337524bfb8c..97411b0384c1 100644 --- a/drivers/media/pci/saa7164/saa7164.h +++ b/drivers/media/pci/saa7164/saa7164.h @@ -263,10 +263,6 @@ struct saa7164_i2c { u32 i2c_rc; }; -struct saa7164_ctrl { - struct v4l2_queryctrl v; -}; - struct saa7164_tvnorm { char *name; v4l2_std_id id; diff --git a/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c b/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c index 67a14c41c227..399164314c28 100644 --- a/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c +++ b/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c @@ -33,7 +33,7 @@ #include "solo6x10-jpeg.h" #define MIN_VID_BUFFERS 2 -#define FRAME_BUF_SIZE (196 * 1024) +#define FRAME_BUF_SIZE (400 * 1024) #define MP4_QS 16 #define DMA_ALIGN 4096 @@ -664,12 +664,9 @@ static int solo_ring_thread(void *data) static int solo_enc_queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, unsigned int sizes[], - void *alloc_ctxs[]) + struct device *alloc_devs[]) { - struct solo_enc_dev *solo_enc = vb2_get_drv_priv(q); - sizes[0] = FRAME_BUF_SIZE; - alloc_ctxs[0] = solo_enc->alloc_ctx; *num_planes = 1; if (*num_buffers < MIN_VID_BUFFERS) @@ -1239,11 +1236,6 @@ static struct solo_enc_dev *solo_enc_alloc(struct solo_dev *solo_dev, return ERR_PTR(-ENOMEM); hdl = &solo_enc->hdl; - solo_enc->alloc_ctx = vb2_dma_sg_init_ctx(&solo_dev->pdev->dev); - if (IS_ERR(solo_enc->alloc_ctx)) { - ret = PTR_ERR(solo_enc->alloc_ctx); - goto hdl_free; - } v4l2_ctrl_handler_init(hdl, 10); v4l2_ctrl_new_std(hdl, &solo_ctrl_ops, V4L2_CID_BRIGHTNESS, 0, 255, 1, 128); @@ -1299,6 +1291,7 @@ static struct solo_enc_dev *solo_enc_alloc(struct solo_dev *solo_dev, solo_enc->vidq.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; solo_enc->vidq.buf_struct_size = sizeof(struct solo_vb2_buf); solo_enc->vidq.lock = &solo_enc->lock; + solo_enc->vidq.dev = &solo_dev->pdev->dev; ret = vb2_queue_init(&solo_enc->vidq); if (ret) goto hdl_free; @@ -1347,7 +1340,6 @@ pci_free: solo_enc->desc_items, solo_enc->desc_dma); hdl_free: v4l2_ctrl_handler_free(hdl); - vb2_dma_sg_cleanup_ctx(solo_enc->alloc_ctx); kfree(solo_enc); return ERR_PTR(ret); } @@ -1362,7 +1354,6 @@ static void solo_enc_free(struct solo_enc_dev *solo_enc) solo_enc->desc_items, solo_enc->desc_dma); video_unregister_device(solo_enc->vfd); v4l2_ctrl_handler_free(&solo_enc->hdl); - vb2_dma_sg_cleanup_ctx(solo_enc->alloc_ctx); kfree(solo_enc); } diff --git a/drivers/media/pci/solo6x10/solo6x10-v4l2.c b/drivers/media/pci/solo6x10/solo6x10-v4l2.c index 721ff5320de7..b4be47969b6b 100644 --- a/drivers/media/pci/solo6x10/solo6x10-v4l2.c +++ b/drivers/media/pci/solo6x10/solo6x10-v4l2.c @@ -315,12 +315,11 @@ static void solo_stop_thread(struct solo_dev *solo_dev) static int solo_queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct solo_dev *solo_dev = vb2_get_drv_priv(q); sizes[0] = solo_image_size(solo_dev); - alloc_ctxs[0] = solo_dev->alloc_ctx; *num_planes = 1; if (*num_buffers < MIN_VID_BUFFERS) @@ -386,26 +385,24 @@ static int solo_querycap(struct file *file, void *priv, static int solo_enum_ext_input(struct solo_dev *solo_dev, struct v4l2_input *input) { - static const char * const dispnames_1[] = { "4UP" }; - static const char * const dispnames_2[] = { "4UP-1", "4UP-2" }; - static const char * const dispnames_5[] = { - "4UP-1", "4UP-2", "4UP-3", "4UP-4", "16UP" - }; - const char * const *dispnames; + int ext = input->index - solo_dev->nr_chans; + unsigned int nup, first; - if (input->index >= (solo_dev->nr_chans + solo_dev->nr_ext)) + if (ext >= solo_dev->nr_ext) return -EINVAL; - if (solo_dev->nr_ext == 5) - dispnames = dispnames_5; - else if (solo_dev->nr_ext == 2) - dispnames = dispnames_2; - else - dispnames = dispnames_1; - - snprintf(input->name, sizeof(input->name), "Multi %s", - dispnames[input->index - solo_dev->nr_chans]); - + nup = (ext == 4) ? 16 : 4; + first = (ext & 3) << 2; /* first channel in the n-up */ + snprintf(input->name, sizeof(input->name), + "Multi %d-up (cameras %d-%d)", + nup, first + 1, first + nup); + /* Possible outputs: + * Multi 4-up (cameras 1-4) + * Multi 4-up (cameras 5-8) + * Multi 4-up (cameras 9-12) + * Multi 4-up (cameras 13-16) + * Multi 16-up (cameras 1-16) + */ return 0; } @@ -681,16 +678,11 @@ int solo_v4l2_init(struct solo_dev *solo_dev, unsigned nr) solo_dev->vidq.gfp_flags = __GFP_DMA32 | __GFP_KSWAPD_RECLAIM; solo_dev->vidq.buf_struct_size = sizeof(struct solo_vb2_buf); solo_dev->vidq.lock = &solo_dev->lock; + solo_dev->vidq.dev = &solo_dev->pdev->dev; ret = vb2_queue_init(&solo_dev->vidq); if (ret < 0) goto fail; - solo_dev->alloc_ctx = vb2_dma_contig_init_ctx(&solo_dev->pdev->dev); - if (IS_ERR(solo_dev->alloc_ctx)) { - dev_err(&solo_dev->pdev->dev, "Can't allocate buffer context"); - return PTR_ERR(solo_dev->alloc_ctx); - } - /* Cycle all the channels and clear */ for (i = 0; i < solo_dev->nr_chans; i++) { solo_v4l2_set_ch(solo_dev, i); @@ -718,7 +710,6 @@ int solo_v4l2_init(struct solo_dev *solo_dev, unsigned nr) fail: video_device_release(solo_dev->vfd); - vb2_dma_contig_cleanup_ctx(solo_dev->alloc_ctx); v4l2_ctrl_handler_free(&solo_dev->disp_hdl); solo_dev->vfd = NULL; return ret; @@ -730,7 +721,6 @@ void solo_v4l2_exit(struct solo_dev *solo_dev) return; video_unregister_device(solo_dev->vfd); - vb2_dma_contig_cleanup_ctx(solo_dev->alloc_ctx); v4l2_ctrl_handler_free(&solo_dev->disp_hdl); solo_dev->vfd = NULL; } diff --git a/drivers/media/pci/solo6x10/solo6x10.h b/drivers/media/pci/solo6x10/solo6x10.h index 4ab6586c0467..5bd498735a66 100644 --- a/drivers/media/pci/solo6x10/solo6x10.h +++ b/drivers/media/pci/solo6x10/solo6x10.h @@ -178,7 +178,6 @@ struct solo_enc_dev { u32 sequence; struct vb2_queue vidq; struct list_head vidq_active; - void *alloc_ctx; int desc_count; int desc_nelts; struct solo_p2m_desc *desc_items; @@ -269,7 +268,6 @@ struct solo_dev { /* Buffer handling */ struct vb2_queue vidq; - struct vb2_alloc_ctx *alloc_ctx; u32 sequence; struct task_struct *kthread; struct mutex lock; diff --git a/drivers/media/pci/sta2x11/sta2x11_vip.c b/drivers/media/pci/sta2x11/sta2x11_vip.c index 1fc195f89686..aeb2b4e2db35 100644 --- a/drivers/media/pci/sta2x11/sta2x11_vip.c +++ b/drivers/media/pci/sta2x11/sta2x11_vip.c @@ -111,7 +111,6 @@ static inline struct vip_buffer *to_vip_buffer(struct vb2_v4l2_buffer *vb2) * @input: input line for video signal ( 0 or 1 ) * @disabled: Device is in power down state * @slock: for excluse acces of registers - * @alloc_ctx: context for videobuf2 * @vb_vidq: queue maintained by videobuf2 layer * @buffer_list: list of buffer in use * @sequence: sequence number of acquired buffer @@ -141,7 +140,6 @@ struct sta2x11_vip { int disabled; spinlock_t slock; - struct vb2_alloc_ctx *alloc_ctx; struct vb2_queue vb_vidq; struct list_head buffer_list; unsigned int sequence; @@ -267,7 +265,7 @@ static void vip_active_buf_next(struct sta2x11_vip *vip) /* Videobuf2 Operations */ static int queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct sta2x11_vip *vip = vb2_get_drv_priv(vq); @@ -276,7 +274,6 @@ static int queue_setup(struct vb2_queue *vq, *nplanes = 1; sizes[0] = vip->format.sizeimage; - alloc_ctxs[0] = vip->alloc_ctx; vip->sequence = 0; vip->active = NULL; @@ -861,25 +858,15 @@ static int sta2x11_vip_init_buffer(struct sta2x11_vip *vip) vip->vb_vidq.ops = &vip_video_qops; vip->vb_vidq.mem_ops = &vb2_dma_contig_memops; vip->vb_vidq.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + vip->vb_vidq.dev = &vip->pdev->dev; err = vb2_queue_init(&vip->vb_vidq); if (err) return err; INIT_LIST_HEAD(&vip->buffer_list); spin_lock_init(&vip->lock); - - - vip->alloc_ctx = vb2_dma_contig_init_ctx(&vip->pdev->dev); - if (IS_ERR(vip->alloc_ctx)) { - v4l2_err(&vip->v4l2_dev, "Can't allocate buffer context"); - return PTR_ERR(vip->alloc_ctx); - } - return 0; } -static void sta2x11_vip_release_buffer(struct sta2x11_vip *vip) -{ - vb2_dma_contig_cleanup_ctx(vip->alloc_ctx); -} + static int sta2x11_vip_init_controls(struct sta2x11_vip *vip) { /* @@ -1120,7 +1107,6 @@ vrelease: video_unregister_device(&vip->video_dev); free_irq(pdev->irq, vip); release_buf: - sta2x11_vip_release_buffer(vip); pci_disable_msi(pdev); unmap: vb2_queue_release(&vip->vb_vidq); diff --git a/drivers/media/pci/tw68/tw68-core.c b/drivers/media/pci/tw68/tw68-core.c index 4e77618fbb2b..8474528be91e 100644 --- a/drivers/media/pci/tw68/tw68-core.c +++ b/drivers/media/pci/tw68/tw68-core.c @@ -305,19 +305,13 @@ static int tw68_initdev(struct pci_dev *pci_dev, /* Then do any initialisation wanted before interrupts are on */ tw68_hw_init1(dev); - dev->alloc_ctx = vb2_dma_sg_init_ctx(&pci_dev->dev); - if (IS_ERR(dev->alloc_ctx)) { - err = PTR_ERR(dev->alloc_ctx); - goto fail3; - } - /* get irq */ err = devm_request_irq(&pci_dev->dev, pci_dev->irq, tw68_irq, IRQF_SHARED, dev->name, dev); if (err < 0) { pr_err("%s: can't get IRQ %d\n", dev->name, pci_dev->irq); - goto fail4; + goto fail3; } /* @@ -331,7 +325,7 @@ static int tw68_initdev(struct pci_dev *pci_dev, if (err < 0) { pr_err("%s: can't register video device\n", dev->name); - goto fail5; + goto fail4; } tw_setl(TW68_INTMASK, dev->pci_irqmask); @@ -340,10 +334,8 @@ static int tw68_initdev(struct pci_dev *pci_dev, return 0; -fail5: - video_unregister_device(&dev->vdev); fail4: - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); + video_unregister_device(&dev->vdev); fail3: iounmap(dev->lmmio); fail2: @@ -367,7 +359,6 @@ static void tw68_finidev(struct pci_dev *pci_dev) /* unregister */ video_unregister_device(&dev->vdev); v4l2_ctrl_handler_free(&dev->hdl); - vb2_dma_sg_cleanup_ctx(dev->alloc_ctx); /* release resources */ iounmap(dev->lmmio); diff --git a/drivers/media/pci/tw68/tw68-video.c b/drivers/media/pci/tw68/tw68-video.c index 07116a87a57b..5e8212845c87 100644 --- a/drivers/media/pci/tw68/tw68-video.c +++ b/drivers/media/pci/tw68/tw68-video.c @@ -378,7 +378,7 @@ static int tw68_buffer_count(unsigned int size, unsigned int count) static int tw68_queue_setup(struct vb2_queue *q, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct tw68_dev *dev = vb2_get_drv_priv(q); unsigned tot_bufs = q->num_buffers + *num_buffers; @@ -388,7 +388,6 @@ static int tw68_queue_setup(struct vb2_queue *q, tot_bufs = 2; tot_bufs = tw68_buffer_count(size, tot_bufs); *num_buffers = tot_bufs - q->num_buffers; - alloc_ctxs[0] = dev->alloc_ctx; /* * We allow create_bufs, but only if the sizeimage is >= as the * current sizeimage. The tw68_buffer_count calculation becomes quite @@ -983,6 +982,7 @@ int tw68_video_init2(struct tw68_dev *dev, int video_nr) dev->vidq.buf_struct_size = sizeof(struct tw68_buf); dev->vidq.lock = &dev->lock; dev->vidq.min_buffers_needed = 2; + dev->vidq.dev = &dev->pci->dev; ret = vb2_queue_init(&dev->vidq); if (ret) return ret; diff --git a/drivers/media/pci/tw68/tw68.h b/drivers/media/pci/tw68/tw68.h index 6c7dcb300f34..5585c7ee23f2 100644 --- a/drivers/media/pci/tw68/tw68.h +++ b/drivers/media/pci/tw68/tw68.h @@ -165,7 +165,6 @@ struct tw68_dev { unsigned field; struct vb2_queue vidq; struct list_head active; - void *alloc_ctx; /* various v4l controls */ const struct tw68_tvnorm *tvnorm; /* video */ diff --git a/drivers/media/pci/tw686x/Kconfig b/drivers/media/pci/tw686x/Kconfig index fb8536974052..34ff37712313 100644 --- a/drivers/media/pci/tw686x/Kconfig +++ b/drivers/media/pci/tw686x/Kconfig @@ -3,6 +3,8 @@ config VIDEO_TW686X depends on PCI && VIDEO_DEV && VIDEO_V4L2 && SND depends on HAS_DMA select VIDEOBUF2_VMALLOC + select VIDEOBUF2_DMA_CONTIG + select VIDEOBUF2_DMA_SG select SND_PCM help Support for Intersil/Techwell TW686x-based frame grabber cards. diff --git a/drivers/media/pci/tw686x/tw686x-audio.c b/drivers/media/pci/tw686x/tw686x-audio.c index 91459ab715b2..96e444c49173 100644 --- a/drivers/media/pci/tw686x/tw686x-audio.c +++ b/drivers/media/pci/tw686x/tw686x-audio.c @@ -62,12 +62,22 @@ void tw686x_audio_irq(struct tw686x_dev *dev, unsigned long requests, } spin_unlock_irqrestore(&ac->lock, flags); + if (!done || !next) + continue; + /* + * Checking for a non-nil dma_desc[pb]->virt buffer is + * the same as checking for memcpy DMA mode. + */ desc = &ac->dma_descs[pb]; - if (done && next && desc->virt) { - memcpy(done->virt, desc->virt, desc->size); - ac->ptr = done->dma - ac->buf[0].dma; - snd_pcm_period_elapsed(ac->ss); + if (desc->virt) { + memcpy(done->virt, desc->virt, + dev->period_size); + } else { + u32 reg = pb ? ADMA_B_ADDR[ch] : ADMA_P_ADDR[ch]; + reg_write(dev, reg, next->dma); } + ac->ptr = done->dma - ac->buf[0].dma; + snd_pcm_period_elapsed(ac->ss); } } @@ -83,10 +93,9 @@ static int tw686x_pcm_hw_free(struct snd_pcm_substream *ss) } /* - * The audio device rate is global and shared among all - * capture channels. The driver makes no effort to prevent - * rate modifications. User is free change the rate, but it - * means changing the rate for all capture sub-devices. + * Audio parameters are global and shared among all + * capture channels. The driver prevents changes to + * the parameters if any audio channel is capturing. */ static const struct snd_pcm_hardware tw686x_capture_hw = { .info = (SNDRV_PCM_INFO_MMAP | @@ -99,9 +108,9 @@ static const struct snd_pcm_hardware tw686x_capture_hw = { .rate_max = 48000, .channels_min = 1, .channels_max = 1, - .buffer_bytes_max = TW686X_AUDIO_PAGE_MAX * TW686X_AUDIO_PAGE_SZ, - .period_bytes_min = TW686X_AUDIO_PAGE_SZ, - .period_bytes_max = TW686X_AUDIO_PAGE_SZ, + .buffer_bytes_max = TW686X_AUDIO_PAGE_MAX * AUDIO_DMA_SIZE_MAX, + .period_bytes_min = AUDIO_DMA_SIZE_MIN, + .period_bytes_max = AUDIO_DMA_SIZE_MAX, .periods_min = TW686X_AUDIO_PERIODS_MIN, .periods_max = TW686X_AUDIO_PERIODS_MAX, }; @@ -143,6 +152,14 @@ static int tw686x_pcm_prepare(struct snd_pcm_substream *ss) int i; spin_lock_irqsave(&dev->lock, flags); + /* + * Given the audio parameters are global (i.e. shared across + * DMA channels), we need to check new params are allowed. + */ + if (((dev->audio_rate != rt->rate) || + (dev->period_size != period_size)) && dev->audio_enabled) + goto err_audio_busy; + tw686x_disable_channel(dev, AUDIO_CHANNEL_OFFSET + ac->ch); spin_unlock_irqrestore(&dev->lock, flags); @@ -156,12 +173,21 @@ static int tw686x_pcm_prepare(struct snd_pcm_substream *ss) reg_write(dev, AUDIO_CONTROL2, reg); } - if (period_size != TW686X_AUDIO_PAGE_SZ || - rt->periods < TW686X_AUDIO_PERIODS_MIN || - rt->periods > TW686X_AUDIO_PERIODS_MAX) { - return -EINVAL; + if (dev->period_size != period_size) { + u32 reg; + + dev->period_size = period_size; + reg = reg_read(dev, AUDIO_CONTROL1); + reg &= ~(AUDIO_DMA_SIZE_MASK << AUDIO_DMA_SIZE_SHIFT); + reg |= period_size << AUDIO_DMA_SIZE_SHIFT; + + reg_write(dev, AUDIO_CONTROL1, reg); } + if (rt->periods < TW686X_AUDIO_PERIODS_MIN || + rt->periods > TW686X_AUDIO_PERIODS_MAX) + return -EINVAL; + spin_lock_irqsave(&ac->lock, flags); INIT_LIST_HEAD(&ac->buf_list); @@ -181,9 +207,19 @@ static int tw686x_pcm_prepare(struct snd_pcm_substream *ss) ac->curr_bufs[0] = p_buf; ac->curr_bufs[1] = b_buf; ac->ptr = 0; + + if (dev->dma_mode != TW686X_DMA_MODE_MEMCPY) { + reg_write(dev, ADMA_P_ADDR[ac->ch], p_buf->dma); + reg_write(dev, ADMA_B_ADDR[ac->ch], b_buf->dma); + } + spin_unlock_irqrestore(&ac->lock, flags); return 0; + +err_audio_busy: + spin_unlock_irqrestore(&dev->lock, flags); + return -EBUSY; } static int tw686x_pcm_trigger(struct snd_pcm_substream *ss, int cmd) @@ -197,6 +233,7 @@ static int tw686x_pcm_trigger(struct snd_pcm_substream *ss, int cmd) case SNDRV_PCM_TRIGGER_START: if (ac->curr_bufs[0] && ac->curr_bufs[1]) { spin_lock_irqsave(&dev->lock, flags); + dev->audio_enabled = 1; tw686x_enable_channel(dev, AUDIO_CHANNEL_OFFSET + ac->ch); spin_unlock_irqrestore(&dev->lock, flags); @@ -209,6 +246,7 @@ static int tw686x_pcm_trigger(struct snd_pcm_substream *ss, int cmd) break; case SNDRV_PCM_TRIGGER_STOP: spin_lock_irqsave(&dev->lock, flags); + dev->audio_enabled = 0; tw686x_disable_channel(dev, AUDIO_CHANNEL_OFFSET + ac->ch); spin_unlock_irqrestore(&dev->lock, flags); @@ -266,8 +304,8 @@ static int tw686x_snd_pcm_init(struct tw686x_dev *dev) return snd_pcm_lib_preallocate_pages_for_all(pcm, SNDRV_DMA_TYPE_DEV, snd_dma_pci_data(dev->pci_dev), - TW686X_AUDIO_PAGE_MAX * TW686X_AUDIO_PAGE_SZ, - TW686X_AUDIO_PAGE_MAX * TW686X_AUDIO_PAGE_SZ); + TW686X_AUDIO_PAGE_MAX * AUDIO_DMA_SIZE_MAX, + TW686X_AUDIO_PAGE_MAX * AUDIO_DMA_SIZE_MAX); } static void tw686x_audio_dma_free(struct tw686x_dev *dev, @@ -290,11 +328,19 @@ static int tw686x_audio_dma_alloc(struct tw686x_dev *dev, { int pb; + /* + * In the memcpy DMA mode we allocate a consistent buffer + * and use it for the DMA capture. Otherwise, DMA + * acts on the ALSA buffers as received in pcm_prepare. + */ + if (dev->dma_mode != TW686X_DMA_MODE_MEMCPY) + return 0; + for (pb = 0; pb < 2; pb++) { u32 reg = pb ? ADMA_B_ADDR[ac->ch] : ADMA_P_ADDR[ac->ch]; void *virt; - virt = pci_alloc_consistent(dev->pci_dev, TW686X_AUDIO_PAGE_SZ, + virt = pci_alloc_consistent(dev->pci_dev, AUDIO_DMA_SIZE_MAX, &ac->dma_descs[pb].phys); if (!virt) { dev_err(&dev->pci_dev->dev, @@ -303,7 +349,7 @@ static int tw686x_audio_dma_alloc(struct tw686x_dev *dev, return -ENOMEM; } ac->dma_descs[pb].virt = virt; - ac->dma_descs[pb].size = TW686X_AUDIO_PAGE_SZ; + ac->dma_descs[pb].size = AUDIO_DMA_SIZE_MAX; reg_write(dev, reg, ac->dma_descs[pb].phys); } return 0; @@ -334,12 +380,8 @@ int tw686x_audio_init(struct tw686x_dev *dev) struct snd_card *card; int err, ch; - /* - * AUDIO_CONTROL1 - * DMA byte length [31:19] = 4096 (i.e. ALSA period) - * External audio enable [0] = enabled - */ - reg_write(dev, AUDIO_CONTROL1, 0x80000001); + /* Enable external audio */ + reg_write(dev, AUDIO_CONTROL1, BIT(0)); err = snd_card_new(&pci_dev->dev, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1, diff --git a/drivers/media/pci/tw686x/tw686x-core.c b/drivers/media/pci/tw686x/tw686x-core.c index cf53b0e97be2..71a0453b1af1 100644 --- a/drivers/media/pci/tw686x/tw686x-core.c +++ b/drivers/media/pci/tw686x/tw686x-core.c @@ -21,12 +21,14 @@ * under stress testings it has been found that the machine can * freeze completely if DMA registers are programmed while streaming * is active. - * This driver tries to access hardware registers as infrequently - * as possible by: - * i. allocating fixed DMA buffers and memcpy'ing into - * vmalloc'ed buffers - * ii. using a timer to mitigate the rate of DMA reset operations, - * on DMA channels error. + * + * Therefore, driver implements a dma_mode called 'memcpy' which + * avoids cycling the DMA buffers, and insteads allocates extra DMA buffers + * and then copies into vmalloc'ed user buffers. + * + * In addition to this, when streaming is on, the driver tries to access + * hardware registers as infrequently as possible. This is done by using + * a timer to limit the rate at which DMA is reset on DMA channels error. */ #include @@ -55,6 +57,42 @@ static u32 dma_interval = 0x00098968; module_param(dma_interval, int, 0444); MODULE_PARM_DESC(dma_interval, "Minimum time span for DMA interrupting host"); +static unsigned int dma_mode = TW686X_DMA_MODE_MEMCPY; +static const char *dma_mode_name(unsigned int mode) +{ + switch (mode) { + case TW686X_DMA_MODE_MEMCPY: + return "memcpy"; + case TW686X_DMA_MODE_CONTIG: + return "contig"; + case TW686X_DMA_MODE_SG: + return "sg"; + default: + return "unknown"; + } +} + +static int tw686x_dma_mode_get(char *buffer, struct kernel_param *kp) +{ + return sprintf(buffer, dma_mode_name(dma_mode)); +} + +static int tw686x_dma_mode_set(const char *val, struct kernel_param *kp) +{ + if (!strcasecmp(val, dma_mode_name(TW686X_DMA_MODE_MEMCPY))) + dma_mode = TW686X_DMA_MODE_MEMCPY; + else if (!strcasecmp(val, dma_mode_name(TW686X_DMA_MODE_CONTIG))) + dma_mode = TW686X_DMA_MODE_CONTIG; + else if (!strcasecmp(val, dma_mode_name(TW686X_DMA_MODE_SG))) + dma_mode = TW686X_DMA_MODE_SG; + else + return -EINVAL; + return 0; +} +module_param_call(dma_mode, tw686x_dma_mode_set, tw686x_dma_mode_get, + &dma_mode, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(dma_mode, "DMA operation mode (memcpy/contig/sg, default=memcpy)"); + void tw686x_disable_channel(struct tw686x_dev *dev, unsigned int channel) { u32 dma_en = reg_read(dev, DMA_CHANNEL_ENABLE); @@ -212,6 +250,7 @@ static int tw686x_probe(struct pci_dev *pci_dev, if (!dev) return -ENOMEM; dev->type = pci_id->driver_data; + dev->dma_mode = dma_mode; sprintf(dev->name, "tw%04X", pci_dev->device); dev->video_channels = kcalloc(max_channels(dev), @@ -228,9 +267,10 @@ static int tw686x_probe(struct pci_dev *pci_dev, goto free_video; } - pr_info("%s: PCI %s, IRQ %d, MMIO 0x%lx\n", dev->name, + pr_info("%s: PCI %s, IRQ %d, MMIO 0x%lx (%s mode)\n", dev->name, pci_name(pci_dev), pci_dev->irq, - (unsigned long)pci_resource_start(pci_dev, 0)); + (unsigned long)pci_resource_start(pci_dev, 0), + dma_mode_name(dma_mode)); dev->pci_dev = pci_dev; if (pci_enable_device(pci_dev)) { diff --git a/drivers/media/pci/tw686x/tw686x-regs.h b/drivers/media/pci/tw686x/tw686x-regs.h index fcef586a4c8c..15a956642ef4 100644 --- a/drivers/media/pci/tw686x/tw686x-regs.h +++ b/drivers/media/pci/tw686x/tw686x-regs.h @@ -105,6 +105,10 @@ 0x2d0, 0x2d1, 0x2d2, 0x2d3 }) #define SYS_MODE_DMA_SHIFT 13 +#define AUDIO_DMA_SIZE_SHIFT 19 +#define AUDIO_DMA_SIZE_MIN SZ_512 +#define AUDIO_DMA_SIZE_MAX SZ_4K +#define AUDIO_DMA_SIZE_MASK (SZ_8K - 1) #define DMA_CMD_ENABLE BIT(31) #define INT_STATUS_DMA_TOUT BIT(17) @@ -119,4 +123,9 @@ #define TW686X_STD_PAL_CN 5 #define TW686X_STD_PAL_60 6 +#define TW686X_FIELD_MODE 0x3 +#define TW686X_FRAME_MODE 0x2 +/* 0x1 is reserved */ +#define TW686X_SG_MODE 0x0 + #define TW686X_FIFO_ERROR(x) (x & ~(0xff)) diff --git a/drivers/media/pci/tw686x/tw686x-video.c b/drivers/media/pci/tw686x/tw686x-video.c index 253e10823ba3..cdb16de770fe 100644 --- a/drivers/media/pci/tw686x/tw686x-video.c +++ b/drivers/media/pci/tw686x/tw686x-video.c @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include "tw686x.h" #include "tw686x-regs.h" @@ -26,6 +28,11 @@ #define TW686X_INPUTS_PER_CH 4 #define TW686X_VIDEO_WIDTH 720 #define TW686X_VIDEO_HEIGHT(id) ((id & V4L2_STD_525_60) ? 480 : 576) +#define TW686X_MAX_FPS(id) ((id & V4L2_STD_525_60) ? 30 : 25) + +#define TW686X_MAX_SG_ENTRY_SIZE 4096 +#define TW686X_MAX_SG_DESC_COUNT 256 /* PAL 720x576 needs 203 4-KB pages */ +#define TW686X_SG_TABLE_SIZE (TW686X_MAX_SG_DESC_COUNT * sizeof(struct tw686x_sg_desc)) static const struct tw686x_format formats[] = { { @@ -43,53 +50,367 @@ static const struct tw686x_format formats[] = { } }; -static unsigned int tw686x_fields_map(v4l2_std_id std, unsigned int fps) +static void tw686x_buf_done(struct tw686x_video_channel *vc, + unsigned int pb) { - static const unsigned int map[15] = { - 0x00000000, 0x00000001, 0x00004001, 0x00104001, 0x00404041, - 0x01041041, 0x01104411, 0x01111111, 0x04444445, 0x04511445, - 0x05145145, 0x05151515, 0x05515455, 0x05551555, 0x05555555 - }; + struct tw686x_dma_desc *desc = &vc->dma_descs[pb]; + struct tw686x_dev *dev = vc->dev; + struct vb2_v4l2_buffer *vb; + struct vb2_buffer *vb2_buf; - static const unsigned int std_625_50[26] = { - 0, 1, 1, 2, 3, 3, 4, 4, 5, 5, 6, 7, 7, - 8, 8, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 0 - }; + if (vc->curr_bufs[pb]) { + vb = &vc->curr_bufs[pb]->vb; - static const unsigned int std_525_60[31] = { - 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, - 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 0, 0 - }; + vb->field = dev->dma_ops->field; + vb->sequence = vc->sequence++; + vb2_buf = &vb->vb2_buf; - unsigned int i; + if (dev->dma_mode == TW686X_DMA_MODE_MEMCPY) + memcpy(vb2_plane_vaddr(vb2_buf, 0), desc->virt, + desc->size); + vb2_buf->timestamp = ktime_get_ns(); + vb2_buffer_done(vb2_buf, VB2_BUF_STATE_DONE); + } + + vc->pb = !pb; +} + +/* + * We can call this even when alloc_dma failed for the given channel + */ +static void tw686x_memcpy_dma_free(struct tw686x_video_channel *vc, + unsigned int pb) +{ + struct tw686x_dma_desc *desc = &vc->dma_descs[pb]; + struct tw686x_dev *dev = vc->dev; + struct pci_dev *pci_dev; + unsigned long flags; + + /* Check device presence. Shouldn't really happen! */ + spin_lock_irqsave(&dev->lock, flags); + pci_dev = dev->pci_dev; + spin_unlock_irqrestore(&dev->lock, flags); + if (!pci_dev) { + WARN(1, "trying to deallocate on missing device\n"); + return; + } + + if (desc->virt) { + pci_free_consistent(dev->pci_dev, desc->size, + desc->virt, desc->phys); + desc->virt = NULL; + } +} + +static int tw686x_memcpy_dma_alloc(struct tw686x_video_channel *vc, + unsigned int pb) +{ + struct tw686x_dev *dev = vc->dev; + u32 reg = pb ? VDMA_B_ADDR[vc->ch] : VDMA_P_ADDR[vc->ch]; + unsigned int len; + void *virt; + + WARN(vc->dma_descs[pb].virt, + "Allocating buffer but previous still here\n"); + + len = (vc->width * vc->height * vc->format->depth) >> 3; + virt = pci_alloc_consistent(dev->pci_dev, len, + &vc->dma_descs[pb].phys); + if (!virt) { + v4l2_err(&dev->v4l2_dev, + "dma%d: unable to allocate %s-buffer\n", + vc->ch, pb ? "B" : "P"); + return -ENOMEM; + } + vc->dma_descs[pb].size = len; + vc->dma_descs[pb].virt = virt; + reg_write(dev, reg, vc->dma_descs[pb].phys); + + return 0; +} + +static void tw686x_memcpy_buf_refill(struct tw686x_video_channel *vc, + unsigned int pb) +{ + struct tw686x_v4l2_buf *buf; + + while (!list_empty(&vc->vidq_queued)) { + + buf = list_first_entry(&vc->vidq_queued, + struct tw686x_v4l2_buf, list); + list_del(&buf->list); + + vc->curr_bufs[pb] = buf; + return; + } + vc->curr_bufs[pb] = NULL; +} - if (std & V4L2_STD_525_60) { - if (fps >= ARRAY_SIZE(std_525_60)) - fps = 30; - i = std_525_60[fps]; +static const struct tw686x_dma_ops memcpy_dma_ops = { + .alloc = tw686x_memcpy_dma_alloc, + .free = tw686x_memcpy_dma_free, + .buf_refill = tw686x_memcpy_buf_refill, + .mem_ops = &vb2_vmalloc_memops, + .hw_dma_mode = TW686X_FRAME_MODE, + .field = V4L2_FIELD_INTERLACED, +}; + +static void tw686x_contig_buf_refill(struct tw686x_video_channel *vc, + unsigned int pb) +{ + struct tw686x_v4l2_buf *buf; + + while (!list_empty(&vc->vidq_queued)) { + u32 reg = pb ? VDMA_B_ADDR[vc->ch] : VDMA_P_ADDR[vc->ch]; + dma_addr_t phys; + + buf = list_first_entry(&vc->vidq_queued, + struct tw686x_v4l2_buf, list); + list_del(&buf->list); + + phys = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0); + reg_write(vc->dev, reg, phys); + + buf->vb.vb2_buf.state = VB2_BUF_STATE_ACTIVE; + vc->curr_bufs[pb] = buf; + return; + } + vc->curr_bufs[pb] = NULL; +} + +static const struct tw686x_dma_ops contig_dma_ops = { + .buf_refill = tw686x_contig_buf_refill, + .mem_ops = &vb2_dma_contig_memops, + .hw_dma_mode = TW686X_FRAME_MODE, + .field = V4L2_FIELD_INTERLACED, +}; + +static int tw686x_sg_desc_fill(struct tw686x_sg_desc *descs, + struct tw686x_v4l2_buf *buf, + unsigned int buf_len) +{ + struct sg_table *vbuf = vb2_dma_sg_plane_desc(&buf->vb.vb2_buf, 0); + unsigned int len, entry_len; + struct scatterlist *sg; + int i, count; + + /* Clear the scatter-gather table */ + memset(descs, 0, TW686X_SG_TABLE_SIZE); + + count = 0; + for_each_sg(vbuf->sgl, sg, vbuf->nents, i) { + dma_addr_t phys = sg_dma_address(sg); + len = sg_dma_len(sg); + + while (len && buf_len) { + + if (count == TW686X_MAX_SG_DESC_COUNT) + return -ENOMEM; + + entry_len = min_t(unsigned int, len, + TW686X_MAX_SG_ENTRY_SIZE); + entry_len = min_t(unsigned int, entry_len, buf_len); + descs[count].phys = cpu_to_le32(phys); + descs[count++].flags_length = + cpu_to_le32(BIT(30) | entry_len); + phys += entry_len; + len -= entry_len; + buf_len -= entry_len; + } + + if (!buf_len) + return 0; + } + + return -ENOMEM; +} + +static void tw686x_sg_buf_refill(struct tw686x_video_channel *vc, + unsigned int pb) +{ + struct tw686x_dev *dev = vc->dev; + struct tw686x_v4l2_buf *buf; + + while (!list_empty(&vc->vidq_queued)) { + unsigned int buf_len; + + buf = list_first_entry(&vc->vidq_queued, + struct tw686x_v4l2_buf, list); + list_del(&buf->list); + + buf_len = (vc->width * vc->height * vc->format->depth) >> 3; + if (tw686x_sg_desc_fill(vc->sg_descs[pb], buf, buf_len)) { + v4l2_err(&dev->v4l2_dev, + "dma%d: unable to fill %s-buffer\n", + vc->ch, pb ? "B" : "P"); + vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR); + continue; + } + + buf->vb.vb2_buf.state = VB2_BUF_STATE_ACTIVE; + vc->curr_bufs[pb] = buf; + return; + } + + vc->curr_bufs[pb] = NULL; +} + +static void tw686x_sg_dma_free(struct tw686x_video_channel *vc, + unsigned int pb) +{ + struct tw686x_dma_desc *desc = &vc->dma_descs[pb]; + struct tw686x_dev *dev = vc->dev; + + if (desc->size) { + pci_free_consistent(dev->pci_dev, desc->size, + desc->virt, desc->phys); + desc->virt = NULL; + } + + vc->sg_descs[pb] = NULL; +} + +static int tw686x_sg_dma_alloc(struct tw686x_video_channel *vc, + unsigned int pb) +{ + struct tw686x_dma_desc *desc = &vc->dma_descs[pb]; + struct tw686x_dev *dev = vc->dev; + u32 reg = pb ? DMA_PAGE_TABLE1_ADDR[vc->ch] : + DMA_PAGE_TABLE0_ADDR[vc->ch]; + void *virt; + + if (desc->size) { + + virt = pci_alloc_consistent(dev->pci_dev, desc->size, + &desc->phys); + if (!virt) { + v4l2_err(&dev->v4l2_dev, + "dma%d: unable to allocate %s-buffer\n", + vc->ch, pb ? "B" : "P"); + return -ENOMEM; + } + desc->virt = virt; + reg_write(dev, reg, desc->phys); } else { - if (fps >= ARRAY_SIZE(std_625_50)) - fps = 25; - i = std_625_50[fps]; + virt = dev->video_channels[0].dma_descs[pb].virt + + vc->ch * TW686X_SG_TABLE_SIZE; } - return map[i]; + vc->sg_descs[pb] = virt; + return 0; +} + +static int tw686x_sg_setup(struct tw686x_dev *dev) +{ + unsigned int sg_table_size, pb, ch, channels; + + if (is_second_gen(dev)) { + /* + * TW6865/TW6869: each channel needs a pair of + * P-B descriptor tables. + */ + channels = max_channels(dev); + sg_table_size = TW686X_SG_TABLE_SIZE; + } else { + /* + * TW6864/TW6868: we need to allocate a pair of + * P-B descriptor tables, common for all channels. + * Each table will be bigger than 4 KB. + */ + channels = 1; + sg_table_size = max_channels(dev) * TW686X_SG_TABLE_SIZE; + } + + for (ch = 0; ch < channels; ch++) { + struct tw686x_video_channel *vc = &dev->video_channels[ch]; + + for (pb = 0; pb < 2; pb++) + vc->dma_descs[pb].size = sg_table_size; + } + + return 0; +} + +static const struct tw686x_dma_ops sg_dma_ops = { + .setup = tw686x_sg_setup, + .alloc = tw686x_sg_dma_alloc, + .free = tw686x_sg_dma_free, + .buf_refill = tw686x_sg_buf_refill, + .mem_ops = &vb2_dma_sg_memops, + .hw_dma_mode = TW686X_SG_MODE, + .field = V4L2_FIELD_SEQ_TB, +}; + +static const unsigned int fps_map[15] = { + /* + * bit 31 enables selecting the field control register + * bits 0-29 are a bitmask with fields that will be output. + * For NTSC (and PAL-M, PAL-60), all 30 bits are used. + * For other PAL standards, only the first 25 bits are used. + */ + 0x00000000, /* output all fields */ + 0x80000006, /* 2 fps (60Hz), 2 fps (50Hz) */ + 0x80018006, /* 4 fps (60Hz), 4 fps (50Hz) */ + 0x80618006, /* 6 fps (60Hz), 6 fps (50Hz) */ + 0x81818186, /* 8 fps (60Hz), 8 fps (50Hz) */ + 0x86186186, /* 10 fps (60Hz), 8 fps (50Hz) */ + 0x86619866, /* 12 fps (60Hz), 10 fps (50Hz) */ + 0x86666666, /* 14 fps (60Hz), 12 fps (50Hz) */ + 0x9999999e, /* 16 fps (60Hz), 14 fps (50Hz) */ + 0x99e6799e, /* 18 fps (60Hz), 16 fps (50Hz) */ + 0x9e79e79e, /* 20 fps (60Hz), 16 fps (50Hz) */ + 0x9e7e7e7e, /* 22 fps (60Hz), 18 fps (50Hz) */ + 0x9fe7f9fe, /* 24 fps (60Hz), 20 fps (50Hz) */ + 0x9ffe7ffe, /* 26 fps (60Hz), 22 fps (50Hz) */ + 0x9ffffffe, /* 28 fps (60Hz), 24 fps (50Hz) */ +}; + +static unsigned int tw686x_real_fps(unsigned int index, unsigned int max_fps) +{ + unsigned long mask; + + if (!index || index >= ARRAY_SIZE(fps_map)) + return max_fps; + + mask = GENMASK(max_fps - 1, 0); + return hweight_long(fps_map[index] & mask); +} + +static unsigned int tw686x_fps_idx(unsigned int fps, unsigned int max_fps) +{ + unsigned int idx, real_fps; + int delta; + + /* First guess */ + idx = (12 + 15 * fps) / max_fps; + + /* Minimal possible framerate is 2 frames per second */ + if (!idx) + return 1; + + /* Check if the difference is bigger than abs(1) and adjust */ + real_fps = tw686x_real_fps(idx, max_fps); + delta = real_fps - fps; + if (delta < -1) + idx++; + else if (delta > 1) + idx--; + + /* Max framerate */ + if (idx >= 15) + return 0; + + return idx; } static void tw686x_set_framerate(struct tw686x_video_channel *vc, unsigned int fps) { - unsigned int map; - - if (vc->fps == fps) - return; + unsigned int i; - map = tw686x_fields_map(vc->video_standard, fps) << 1; - map |= map << 1; - if (map > 0) - map |= BIT(31); - reg_write(vc->dev, VIDEO_FIELD_CTRL[vc->ch], map); - vc->fps = fps; + i = tw686x_fps_idx(fps, TW686X_MAX_FPS(vc->video_standard)); + reg_write(vc->dev, VIDEO_FIELD_CTRL[vc->ch], fps_map[i]); + vc->fps = tw686x_real_fps(i, TW686X_MAX_FPS(vc->video_standard)); } static const struct tw686x_format *format_by_fourcc(unsigned int fourcc) @@ -104,7 +425,7 @@ static const struct tw686x_format *format_by_fourcc(unsigned int fourcc) static int tw686x_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct tw686x_video_channel *vc = vb2_get_drv_priv(vq); unsigned int szimage = @@ -152,75 +473,6 @@ static void tw686x_buf_queue(struct vb2_buffer *vb) spin_unlock_irqrestore(&vc->qlock, flags); } -/* - * We can call this even when alloc_dma failed for the given channel - */ -static void tw686x_free_dma(struct tw686x_video_channel *vc, unsigned int pb) -{ - struct tw686x_dma_desc *desc = &vc->dma_descs[pb]; - struct tw686x_dev *dev = vc->dev; - struct pci_dev *pci_dev; - unsigned long flags; - - /* Check device presence. Shouldn't really happen! */ - spin_lock_irqsave(&dev->lock, flags); - pci_dev = dev->pci_dev; - spin_unlock_irqrestore(&dev->lock, flags); - if (!pci_dev) { - WARN(1, "trying to deallocate on missing device\n"); - return; - } - - if (desc->virt) { - pci_free_consistent(dev->pci_dev, desc->size, - desc->virt, desc->phys); - desc->virt = NULL; - } -} - -static int tw686x_alloc_dma(struct tw686x_video_channel *vc, unsigned int pb) -{ - struct tw686x_dev *dev = vc->dev; - u32 reg = pb ? VDMA_B_ADDR[vc->ch] : VDMA_P_ADDR[vc->ch]; - unsigned int len; - void *virt; - - WARN(vc->dma_descs[pb].virt, - "Allocating buffer but previous still here\n"); - - len = (vc->width * vc->height * vc->format->depth) >> 3; - virt = pci_alloc_consistent(dev->pci_dev, len, - &vc->dma_descs[pb].phys); - if (!virt) { - v4l2_err(&dev->v4l2_dev, - "dma%d: unable to allocate %s-buffer\n", - vc->ch, pb ? "B" : "P"); - return -ENOMEM; - } - vc->dma_descs[pb].size = len; - vc->dma_descs[pb].virt = virt; - reg_write(dev, reg, vc->dma_descs[pb].phys); - - return 0; -} - -static void tw686x_buffer_refill(struct tw686x_video_channel *vc, - unsigned int pb) -{ - struct tw686x_v4l2_buf *buf; - - while (!list_empty(&vc->vidq_queued)) { - - buf = list_first_entry(&vc->vidq_queued, - struct tw686x_v4l2_buf, list); - list_del(&buf->list); - - vc->curr_bufs[pb] = buf; - return; - } - vc->curr_bufs[pb] = NULL; -} - static void tw686x_clear_queue(struct tw686x_video_channel *vc, enum vb2_buffer_state state) { @@ -262,7 +514,8 @@ static int tw686x_start_streaming(struct vb2_queue *vq, unsigned int count) spin_lock_irqsave(&vc->qlock, flags); /* Sanity check */ - if (!vc->dma_descs[0].virt || !vc->dma_descs[1].virt) { + if (dev->dma_mode == TW686X_DMA_MODE_MEMCPY && + (!vc->dma_descs[0].virt || !vc->dma_descs[1].virt)) { spin_unlock_irqrestore(&vc->qlock, flags); v4l2_err(&dev->v4l2_dev, "video%d: refusing to start without DMA buffers\n", @@ -272,7 +525,7 @@ static int tw686x_start_streaming(struct vb2_queue *vq, unsigned int count) } for (pb = 0; pb < 2; pb++) - tw686x_buffer_refill(vc, pb); + dev->dma_ops->buf_refill(vc, pb); spin_unlock_irqrestore(&vc->qlock, flags); vc->sequence = 0; @@ -375,10 +628,11 @@ static int tw686x_g_fmt_vid_cap(struct file *file, void *priv, struct v4l2_format *f) { struct tw686x_video_channel *vc = video_drvdata(file); + struct tw686x_dev *dev = vc->dev; f->fmt.pix.width = vc->width; f->fmt.pix.height = vc->height; - f->fmt.pix.field = V4L2_FIELD_INTERLACED; + f->fmt.pix.field = dev->dma_ops->field; f->fmt.pix.pixelformat = vc->format->fourcc; f->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M; f->fmt.pix.bytesperline = (f->fmt.pix.width * vc->format->depth) / 8; @@ -390,6 +644,7 @@ static int tw686x_try_fmt_vid_cap(struct file *file, void *priv, struct v4l2_format *f) { struct tw686x_video_channel *vc = video_drvdata(file); + struct tw686x_dev *dev = vc->dev; unsigned int video_height = TW686X_VIDEO_HEIGHT(vc->video_standard); const struct tw686x_format *format; @@ -412,7 +667,7 @@ static int tw686x_try_fmt_vid_cap(struct file *file, void *priv, f->fmt.pix.bytesperline = (f->fmt.pix.width * format->depth) / 8; f->fmt.pix.sizeimage = f->fmt.pix.height * f->fmt.pix.bytesperline; f->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M; - f->fmt.pix.field = V4L2_FIELD_INTERLACED; + f->fmt.pix.field = dev->dma_ops->field; return 0; } @@ -421,6 +676,7 @@ static int tw686x_s_fmt_vid_cap(struct file *file, void *priv, struct v4l2_format *f) { struct tw686x_video_channel *vc = video_drvdata(file); + struct tw686x_dev *dev = vc->dev; u32 val, width, line_width, height; unsigned long bitsperframe; int err, pb; @@ -438,15 +694,16 @@ static int tw686x_s_fmt_vid_cap(struct file *file, void *priv, vc->height = f->fmt.pix.height; /* We need new DMA buffers if the framesize has changed */ - if (bitsperframe != vc->width * vc->height * vc->format->depth) { + if (dev->dma_ops->alloc && + bitsperframe != vc->width * vc->height * vc->format->depth) { for (pb = 0; pb < 2; pb++) - tw686x_free_dma(vc, pb); + dev->dma_ops->free(vc, pb); for (pb = 0; pb < 2; pb++) { - err = tw686x_alloc_dma(vc, pb); + err = dev->dma_ops->alloc(vc, pb); if (err) { if (pb > 0) - tw686x_free_dma(vc, 0); + dev->dma_ops->free(vc, 0); return err; } } @@ -464,6 +721,19 @@ static int tw686x_s_fmt_vid_cap(struct file *file, void *priv, else val &= ~BIT(24); + val &= ~0x7ffff; + + /* Program the DMA scatter-gather */ + if (dev->dma_mode == TW686X_DMA_MODE_SG) { + u32 start_idx, end_idx; + + start_idx = is_second_gen(dev) ? + 0 : vc->ch * TW686X_MAX_SG_DESC_COUNT; + end_idx = start_idx + TW686X_MAX_SG_DESC_COUNT - 1; + + val |= (end_idx << 10) | start_idx; + } + val &= ~(0x7 << 20); val |= vc->format->mode << 20; reg_write(vc->dev, VDMA_CHANNEL_CONFIG[vc->ch], val); @@ -540,6 +810,12 @@ static int tw686x_s_std(struct file *file, void *priv, v4l2_std_id id) ret = tw686x_g_fmt_vid_cap(file, priv, &f); if (!ret) tw686x_s_fmt_vid_cap(file, priv, &f); + + /* + * Frame decimation depends on the chosen standard, + * so reset it to the current value. + */ + tw686x_set_framerate(vc, vc->fps); return 0; } @@ -609,6 +885,40 @@ static int tw686x_g_std(struct file *file, void *priv, v4l2_std_id *id) return 0; } +static int tw686x_g_parm(struct file *file, void *priv, + struct v4l2_streamparm *sp) +{ + struct tw686x_video_channel *vc = video_drvdata(file); + struct v4l2_captureparm *cp = &sp->parm.capture; + + if (sp->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) + return -EINVAL; + sp->parm.capture.readbuffers = 3; + + cp->capability = V4L2_CAP_TIMEPERFRAME; + cp->timeperframe.numerator = 1; + cp->timeperframe.denominator = vc->fps; + return 0; +} + +static int tw686x_s_parm(struct file *file, void *priv, + struct v4l2_streamparm *sp) +{ + struct tw686x_video_channel *vc = video_drvdata(file); + struct v4l2_captureparm *cp = &sp->parm.capture; + unsigned int denominator = cp->timeperframe.denominator; + unsigned int numerator = cp->timeperframe.numerator; + unsigned int fps; + + if (vb2_is_busy(&vc->vidq)) + return -EBUSY; + + fps = (!numerator || !denominator) ? 0 : denominator / numerator; + if (vc->fps != fps) + tw686x_set_framerate(vc, fps); + return tw686x_g_parm(file, priv, sp); +} + static int tw686x_enum_fmt_vid_cap(struct file *file, void *priv, struct v4l2_fmtdesc *f) { @@ -695,6 +1005,9 @@ static const struct v4l2_ioctl_ops tw686x_video_ioctl_ops = { .vidioc_g_std = tw686x_g_std, .vidioc_s_std = tw686x_s_std, + .vidioc_g_parm = tw686x_g_parm, + .vidioc_s_parm = tw686x_s_parm, + .vidioc_enum_input = tw686x_enum_input, .vidioc_g_input = tw686x_g_input, .vidioc_s_input = tw686x_s_input, @@ -713,26 +1026,11 @@ static const struct v4l2_ioctl_ops tw686x_video_ioctl_ops = { .vidioc_unsubscribe_event = v4l2_event_unsubscribe, }; -static void tw686x_buffer_copy(struct tw686x_video_channel *vc, - unsigned int pb, struct vb2_v4l2_buffer *vb) -{ - struct tw686x_dma_desc *desc = &vc->dma_descs[pb]; - struct vb2_buffer *vb2_buf = &vb->vb2_buf; - - vb->field = V4L2_FIELD_INTERLACED; - vb->sequence = vc->sequence++; - - memcpy(vb2_plane_vaddr(vb2_buf, 0), desc->virt, desc->size); - vb2_buf->timestamp = ktime_get_ns(); - vb2_buffer_done(vb2_buf, VB2_BUF_STATE_DONE); -} - void tw686x_video_irq(struct tw686x_dev *dev, unsigned long requests, unsigned int pb_status, unsigned int fifo_status, unsigned int *reset_ch) { struct tw686x_video_channel *vc; - struct vb2_v4l2_buffer *vb; unsigned long flags; unsigned int ch, pb; @@ -781,14 +1079,9 @@ void tw686x_video_irq(struct tw686x_dev *dev, unsigned long requests, continue; } - /* handle video stream */ spin_lock_irqsave(&vc->qlock, flags); - if (vc->curr_bufs[pb]) { - vb = &vc->curr_bufs[pb]->vb; - tw686x_buffer_copy(vc, pb, vb); - } - vc->pb = !pb; - tw686x_buffer_refill(vc, pb); + tw686x_buf_done(vc, pb); + dev->dma_ops->buf_refill(vc, pb); spin_unlock_irqrestore(&vc->qlock, flags); } } @@ -803,8 +1096,9 @@ void tw686x_video_free(struct tw686x_dev *dev) if (vc->device) video_unregister_device(vc->device); - for (pb = 0; pb < 2; pb++) - tw686x_free_dma(vc, pb); + if (dev->dma_ops->free) + for (pb = 0; pb < 2; pb++) + dev->dma_ops->free(vc, pb); } } @@ -813,10 +1107,25 @@ int tw686x_video_init(struct tw686x_dev *dev) unsigned int ch, val, pb; int err; + if (dev->dma_mode == TW686X_DMA_MODE_MEMCPY) + dev->dma_ops = &memcpy_dma_ops; + else if (dev->dma_mode == TW686X_DMA_MODE_CONTIG) + dev->dma_ops = &contig_dma_ops; + else if (dev->dma_mode == TW686X_DMA_MODE_SG) + dev->dma_ops = &sg_dma_ops; + else + return -EINVAL; + err = v4l2_device_register(&dev->pci_dev->dev, &dev->v4l2_dev); if (err) return err; + if (dev->dma_ops->setup) { + err = dev->dma_ops->setup(dev); + if (err) + return err; + } + for (ch = 0; ch < max_channels(dev); ch++) { struct tw686x_video_channel *vc = &dev->video_channels[ch]; struct video_device *vdev; @@ -842,10 +1151,12 @@ int tw686x_video_init(struct tw686x_dev *dev) reg_write(dev, HACTIVE_LO[ch], 0xd0); reg_write(dev, VIDEO_SIZE[ch], 0); - for (pb = 0; pb < 2; pb++) { - err = tw686x_alloc_dma(vc, pb); - if (err) - goto error; + if (dev->dma_ops->alloc) { + for (pb = 0; pb < 2; pb++) { + err = dev->dma_ops->alloc(vc, pb); + if (err) + goto error; + } } vc->vidq.io_modes = VB2_READ | VB2_MMAP | VB2_DMABUF; @@ -853,11 +1164,12 @@ int tw686x_video_init(struct tw686x_dev *dev) vc->vidq.drv_priv = vc; vc->vidq.buf_struct_size = sizeof(struct tw686x_v4l2_buf); vc->vidq.ops = &tw686x_video_qops; - vc->vidq.mem_ops = &vb2_vmalloc_memops; + vc->vidq.mem_ops = dev->dma_ops->mem_ops; vc->vidq.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; vc->vidq.min_buffers_needed = 2; vc->vidq.lock = &vc->vb_mutex; vc->vidq.gfp_flags = GFP_DMA32; + vc->vidq.dev = &dev->pci_dev->dev; err = vb2_queue_init(&vc->vidq); if (err) { @@ -915,10 +1227,9 @@ int tw686x_video_init(struct tw686x_dev *dev) vc->num = vdev->num; } - /* Set DMA frame mode on all channels. Only supported mode for now. */ val = TW686X_DEF_PHASE_REF; for (ch = 0; ch < max_channels(dev); ch++) - val |= TW686X_FRAME_MODE << (16 + ch * 2); + val |= dev->dma_ops->hw_dma_mode << (16 + ch * 2); reg_write(dev, PHASE_REF, val); reg_write(dev, MISC2[0], 0xe7); diff --git a/drivers/media/pci/tw686x/tw686x.h b/drivers/media/pci/tw686x/tw686x.h index 44b5755acf02..f24a2a9bcdb2 100644 --- a/drivers/media/pci/tw686x/tw686x.h +++ b/drivers/media/pci/tw686x/tw686x.h @@ -27,16 +27,14 @@ #define TYPE_SECOND_GEN 0x10 #define TW686X_DEF_PHASE_REF 0x1518 -#define TW686X_FIELD_MODE 0x3 -#define TW686X_FRAME_MODE 0x2 -/* 0x1 is reserved */ -#define TW686X_SG_MODE 0x0 - -#define TW686X_AUDIO_PAGE_SZ 4096 #define TW686X_AUDIO_PAGE_MAX 16 #define TW686X_AUDIO_PERIODS_MIN 2 #define TW686X_AUDIO_PERIODS_MAX TW686X_AUDIO_PAGE_MAX +#define TW686X_DMA_MODE_MEMCPY 0 +#define TW686X_DMA_MODE_CONTIG 1 +#define TW686X_DMA_MODE_SG 2 + struct tw686x_format { char *name; unsigned int fourcc; @@ -50,6 +48,12 @@ struct tw686x_dma_desc { unsigned int size; }; +struct tw686x_sg_desc { + /* 3 MSBits for flags, 13 LSBits for length */ + __le32 flags_length; + __le32 phys; +}; + struct tw686x_audio_buf { dma_addr_t dma; void *virt; @@ -82,6 +86,7 @@ struct tw686x_video_channel { struct video_device *device; struct tw686x_v4l2_buf *curr_bufs[2]; struct tw686x_dma_desc dma_descs[2]; + struct tw686x_sg_desc *sg_descs[2]; struct v4l2_ctrl_handler ctrl_handler; const struct tw686x_format *format; @@ -99,6 +104,16 @@ struct tw686x_video_channel { bool no_signal; }; +struct tw686x_dma_ops { + int (*setup)(struct tw686x_dev *dev); + int (*alloc)(struct tw686x_video_channel *vc, unsigned int pb); + void (*free)(struct tw686x_video_channel *vc, unsigned int pb); + void (*buf_refill)(struct tw686x_video_channel *vc, unsigned int pb); + const struct vb2_mem_ops *mem_ops; + enum v4l2_field field; + u32 hw_dma_mode; +}; + /** * struct tw686x_dev - global device status * @lock: spinlock controlling access to the @@ -112,15 +127,18 @@ struct tw686x_dev { char name[32]; unsigned int type; + unsigned int dma_mode; struct pci_dev *pci_dev; __u32 __iomem *mmio; - void *alloc_ctx; - + const struct tw686x_dma_ops *dma_ops; struct tw686x_video_channel *video_channels; struct tw686x_audio_channel *audio_channels; - int audio_rate; /* per-device value */ + /* Per-device audio parameters */ + int audio_rate; + int period_size; + int audio_enabled; struct timer_list dma_delay_timer; u32 pending_dma_en; /* must be protected by lock */ @@ -143,6 +161,12 @@ static inline unsigned int max_channels(struct tw686x_dev *dev) return dev->type & TYPE_MAX_CHANNELS; /* 4 or 8 channels */ } +static inline unsigned is_second_gen(struct tw686x_dev *dev) +{ + /* each channel has its own DMA SG table */ + return dev->type & TYPE_SECOND_GEN; +} + void tw686x_enable_channel(struct tw686x_dev *dev, unsigned int channel); void tw686x_disable_channel(struct tw686x_dev *dev, unsigned int channel); diff --git a/drivers/media/pci/zoran/zr36016.c b/drivers/media/pci/zoran/zr36016.c index b87ddba8608f..c12ca9f96bac 100644 --- a/drivers/media/pci/zoran/zr36016.c +++ b/drivers/media/pci/zoran/zr36016.c @@ -246,10 +246,6 @@ static int zr36016_pushit (struct zr36016 *ptr, //TODO// ========================================================================= */ -// needed offset values PAL NTSC SECAM -static const int zr016_xoff[] = { 20, 20, 20 }; -static const int zr016_yoff[] = { 8, 9, 7 }; - static void zr36016_init (struct zr36016 *ptr) { diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index 84e041c0a70e..f25344bc7912 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -110,6 +110,7 @@ source "drivers/media/platform/exynos4-is/Kconfig" source "drivers/media/platform/s5p-tv/Kconfig" source "drivers/media/platform/am437x/Kconfig" source "drivers/media/platform/xilinx/Kconfig" +source "drivers/media/platform/rcar-vin/Kconfig" config VIDEO_TI_CAL tristate "TI CAL (Camera Adaptation Layer) driver" @@ -152,6 +153,36 @@ config VIDEO_CODA Coda is a range of video codec IPs that supports H.264, MPEG-4, and other video formats. +config VIDEO_MEDIATEK_VPU + tristate "Mediatek Video Processor Unit" + depends on VIDEO_DEV && VIDEO_V4L2 + depends on ARCH_MEDIATEK || COMPILE_TEST + ---help--- + This driver provides downloading VPU firmware and + communicating with VPU. This driver for hw video + codec embedded in Mediatek's MT8173 SOCs. It is able + to handle video decoding/encoding in a range of formats. + + To compile this driver as a module, choose M here: the + module will be called mtk-vpu. + +config VIDEO_MEDIATEK_VCODEC + tristate "Mediatek Video Codec driver" + depends on MTK_IOMMU || COMPILE_TEST + depends on VIDEO_DEV && VIDEO_V4L2 + depends on ARCH_MEDIATEK || COMPILE_TEST + select VIDEOBUF2_DMA_CONTIG + select V4L2_MEM2MEM_DEV + select VIDEO_MEDIATEK_VPU + default n + ---help--- + Mediatek video codec driver provides HW capability to + encode and decode in a range of video formats + This driver rely on VPU driver to communicate with VPU. + + To compile this driver as a module, choose M here: the + module will be called mtk-vcodec + config VIDEO_MEM2MEM_DEINTERLACE tristate "Deinterlace support" depends on VIDEO_DEV && VIDEO_V4L2 && DMA_ENGINE @@ -247,10 +278,24 @@ config VIDEO_RENESAS_JPU To compile this driver as a module, choose M here: the module will be called rcar_jpu. +config VIDEO_RENESAS_FCP + tristate "Renesas Frame Compression Processor" + depends on ARCH_RENESAS || COMPILE_TEST + depends on OF + ---help--- + This is a driver for the Renesas Frame Compression Processor (FCP). + The FCP is a companion module of video processing modules in the + Renesas R-Car Gen3 SoCs. It handles memory access for the codec, + VSP and FDP modules. + + To compile this driver as a module, choose M here: the module + will be called rcar-fcp. + config VIDEO_RENESAS_VSP1 tristate "Renesas VSP1 Video Processing Engine" depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && HAS_DMA depends on (ARCH_RENESAS && OF) || COMPILE_TEST + depends on (!ARM64 && !VIDEO_RENESAS_FCP) || VIDEO_RENESAS_FCP select VIDEOBUF2_DMA_CONTIG ---help--- This is a V4L2 driver for the Renesas VSP1 video processing engine. diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile index bbb7bd1eb268..21771c1a13fb 100644 --- a/drivers/media/platform/Makefile +++ b/drivers/media/platform/Makefile @@ -46,6 +46,7 @@ obj-$(CONFIG_VIDEO_SH_VOU) += sh_vou.o obj-$(CONFIG_SOC_CAMERA) += soc_camera/ +obj-$(CONFIG_VIDEO_RENESAS_FCP) += rcar-fcp.o obj-$(CONFIG_VIDEO_RENESAS_JPU) += rcar_jpu.o obj-$(CONFIG_VIDEO_RENESAS_VSP1) += vsp1/ @@ -55,4 +56,10 @@ obj-$(CONFIG_VIDEO_AM437X_VPFE) += am437x/ obj-$(CONFIG_VIDEO_XILINX) += xilinx/ +obj-$(CONFIG_VIDEO_RCAR_VIN) += rcar-vin/ + ccflags-y += -I$(srctree)/drivers/media/i2c + +obj-$(CONFIG_VIDEO_MEDIATEK_VPU) += mtk-vpu/ + +obj-$(CONFIG_VIDEO_MEDIATEK_VCODEC) += mtk-vcodec/ diff --git a/drivers/media/platform/am437x/am437x-vpfe.c b/drivers/media/platform/am437x/am437x-vpfe.c index e749eb7c3be9..b33b9e35e60e 100644 --- a/drivers/media/platform/am437x/am437x-vpfe.c +++ b/drivers/media/platform/am437x/am437x-vpfe.c @@ -1901,21 +1901,20 @@ static void vpfe_calculate_offsets(struct vpfe_device *vpfe) * @nbuffers: ptr to number of buffers requested by application * @nplanes:: contains number of distinct video planes needed to hold a frame * @sizes[]: contains the size (in bytes) of each plane. - * @alloc_ctxs: ptr to allocation context + * @alloc_devs: ptr to allocation context * * This callback function is called when reqbuf() is called to adjust * the buffer count and buffer size */ static int vpfe_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct vpfe_device *vpfe = vb2_get_drv_priv(vq); unsigned size = vpfe->fmt.fmt.pix.sizeimage; if (vq->num_buffers + *nbuffers < 3) *nbuffers = 3 - vq->num_buffers; - alloc_ctxs[0] = vpfe->alloc_ctx; if (*nplanes) { if (sizes[0] < size) @@ -2364,13 +2363,6 @@ static int vpfe_probe_complete(struct vpfe_device *vpfe) goto probe_out; /* Initialize videobuf2 queue as per the buffer type */ - vpfe->alloc_ctx = vb2_dma_contig_init_ctx(vpfe->pdev); - if (IS_ERR(vpfe->alloc_ctx)) { - vpfe_err(vpfe, "Failed to get the context\n"); - err = PTR_ERR(vpfe->alloc_ctx); - goto probe_out; - } - q = &vpfe->buffer_queue; q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; q->io_modes = VB2_MMAP | VB2_DMABUF | VB2_READ; @@ -2381,11 +2373,11 @@ static int vpfe_probe_complete(struct vpfe_device *vpfe) q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &vpfe->lock; q->min_buffers_needed = 1; + q->dev = vpfe->pdev; err = vb2_queue_init(q); if (err) { vpfe_err(vpfe, "vb2_queue_init() failed\n"); - vb2_dma_contig_cleanup_ctx(vpfe->alloc_ctx); goto probe_out; } diff --git a/drivers/media/platform/am437x/am437x-vpfe.h b/drivers/media/platform/am437x/am437x-vpfe.h index 777bf97fea57..17d7aa426788 100644 --- a/drivers/media/platform/am437x/am437x-vpfe.h +++ b/drivers/media/platform/am437x/am437x-vpfe.h @@ -264,8 +264,6 @@ struct vpfe_device { struct v4l2_rect crop; /* Buffer queue used in video-buf */ struct vb2_queue buffer_queue; - /* Allocator-specific contexts for each plane */ - struct vb2_alloc_ctx *alloc_ctx; /* Queue of filled frames */ struct list_head dma_queue; /* IRQ lock for DMA queue */ diff --git a/drivers/media/platform/blackfin/bfin_capture.c b/drivers/media/platform/blackfin/bfin_capture.c index d0092dae7a57..8eb03397d736 100644 --- a/drivers/media/platform/blackfin/bfin_capture.c +++ b/drivers/media/platform/blackfin/bfin_capture.c @@ -91,8 +91,6 @@ struct bcap_device { struct bcap_buffer *cur_frm; /* buffer queue used in videobuf2 */ struct vb2_queue buffer_queue; - /* allocator-specific contexts for each plane */ - struct vb2_alloc_ctx *alloc_ctx; /* queue of filled frames */ struct list_head dma_queue; /* used in videobuf2 callback */ @@ -203,13 +201,12 @@ static void bcap_free_sensor_formats(struct bcap_device *bcap_dev) static int bcap_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct bcap_device *bcap_dev = vb2_get_drv_priv(vq); if (vq->num_buffers + *nbuffers < 2) *nbuffers = 2; - alloc_ctxs[0] = bcap_dev->alloc_ctx; if (*nplanes) return sizes[0] < bcap_dev->fmt.sizeimage ? -EINVAL : 0; @@ -820,12 +817,6 @@ static int bcap_probe(struct platform_device *pdev) } bcap_dev->ppi->priv = bcap_dev; - bcap_dev->alloc_ctx = vb2_dma_contig_init_ctx(&pdev->dev); - if (IS_ERR(bcap_dev->alloc_ctx)) { - ret = PTR_ERR(bcap_dev->alloc_ctx); - goto err_free_ppi; - } - vfd = &bcap_dev->video_dev; /* initialize field of video device */ vfd->release = video_device_release_empty; @@ -839,7 +830,7 @@ static int bcap_probe(struct platform_device *pdev) if (ret) { v4l2_err(pdev->dev.driver, "Unable to register v4l2 device\n"); - goto err_cleanup_ctx; + goto err_free_ppi; } v4l2_info(&bcap_dev->v4l2_dev, "v4l2 device registered\n"); @@ -863,6 +854,7 @@ static int bcap_probe(struct platform_device *pdev) q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &bcap_dev->mutex; q->min_buffers_needed = 1; + q->dev = &pdev->dev; ret = vb2_queue_init(q); if (ret) @@ -967,8 +959,6 @@ err_free_handler: v4l2_ctrl_handler_free(&bcap_dev->ctrl_handler); err_unreg_v4l2: v4l2_device_unregister(&bcap_dev->v4l2_dev); -err_cleanup_ctx: - vb2_dma_contig_cleanup_ctx(bcap_dev->alloc_ctx); err_free_ppi: ppi_delete_instance(bcap_dev->ppi); err_free_dev: @@ -986,7 +976,6 @@ static int bcap_remove(struct platform_device *pdev) video_unregister_device(&bcap_dev->video_dev); v4l2_ctrl_handler_free(&bcap_dev->ctrl_handler); v4l2_device_unregister(v4l2_dev); - vb2_dma_contig_cleanup_ctx(bcap_dev->alloc_ctx); ppi_delete_instance(bcap_dev->ppi); kfree(bcap_dev); return 0; diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c index 133ab9f70f85..c39718a63e5e 100644 --- a/drivers/media/platform/coda/coda-common.c +++ b/drivers/media/platform/coda/coda-common.c @@ -1139,7 +1139,7 @@ static void set_default_params(struct coda_ctx *ctx) */ static int coda_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct coda_ctx *ctx = vb2_get_drv_priv(vq); struct coda_q_data *q_data; @@ -1151,9 +1151,6 @@ static int coda_queue_setup(struct vb2_queue *vq, *nplanes = 1; sizes[0] = size; - /* Set to vb2-dma-contig allocator context, ignored by vb2-vmalloc */ - alloc_ctxs[0] = ctx->dev->alloc_ctx; - v4l2_dbg(1, coda_debug, &ctx->dev->v4l2_dev, "get %d buffer(s) of size %d each.\n", *nbuffers, size); @@ -1599,6 +1596,7 @@ static int coda_queue_init(struct coda_ctx *ctx, struct vb2_queue *vq) * that videobuf2 will keep the value of bytesused intact. */ vq->allow_zero_bytesused = 1; + vq->dev = &ctx->dev->plat_dev->dev; return vb2_queue_init(vq); } @@ -2040,16 +2038,10 @@ static void coda_fw_callback(const struct firmware *fw, void *context) if (ret < 0) goto put_pm; - dev->alloc_ctx = vb2_dma_contig_init_ctx(&pdev->dev); - if (IS_ERR(dev->alloc_ctx)) { - v4l2_err(&dev->v4l2_dev, "Failed to alloc vb2 context\n"); - goto put_pm; - } - dev->m2m_dev = v4l2_m2m_init(&coda_m2m_ops); if (IS_ERR(dev->m2m_dev)) { v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem device\n"); - goto rel_ctx; + goto put_pm; } for (i = 0; i < dev->devtype->num_vdevs; i++) { @@ -2072,8 +2064,6 @@ rel_vfd: while (--i >= 0) video_unregister_device(&dev->vfd[i]); v4l2_m2m_release(dev->m2m_dev); -rel_ctx: - vb2_dma_contig_cleanup_ctx(dev->alloc_ctx); put_pm: pm_runtime_put_sync(&pdev->dev); } @@ -2226,7 +2216,7 @@ static int coda_probe(struct platform_device *pdev) dev->rstc = devm_reset_control_get_optional(&pdev->dev, NULL); if (IS_ERR(dev->rstc)) { ret = PTR_ERR(dev->rstc); - if (ret == -ENOENT || ret == -ENOSYS) { + if (ret == -ENOENT || ret == -ENOTSUPP) { dev->rstc = NULL; } else { dev_err(&pdev->dev, "failed get reset control: %d\n", @@ -2324,8 +2314,6 @@ static int coda_remove(struct platform_device *pdev) if (dev->m2m_dev) v4l2_m2m_release(dev->m2m_dev); pm_runtime_disable(&pdev->dev); - if (dev->alloc_ctx) - vb2_dma_contig_cleanup_ctx(dev->alloc_ctx); v4l2_device_unregister(&dev->v4l2_dev); destroy_workqueue(dev->workqueue); if (dev->iram.vaddr) diff --git a/drivers/media/platform/coda/coda.h b/drivers/media/platform/coda/coda.h index 8f2c71e06966..53f96661683c 100644 --- a/drivers/media/platform/coda/coda.h +++ b/drivers/media/platform/coda/coda.h @@ -92,7 +92,6 @@ struct coda_dev { struct mutex coda_mutex; struct workqueue_struct *workqueue; struct v4l2_m2m_dev *m2m_dev; - struct vb2_alloc_ctx *alloc_ctx; struct list_head instances; unsigned long instance_mask; struct dentry *debugfs_root; diff --git a/drivers/media/platform/davinci/ccdc_hw_device.h b/drivers/media/platform/davinci/ccdc_hw_device.h index 86b9b3518965..ae5605de7679 100644 --- a/drivers/media/platform/davinci/ccdc_hw_device.h +++ b/drivers/media/platform/davinci/ccdc_hw_device.h @@ -80,13 +80,6 @@ struct ccdc_hw_ops { /* Pointer to function to get line length */ unsigned int (*get_line_length) (void); - /* Query CCDC control IDs */ - int (*queryctrl)(struct v4l2_queryctrl *qctrl); - /* Set CCDC control */ - int (*set_control)(struct v4l2_control *ctrl); - /* Get CCDC control */ - int (*get_control)(struct v4l2_control *ctrl); - /* Pointer to function to set frame buffer address */ void (*setfbaddr) (unsigned long addr); /* Pointer to function to get field id */ diff --git a/drivers/media/platform/davinci/vpbe_display.c b/drivers/media/platform/davinci/vpbe_display.c index 0abcdfe97a6c..0b1709e96673 100644 --- a/drivers/media/platform/davinci/vpbe_display.c +++ b/drivers/media/platform/davinci/vpbe_display.c @@ -230,7 +230,7 @@ static int vpbe_buffer_prepare(struct vb2_buffer *vb) static int vpbe_buffer_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { /* Get the file handle object and layer object */ @@ -242,7 +242,6 @@ vpbe_buffer_queue_setup(struct vb2_queue *vq, /* Store number of buffers allocated in numbuffer member */ if (vq->num_buffers + *nbuffers < VPBE_DEFAULT_NUM_BUFS) *nbuffers = VPBE_DEFAULT_NUM_BUFS - vq->num_buffers; - alloc_ctxs[0] = layer->alloc_ctx; if (*nplanes) return sizes[0] < layer->pix_fmt.sizeimage ? -EINVAL : 0; @@ -1451,20 +1450,13 @@ static int vpbe_display_probe(struct platform_device *pdev) q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->min_buffers_needed = 1; q->lock = &disp_dev->dev[i]->opslock; + q->dev = disp_dev->vpbe_dev->pdev; err = vb2_queue_init(q); if (err) { v4l2_err(v4l2_dev, "vb2_queue_init() failed\n"); goto probe_out; } - disp_dev->dev[i]->alloc_ctx = - vb2_dma_contig_init_ctx(disp_dev->vpbe_dev->pdev); - if (IS_ERR(disp_dev->dev[i]->alloc_ctx)) { - v4l2_err(v4l2_dev, "Failed to get the context\n"); - err = PTR_ERR(disp_dev->dev[i]->alloc_ctx); - goto probe_out; - } - INIT_LIST_HEAD(&disp_dev->dev[i]->dma_queue); if (register_device(disp_dev->dev[i], disp_dev, pdev)) { @@ -1482,7 +1474,6 @@ probe_out: for (k = 0; k < VPBE_DISPLAY_MAX_DEVICES; k++) { /* Unregister video device */ if (disp_dev->dev[k] != NULL) { - vb2_dma_contig_cleanup_ctx(disp_dev->dev[k]->alloc_ctx); video_unregister_device(&disp_dev->dev[k]->video_dev); kfree(disp_dev->dev[k]); } @@ -1510,7 +1501,6 @@ static int vpbe_display_remove(struct platform_device *pdev) for (i = 0; i < VPBE_DISPLAY_MAX_DEVICES; i++) { /* Get the pointer to the layer object */ vpbe_display_layer = disp_dev->dev[i]; - vb2_dma_contig_cleanup_ctx(vpbe_display_layer->alloc_ctx); /* Unregister video device */ video_unregister_device(&vpbe_display_layer->video_dev); diff --git a/drivers/media/platform/davinci/vpif_capture.c b/drivers/media/platform/davinci/vpif_capture.c index 08f7028c7560..5104cc0ee40e 100644 --- a/drivers/media/platform/davinci/vpif_capture.c +++ b/drivers/media/platform/davinci/vpif_capture.c @@ -107,14 +107,14 @@ static int vpif_buffer_prepare(struct vb2_buffer *vb) * @nbuffers: ptr to number of buffers requested by application * @nplanes:: contains number of distinct video planes needed to hold a frame * @sizes[]: contains the size (in bytes) of each plane. - * @alloc_ctxs: ptr to allocation context + * @alloc_devs: ptr to allocation context * * This callback function is called when reqbuf() is called to adjust * the buffer count and buffer size */ static int vpif_buffer_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct channel_obj *ch = vb2_get_drv_priv(vq); struct common_obj *common = &ch->common[VPIF_VIDEO_INDEX]; @@ -133,7 +133,6 @@ static int vpif_buffer_queue_setup(struct vb2_queue *vq, *nplanes = 1; sizes[0] = size; - alloc_ctxs[0] = common->alloc_ctx; /* Calculate the offset for Y and C data in the buffer */ vpif_calculate_offsets(ch); @@ -1371,6 +1370,7 @@ static int vpif_probe_complete(void) q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->min_buffers_needed = 1; q->lock = &common->lock; + q->dev = vpif_dev; err = vb2_queue_init(q); if (err) { @@ -1378,13 +1378,6 @@ static int vpif_probe_complete(void) goto probe_out; } - common->alloc_ctx = vb2_dma_contig_init_ctx(vpif_dev); - if (IS_ERR(common->alloc_ctx)) { - vpif_err("Failed to get the context\n"); - err = PTR_ERR(common->alloc_ctx); - goto probe_out; - } - INIT_LIST_HEAD(&common->dma_queue); /* Initialize the video_device structure */ @@ -1412,7 +1405,6 @@ probe_out: /* Get the pointer to the channel object */ ch = vpif_obj.dev[k]; common = &ch->common[k]; - vb2_dma_contig_cleanup_ctx(common->alloc_ctx); /* Unregister video device */ video_unregister_device(&ch->video_dev); } @@ -1546,7 +1538,6 @@ static int vpif_remove(struct platform_device *device) /* Get the pointer to the channel object */ ch = vpif_obj.dev[i]; common = &ch->common[VPIF_VIDEO_INDEX]; - vb2_dma_contig_cleanup_ctx(common->alloc_ctx); /* Unregister video device */ video_unregister_device(&ch->video_dev); kfree(vpif_obj.dev[i]); diff --git a/drivers/media/platform/davinci/vpif_capture.h b/drivers/media/platform/davinci/vpif_capture.h index 4a7600929b61..9e35b6771d22 100644 --- a/drivers/media/platform/davinci/vpif_capture.h +++ b/drivers/media/platform/davinci/vpif_capture.h @@ -65,8 +65,6 @@ struct common_obj { struct v4l2_format fmt; /* Buffer queue used in video-buf */ struct vb2_queue buffer_queue; - /* allocator-specific contexts for each plane */ - struct vb2_alloc_ctx *alloc_ctx; /* Queue of filled frames */ struct list_head dma_queue; /* Used in video-buf */ diff --git a/drivers/media/platform/davinci/vpif_display.c b/drivers/media/platform/davinci/vpif_display.c index f40755cf1bf2..75b27233ec2f 100644 --- a/drivers/media/platform/davinci/vpif_display.c +++ b/drivers/media/platform/davinci/vpif_display.c @@ -102,14 +102,14 @@ static int vpif_buffer_prepare(struct vb2_buffer *vb) * @nbuffers: ptr to number of buffers requested by application * @nplanes:: contains number of distinct video planes needed to hold a frame * @sizes[]: contains the size (in bytes) of each plane. - * @alloc_ctxs: ptr to allocation context + * @alloc_devs: ptr to allocation context * * This callback function is called when reqbuf() is called to adjust * the buffer count and buffer size */ static int vpif_buffer_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct channel_obj *ch = vb2_get_drv_priv(vq); struct common_obj *common = &ch->common[VPIF_VIDEO_INDEX]; @@ -126,7 +126,6 @@ static int vpif_buffer_queue_setup(struct vb2_queue *vq, *nplanes = 1; sizes[0] = size; - alloc_ctxs[0] = common->alloc_ctx; /* Calculate the offset for Y and C data in the buffer */ vpif_calculate_offsets(ch); @@ -1191,19 +1190,13 @@ static int vpif_probe_complete(void) q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->min_buffers_needed = 1; q->lock = &common->lock; + q->dev = vpif_dev; err = vb2_queue_init(q); if (err) { vpif_err("vpif_display: vb2_queue_init() failed\n"); goto probe_out; } - common->alloc_ctx = vb2_dma_contig_init_ctx(vpif_dev); - if (IS_ERR(common->alloc_ctx)) { - vpif_err("Failed to get the context\n"); - err = PTR_ERR(common->alloc_ctx); - goto probe_out; - } - INIT_LIST_HEAD(&common->dma_queue); /* register video device */ @@ -1233,7 +1226,6 @@ probe_out: for (k = 0; k < j; k++) { ch = vpif_obj.dev[k]; common = &ch->common[k]; - vb2_dma_contig_cleanup_ctx(common->alloc_ctx); video_unregister_device(&ch->video_dev); } return err; @@ -1355,7 +1347,6 @@ static int vpif_remove(struct platform_device *device) /* Get the pointer to the channel object */ ch = vpif_obj.dev[i]; common = &ch->common[VPIF_VIDEO_INDEX]; - vb2_dma_contig_cleanup_ctx(common->alloc_ctx); /* Unregister video device */ video_unregister_device(&ch->video_dev); kfree(vpif_obj.dev[i]); diff --git a/drivers/media/platform/davinci/vpif_display.h b/drivers/media/platform/davinci/vpif_display.h index e7a1723a1b7a..af2765fdcea8 100644 --- a/drivers/media/platform/davinci/vpif_display.h +++ b/drivers/media/platform/davinci/vpif_display.h @@ -74,8 +74,6 @@ struct common_obj { struct v4l2_format fmt; /* Used to store the format */ struct vb2_queue buffer_queue; /* Buffer queue used in * video-buf */ - /* allocator-specific contexts for each plane */ - struct vb2_alloc_ctx *alloc_ctx; struct list_head dma_queue; /* Queue of filled frames */ spinlock_t irqlock; /* Used in video-buf */ diff --git a/drivers/media/platform/exynos-gsc/gsc-core.c b/drivers/media/platform/exynos-gsc/gsc-core.c index c04973669a47..787bd16c19e5 100644 --- a/drivers/media/platform/exynos-gsc/gsc-core.c +++ b/drivers/media/platform/exynos-gsc/gsc-core.c @@ -1123,19 +1123,13 @@ static int gsc_probe(struct platform_device *pdev) if (ret < 0) goto err_m2m; - /* Initialize continious memory allocator */ - gsc->alloc_ctx = vb2_dma_contig_init_ctx(dev); - if (IS_ERR(gsc->alloc_ctx)) { - ret = PTR_ERR(gsc->alloc_ctx); - goto err_pm; - } + vb2_dma_contig_set_max_seg_size(dev, DMA_BIT_MASK(32)); dev_dbg(dev, "gsc-%d registered successfully\n", gsc->id); pm_runtime_put(dev); return 0; -err_pm: - pm_runtime_put(dev); + err_m2m: gsc_unregister_m2m_device(gsc); err_v4l2: @@ -1152,7 +1146,7 @@ static int gsc_remove(struct platform_device *pdev) gsc_unregister_m2m_device(gsc); v4l2_device_unregister(&gsc->v4l2_dev); - vb2_dma_contig_cleanup_ctx(gsc->alloc_ctx); + vb2_dma_contig_clear_max_seg_size(&pdev->dev); pm_runtime_disable(&pdev->dev); gsc_clk_put(gsc); diff --git a/drivers/media/platform/exynos-gsc/gsc-core.h b/drivers/media/platform/exynos-gsc/gsc-core.h index ec4000c72172..7ad7b9dc2243 100644 --- a/drivers/media/platform/exynos-gsc/gsc-core.h +++ b/drivers/media/platform/exynos-gsc/gsc-core.h @@ -327,7 +327,6 @@ struct gsc_driverdata { * @irq_queue: interrupt handler waitqueue * @m2m: memory-to-memory V4L2 device information * @state: flags used to synchronize m2m and capture mode operation - * @alloc_ctx: videobuf2 memory allocator context * @vdev: video device for G-Scaler instance */ struct gsc_dev { @@ -341,7 +340,6 @@ struct gsc_dev { wait_queue_head_t irq_queue; struct gsc_m2m_device m2m; unsigned long state; - struct vb2_alloc_ctx *alloc_ctx; struct video_device vdev; struct v4l2_device v4l2_dev; }; diff --git a/drivers/media/platform/exynos-gsc/gsc-m2m.c b/drivers/media/platform/exynos-gsc/gsc-m2m.c index a600e32e2543..ec6494cbdd45 100644 --- a/drivers/media/platform/exynos-gsc/gsc-m2m.c +++ b/drivers/media/platform/exynos-gsc/gsc-m2m.c @@ -213,7 +213,7 @@ put_device: static int gsc_m2m_queue_setup(struct vb2_queue *vq, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *allocators[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct gsc_ctx *ctx = vb2_get_drv_priv(vq); struct gsc_frame *frame; @@ -227,10 +227,8 @@ static int gsc_m2m_queue_setup(struct vb2_queue *vq, return -EINVAL; *num_planes = frame->fmt->num_planes; - for (i = 0; i < frame->fmt->num_planes; i++) { + for (i = 0; i < frame->fmt->num_planes; i++) sizes[i] = frame->payload[i]; - allocators[i] = ctx->gsc_dev->alloc_ctx; - } return 0; } @@ -591,6 +589,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq, src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; src_vq->lock = &ctx->gsc_dev->lock; + src_vq->dev = &ctx->gsc_dev->pdev->dev; ret = vb2_queue_init(src_vq); if (ret) @@ -605,6 +604,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; dst_vq->lock = &ctx->gsc_dev->lock; + dst_vq->dev = &ctx->gsc_dev->pdev->dev; return vb2_queue_init(dst_vq); } diff --git a/drivers/media/platform/exynos4-is/fimc-capture.c b/drivers/media/platform/exynos4-is/fimc-capture.c index bf47d3b9cbe7..fdec499fbbda 100644 --- a/drivers/media/platform/exynos4-is/fimc-capture.c +++ b/drivers/media/platform/exynos4-is/fimc-capture.c @@ -340,7 +340,7 @@ int fimc_capture_resume(struct fimc_dev *fimc) static int queue_setup(struct vb2_queue *vq, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *allocators[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct fimc_ctx *ctx = vq->drv_priv; struct fimc_frame *frame = &ctx->d_frame; @@ -354,11 +354,9 @@ static int queue_setup(struct vb2_queue *vq, if (*num_planes) { if (*num_planes != fmt->memplanes) return -EINVAL; - for (i = 0; i < *num_planes; i++) { + for (i = 0; i < *num_planes; i++) if (sizes[i] < (wh * fmt->depth[i]) / 8) return -EINVAL; - allocators[i] = ctx->fimc_dev->alloc_ctx; - } return 0; } @@ -371,8 +369,6 @@ static int queue_setup(struct vb2_queue *vq, sizes[i] = frame->payload[i]; else sizes[i] = max_t(u32, size, frame->payload[i]); - - allocators[i] = ctx->fimc_dev->alloc_ctx; } return 0; @@ -1779,6 +1775,7 @@ static int fimc_register_capture_device(struct fimc_dev *fimc, q->buf_struct_size = sizeof(struct fimc_vid_buffer); q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &fimc->lock; + q->dev = &fimc->pdev->dev; ret = vb2_queue_init(q); if (ret) diff --git a/drivers/media/platform/exynos4-is/fimc-core.c b/drivers/media/platform/exynos4-is/fimc-core.c index b1c1cea82a27..8f89ca21b631 100644 --- a/drivers/media/platform/exynos4-is/fimc-core.c +++ b/drivers/media/platform/exynos4-is/fimc-core.c @@ -1018,19 +1018,11 @@ static int fimc_probe(struct platform_device *pdev) goto err_sd; } - /* Initialize contiguous memory allocator */ - fimc->alloc_ctx = vb2_dma_contig_init_ctx(dev); - if (IS_ERR(fimc->alloc_ctx)) { - ret = PTR_ERR(fimc->alloc_ctx); - goto err_gclk; - } + vb2_dma_contig_set_max_seg_size(dev, DMA_BIT_MASK(32)); dev_dbg(dev, "FIMC.%d registered successfully\n", fimc->id); return 0; -err_gclk: - if (!pm_runtime_enabled(dev)) - clk_disable(fimc->clock[CLK_GATE]); err_sd: fimc_unregister_capture_subdev(fimc); err_sclk: @@ -1123,7 +1115,7 @@ static int fimc_remove(struct platform_device *pdev) pm_runtime_set_suspended(&pdev->dev); fimc_unregister_capture_subdev(fimc); - vb2_dma_contig_cleanup_ctx(fimc->alloc_ctx); + vb2_dma_contig_clear_max_seg_size(&pdev->dev); clk_disable(fimc->clock[CLK_BUS]); fimc_clk_put(fimc); diff --git a/drivers/media/platform/exynos4-is/fimc-core.h b/drivers/media/platform/exynos4-is/fimc-core.h index 6b7435453d2a..5615fefbf7af 100644 --- a/drivers/media/platform/exynos4-is/fimc-core.h +++ b/drivers/media/platform/exynos4-is/fimc-core.h @@ -307,7 +307,6 @@ struct fimc_m2m_device { */ struct fimc_vid_cap { struct fimc_ctx *ctx; - struct vb2_alloc_ctx *alloc_ctx; struct v4l2_subdev subdev; struct exynos_video_entity ve; struct media_pad vd_pad; @@ -417,7 +416,6 @@ struct fimc_ctx; * @m2m: memory-to-memory V4L2 device information * @vid_cap: camera capture device information * @state: flags used to synchronize m2m and capture mode operation - * @alloc_ctx: videobuf2 memory allocator context * @pipeline: fimc video capture pipeline data structure */ struct fimc_dev { @@ -436,7 +434,6 @@ struct fimc_dev { struct fimc_m2m_device m2m; struct fimc_vid_cap vid_cap; unsigned long state; - struct vb2_alloc_ctx *alloc_ctx; }; /** diff --git a/drivers/media/platform/exynos4-is/fimc-is.c b/drivers/media/platform/exynos4-is/fimc-is.c index 979c388ebf60..32ca55f16677 100644 --- a/drivers/media/platform/exynos4-is/fimc-is.c +++ b/drivers/media/platform/exynos4-is/fimc-is.c @@ -204,9 +204,6 @@ static int fimc_is_register_subdevs(struct fimc_is *is) if (ret < 0) return ret; - /* Initialize memory allocator context for the ISP DMA. */ - is->isp.alloc_ctx = is->alloc_ctx; - for_each_compatible_node(i2c_bus, NULL, FIMC_IS_I2C_COMPATIBLE) { for_each_available_child_of_node(i2c_bus, child) { ret = fimc_is_parse_sensor_config(is, index, child); @@ -847,18 +844,14 @@ static int fimc_is_probe(struct platform_device *pdev) if (ret < 0) goto err_pm; - is->alloc_ctx = vb2_dma_contig_init_ctx(dev); - if (IS_ERR(is->alloc_ctx)) { - ret = PTR_ERR(is->alloc_ctx); - goto err_pm; - } + vb2_dma_contig_set_max_seg_size(dev, DMA_BIT_MASK(32)); /* * Register FIMC-IS V4L2 subdevs to this driver. The video nodes * will be created within the subdev's registered() callback. */ ret = fimc_is_register_subdevs(is); if (ret < 0) - goto err_vb; + goto err_pm; ret = fimc_is_debugfs_create(is); if (ret < 0) @@ -877,8 +870,6 @@ err_dfs: fimc_is_debugfs_remove(is); err_sd: fimc_is_unregister_subdevs(is); -err_vb: - vb2_dma_contig_cleanup_ctx(is->alloc_ctx); err_pm: if (!pm_runtime_enabled(dev)) fimc_is_runtime_suspend(dev); @@ -939,7 +930,7 @@ static int fimc_is_remove(struct platform_device *pdev) fimc_is_runtime_suspend(dev); free_irq(is->irq, is); fimc_is_unregister_subdevs(is); - vb2_dma_contig_cleanup_ctx(is->alloc_ctx); + vb2_dma_contig_clear_max_seg_size(dev); fimc_is_put_clocks(is); fimc_is_debugfs_remove(is); release_firmware(is->fw.f_w); diff --git a/drivers/media/platform/exynos4-is/fimc-is.h b/drivers/media/platform/exynos4-is/fimc-is.h index 386eb49ece7e..3a82c6a214c7 100644 --- a/drivers/media/platform/exynos4-is/fimc-is.h +++ b/drivers/media/platform/exynos4-is/fimc-is.h @@ -233,7 +233,6 @@ struct chain_config { * @pdev: pointer to FIMC-IS platform device * @pctrl: pointer to pinctrl structure for this device * @v4l2_dev: pointer to top the level v4l2_device - * @alloc_ctx: videobuf2 memory allocator context * @lock: mutex serializing video device and the subdev operations * @slock: spinlock protecting this data structure and the hw registers * @clocks: FIMC-LITE gate clock @@ -256,7 +255,6 @@ struct fimc_is { struct fimc_is_sensor sensor[FIMC_IS_SENSORS_NUM]; struct fimc_is_setfile setfile; - struct vb2_alloc_ctx *alloc_ctx; struct v4l2_ctrl_handler ctrl_handler; struct mutex lock; diff --git a/drivers/media/platform/exynos4-is/fimc-isp-video.c b/drivers/media/platform/exynos4-is/fimc-isp-video.c index c0816728cbfe..400ce0cb0c0d 100644 --- a/drivers/media/platform/exynos4-is/fimc-isp-video.c +++ b/drivers/media/platform/exynos4-is/fimc-isp-video.c @@ -40,7 +40,7 @@ static int isp_video_capture_queue_setup(struct vb2_queue *vq, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *allocators[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct fimc_isp *isp = vb2_get_drv_priv(vq); struct v4l2_pix_format_mplane *vid_fmt = &isp->video_capture.pixfmt; @@ -57,20 +57,16 @@ static int isp_video_capture_queue_setup(struct vb2_queue *vq, if (*num_planes) { if (*num_planes != fmt->memplanes) return -EINVAL; - for (i = 0; i < *num_planes; i++) { + for (i = 0; i < *num_planes; i++) if (sizes[i] < (wh * fmt->depth[i]) / 8) return -EINVAL; - allocators[i] = isp->alloc_ctx; - } return 0; } *num_planes = fmt->memplanes; - for (i = 0; i < fmt->memplanes; i++) { + for (i = 0; i < fmt->memplanes; i++) sizes[i] = (wh * fmt->depth[i]) / 8; - allocators[i] = isp->alloc_ctx; - } return 0; } @@ -597,6 +593,7 @@ int fimc_isp_video_device_register(struct fimc_isp *isp, q->drv_priv = isp; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &isp->video_lock; + q->dev = &isp->pdev->dev; ret = vb2_queue_init(q); if (ret < 0) diff --git a/drivers/media/platform/exynos4-is/fimc-isp.h b/drivers/media/platform/exynos4-is/fimc-isp.h index e0686b5f1bf8..3cdd52491294 100644 --- a/drivers/media/platform/exynos4-is/fimc-isp.h +++ b/drivers/media/platform/exynos4-is/fimc-isp.h @@ -148,7 +148,6 @@ struct fimc_is_video { /** * struct fimc_isp - FIMC-IS ISP data structure * @pdev: pointer to FIMC-IS platform device - * @alloc_ctx: videobuf2 memory allocator context * @subdev: ISP v4l2_subdev * @subdev_pads: the ISP subdev media pads * @test_pattern: test pattern controls @@ -161,7 +160,6 @@ struct fimc_is_video { */ struct fimc_isp { struct platform_device *pdev; - struct vb2_alloc_ctx *alloc_ctx; struct v4l2_subdev subdev; struct media_pad subdev_pads[FIMC_ISP_SD_PADS_NUM]; struct v4l2_mbus_framefmt src_fmt; diff --git a/drivers/media/platform/exynos4-is/fimc-lite.c b/drivers/media/platform/exynos4-is/fimc-lite.c index dc1b929f7a33..a0f149fb88e1 100644 --- a/drivers/media/platform/exynos4-is/fimc-lite.c +++ b/drivers/media/platform/exynos4-is/fimc-lite.c @@ -357,7 +357,7 @@ static void stop_streaming(struct vb2_queue *q) static int queue_setup(struct vb2_queue *vq, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *allocators[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct fimc_lite *fimc = vq->drv_priv; struct flite_frame *frame = &fimc->out_frame; @@ -371,20 +371,16 @@ static int queue_setup(struct vb2_queue *vq, if (*num_planes) { if (*num_planes != fmt->memplanes) return -EINVAL; - for (i = 0; i < *num_planes; i++) { + for (i = 0; i < *num_planes; i++) if (sizes[i] < (wh * fmt->depth[i]) / 8) return -EINVAL; - allocators[i] = fimc->alloc_ctx; - } return 0; } *num_planes = fmt->memplanes; - for (i = 0; i < fmt->memplanes; i++) { + for (i = 0; i < fmt->memplanes; i++) sizes[i] = (wh * fmt->depth[i]) / 8; - allocators[i] = fimc->alloc_ctx; - } return 0; } @@ -1300,6 +1296,7 @@ static int fimc_lite_subdev_registered(struct v4l2_subdev *sd) q->drv_priv = fimc; q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; q->lock = &fimc->lock; + q->dev = &fimc->pdev->dev; ret = vb2_queue_init(q); if (ret < 0) @@ -1551,11 +1548,7 @@ static int fimc_lite_probe(struct platform_device *pdev) goto err_sd; } - fimc->alloc_ctx = vb2_dma_contig_init_ctx(dev); - if (IS_ERR(fimc->alloc_ctx)) { - ret = PTR_ERR(fimc->alloc_ctx); - goto err_clk_dis; - } + vb2_dma_contig_set_max_seg_size(dev, DMA_BIT_MASK(32)); fimc_lite_set_default_config(fimc); @@ -1563,9 +1556,6 @@ static int fimc_lite_probe(struct platform_device *pdev) fimc->index); return 0; -err_clk_dis: - if (!pm_runtime_enabled(dev)) - clk_disable(fimc->clock); err_sd: fimc_lite_unregister_capture_subdev(fimc); err_clk_put: @@ -1651,7 +1641,7 @@ static int fimc_lite_remove(struct platform_device *pdev) pm_runtime_disable(dev); pm_runtime_set_suspended(dev); fimc_lite_unregister_capture_subdev(fimc); - vb2_dma_contig_cleanup_ctx(fimc->alloc_ctx); + vb2_dma_contig_clear_max_seg_size(dev); fimc_lite_clk_put(fimc); dev_info(dev, "Driver unloaded\n"); diff --git a/drivers/media/platform/exynos4-is/fimc-lite.h b/drivers/media/platform/exynos4-is/fimc-lite.h index 11690d563e06..9ae1e96a1bc7 100644 --- a/drivers/media/platform/exynos4-is/fimc-lite.h +++ b/drivers/media/platform/exynos4-is/fimc-lite.h @@ -113,7 +113,6 @@ struct flite_buffer { * @ve: exynos video device entity structure * @v4l2_dev: pointer to top the level v4l2_device * @fh: v4l2 file handle - * @alloc_ctx: videobuf2 memory allocator context * @subdev: FIMC-LITE subdev * @vd_pad: media (sink) pad for the capture video node * @subdev_pads: the subdev media pads @@ -148,7 +147,6 @@ struct fimc_lite { struct exynos_video_entity ve; struct v4l2_device *v4l2_dev; struct v4l2_fh fh; - struct vb2_alloc_ctx *alloc_ctx; struct v4l2_subdev subdev; struct media_pad vd_pad; struct media_pad subdev_pads[FLITE_SD_PADS_NUM]; diff --git a/drivers/media/platform/exynos4-is/fimc-m2m.c b/drivers/media/platform/exynos4-is/fimc-m2m.c index 55ec4c99d484..b1309e114edb 100644 --- a/drivers/media/platform/exynos4-is/fimc-m2m.c +++ b/drivers/media/platform/exynos4-is/fimc-m2m.c @@ -50,30 +50,28 @@ void fimc_m2m_job_finish(struct fimc_ctx *ctx, int vb_state) src_vb = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); dst_vb = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); - if (src_vb && dst_vb) { + if (src_vb) v4l2_m2m_buf_done(src_vb, vb_state); + if (dst_vb) v4l2_m2m_buf_done(dst_vb, vb_state); + if (src_vb && dst_vb) v4l2_m2m_job_finish(ctx->fimc_dev->m2m.m2m_dev, ctx->fh.m2m_ctx); - } } /* Complete the transaction which has been scheduled for execution. */ -static int fimc_m2m_shutdown(struct fimc_ctx *ctx) +static void fimc_m2m_shutdown(struct fimc_ctx *ctx) { struct fimc_dev *fimc = ctx->fimc_dev; - int ret; if (!fimc_m2m_pending(fimc)) - return 0; + return; fimc_ctx_state_set(FIMC_CTX_SHUT, ctx); - ret = wait_event_timeout(fimc->irq_queue, - !fimc_ctx_state_is_set(FIMC_CTX_SHUT, ctx), - FIMC_SHUTDOWN_TIMEOUT); - - return ret == 0 ? -ETIMEDOUT : ret; + wait_event_timeout(fimc->irq_queue, + !fimc_ctx_state_is_set(FIMC_CTX_SHUT, ctx), + FIMC_SHUTDOWN_TIMEOUT); } static int start_streaming(struct vb2_queue *q, unsigned int count) @@ -88,12 +86,10 @@ static int start_streaming(struct vb2_queue *q, unsigned int count) static void stop_streaming(struct vb2_queue *q) { struct fimc_ctx *ctx = q->drv_priv; - int ret; - ret = fimc_m2m_shutdown(ctx); - if (ret == -ETIMEDOUT) - fimc_m2m_job_finish(ctx, VB2_BUF_STATE_ERROR); + fimc_m2m_shutdown(ctx); + fimc_m2m_job_finish(ctx, VB2_BUF_STATE_ERROR); pm_runtime_put(&ctx->fimc_dev->pdev->dev); } @@ -178,7 +174,7 @@ static void fimc_job_abort(void *priv) static int fimc_queue_setup(struct vb2_queue *vq, unsigned int *num_buffers, unsigned int *num_planes, - unsigned int sizes[], void *allocators[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct fimc_ctx *ctx = vb2_get_drv_priv(vq); struct fimc_frame *f; @@ -195,10 +191,8 @@ static int fimc_queue_setup(struct vb2_queue *vq, return -EINVAL; *num_planes = f->fmt->memplanes; - for (i = 0; i < f->fmt->memplanes; i++) { + for (i = 0; i < f->fmt->memplanes; i++) sizes[i] = f->payload[i]; - allocators[i] = ctx->fimc_dev->alloc_ctx; - } return 0; } @@ -562,6 +556,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq, src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; src_vq->lock = &ctx->fimc_dev->lock; + src_vq->dev = &ctx->fimc_dev->pdev->dev; ret = vb2_queue_init(src_vq); if (ret) @@ -575,6 +570,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; dst_vq->lock = &ctx->fimc_dev->lock; + dst_vq->dev = &ctx->fimc_dev->pdev->dev; return vb2_queue_init(dst_vq); } diff --git a/drivers/media/platform/exynos4-is/mipi-csis.c b/drivers/media/platform/exynos4-is/mipi-csis.c index bf954424e7be..86e681daa89d 100644 --- a/drivers/media/platform/exynos4-is/mipi-csis.c +++ b/drivers/media/platform/exynos4-is/mipi-csis.c @@ -649,23 +649,6 @@ static int s5pcsis_log_status(struct v4l2_subdev *sd) return 0; } -static int s5pcsis_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) -{ - struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd, fh->pad, 0); - - format->colorspace = V4L2_COLORSPACE_JPEG; - format->code = s5pcsis_formats[0].code; - format->width = S5PCSIS_DEF_PIX_WIDTH; - format->height = S5PCSIS_DEF_PIX_HEIGHT; - format->field = V4L2_FIELD_NONE; - - return 0; -} - -static const struct v4l2_subdev_internal_ops s5pcsis_sd_internal_ops = { - .open = s5pcsis_open, -}; - static struct v4l2_subdev_core_ops s5pcsis_core_ops = { .s_power = s5pcsis_s_power, .log_status = s5pcsis_log_status, diff --git a/drivers/media/platform/m2m-deinterlace.c b/drivers/media/platform/m2m-deinterlace.c index 7383818c2be6..0fcb5c78031d 100644 --- a/drivers/media/platform/m2m-deinterlace.c +++ b/drivers/media/platform/m2m-deinterlace.c @@ -136,7 +136,6 @@ struct deinterlace_dev { struct dma_chan *dma_chan; struct v4l2_m2m_dev *m2m_dev; - struct vb2_alloc_ctx *alloc_ctx; }; struct deinterlace_ctx { @@ -799,7 +798,7 @@ struct vb2_dc_conf { static int deinterlace_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct deinterlace_ctx *ctx = vb2_get_drv_priv(vq); struct deinterlace_q_data *q_data; @@ -820,8 +819,6 @@ static int deinterlace_queue_setup(struct vb2_queue *vq, *nbuffers = count; sizes[0] = size; - alloc_ctxs[0] = ctx->dev->alloc_ctx; - dprintk(ctx->dev, "get %d buffer(s) of size %d each.\n", count, size); return 0; @@ -874,6 +871,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq, src_vq->ops = &deinterlace_qops; src_vq->mem_ops = &vb2_dma_contig_memops; src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + src_vq->dev = ctx->dev->v4l2_dev.dev; q_data[V4L2_M2M_SRC].fmt = &formats[0]; q_data[V4L2_M2M_SRC].width = 640; q_data[V4L2_M2M_SRC].height = 480; @@ -891,6 +889,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->ops = &deinterlace_qops; dst_vq->mem_ops = &vb2_dma_contig_memops; dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + dst_vq->dev = ctx->dev->v4l2_dev.dev; q_data[V4L2_M2M_DST].fmt = &formats[0]; q_data[V4L2_M2M_DST].width = 640; q_data[V4L2_M2M_DST].height = 480; @@ -1046,13 +1045,6 @@ static int deinterlace_probe(struct platform_device *pdev) platform_set_drvdata(pdev, pcdev); - pcdev->alloc_ctx = vb2_dma_contig_init_ctx(&pdev->dev); - if (IS_ERR(pcdev->alloc_ctx)) { - v4l2_err(&pcdev->v4l2_dev, "Failed to alloc vb2 context\n"); - ret = PTR_ERR(pcdev->alloc_ctx); - goto err_ctx; - } - pcdev->m2m_dev = v4l2_m2m_init(&m2m_ops); if (IS_ERR(pcdev->m2m_dev)) { v4l2_err(&pcdev->v4l2_dev, "Failed to init mem2mem device\n"); @@ -1064,8 +1056,6 @@ static int deinterlace_probe(struct platform_device *pdev) err_m2m: video_unregister_device(&pcdev->vfd); -err_ctx: - vb2_dma_contig_cleanup_ctx(pcdev->alloc_ctx); unreg_dev: v4l2_device_unregister(&pcdev->v4l2_dev); rel_dma: @@ -1082,7 +1072,6 @@ static int deinterlace_remove(struct platform_device *pdev) v4l2_m2m_release(pcdev->m2m_dev); video_unregister_device(&pcdev->vfd); v4l2_device_unregister(&pcdev->v4l2_dev); - vb2_dma_contig_cleanup_ctx(pcdev->alloc_ctx); dma_release_channel(pcdev->dma_chan); return 0; diff --git a/drivers/media/platform/marvell-ccic/mcam-core.c b/drivers/media/platform/marvell-ccic/mcam-core.c index 9b878deb1437..af59bf4dca2d 100644 --- a/drivers/media/platform/marvell-ccic/mcam-core.c +++ b/drivers/media/platform/marvell-ccic/mcam-core.c @@ -973,7 +973,7 @@ static int mcam_cam_set_flip(struct mcam_camera *cam) memset(&ctrl, 0, sizeof(ctrl)); ctrl.id = V4L2_CID_VFLIP; ctrl.value = flip; - return sensor_call(cam, core, s_ctrl, &ctrl); + return v4l2_s_ctrl(NULL, cam->sensor->ctrl_handler, &ctrl); } @@ -1051,7 +1051,7 @@ static int mcam_read_setup(struct mcam_camera *cam) static int mcam_vb_queue_setup(struct vb2_queue *vq, unsigned int *nbufs, unsigned int *num_planes, unsigned int sizes[], - void *alloc_ctxs[]) + struct device *alloc_devs[]) { struct mcam_camera *cam = vb2_get_drv_priv(vq); int minbufs = (cam->buffer_mode == B_DMA_contig) ? 3 : 2; @@ -1059,10 +1059,6 @@ static int mcam_vb_queue_setup(struct vb2_queue *vq, if (*nbufs < minbufs) *nbufs = minbufs; - if (cam->buffer_mode == B_DMA_contig) - alloc_ctxs[0] = cam->vb_alloc_ctx; - else if (cam->buffer_mode == B_DMA_sg) - alloc_ctxs[0] = cam->vb_alloc_ctx_sg; if (*num_planes) return sizes[0] < size ? -EINVAL : 0; @@ -1271,6 +1267,7 @@ static int mcam_setup_vb2(struct mcam_camera *cam) vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; vq->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF | VB2_READ; vq->buf_struct_size = sizeof(struct mcam_vb_buffer); + vq->dev = cam->dev; INIT_LIST_HEAD(&cam->buffers); switch (cam->buffer_mode) { case B_DMA_contig: @@ -1279,9 +1276,6 @@ static int mcam_setup_vb2(struct mcam_camera *cam) vq->mem_ops = &vb2_dma_contig_memops; cam->dma_setup = mcam_ctlr_dma_contig; cam->frame_complete = mcam_dma_contig_done; - cam->vb_alloc_ctx = vb2_dma_contig_init_ctx(cam->dev); - if (IS_ERR(cam->vb_alloc_ctx)) - return PTR_ERR(cam->vb_alloc_ctx); #endif break; case B_DMA_sg: @@ -1290,9 +1284,6 @@ static int mcam_setup_vb2(struct mcam_camera *cam) vq->mem_ops = &vb2_dma_sg_memops; cam->dma_setup = mcam_ctlr_dma_sg; cam->frame_complete = mcam_dma_sg_done; - cam->vb_alloc_ctx_sg = vb2_dma_sg_init_ctx(cam->dev); - if (IS_ERR(cam->vb_alloc_ctx_sg)) - return PTR_ERR(cam->vb_alloc_ctx_sg); #endif break; case B_vmalloc: @@ -1309,18 +1300,6 @@ static int mcam_setup_vb2(struct mcam_camera *cam) return vb2_queue_init(vq); } -static void mcam_cleanup_vb2(struct mcam_camera *cam) -{ -#ifdef MCAM_MODE_DMA_CONTIG - if (cam->buffer_mode == B_DMA_contig) - vb2_dma_contig_cleanup_ctx(cam->vb_alloc_ctx); -#endif -#ifdef MCAM_MODE_DMA_SG - if (cam->buffer_mode == B_DMA_sg) - vb2_dma_sg_cleanup_ctx(cam->vb_alloc_ctx_sg); -#endif -} - /* ---------------------------------------------------------------------- */ /* @@ -1875,7 +1854,6 @@ void mccic_shutdown(struct mcam_camera *cam) cam_warn(cam, "Removing a device with users!\n"); mcam_ctlr_power_down(cam); } - mcam_cleanup_vb2(cam); if (cam->buffer_mode == B_vmalloc) mcam_free_dma_bufs(cam); video_unregister_device(&cam->vdev); diff --git a/drivers/media/platform/marvell-ccic/mcam-core.h b/drivers/media/platform/marvell-ccic/mcam-core.h index 35cd9e5aedf8..beb339f5561f 100644 --- a/drivers/media/platform/marvell-ccic/mcam-core.h +++ b/drivers/media/platform/marvell-ccic/mcam-core.h @@ -176,8 +176,6 @@ struct mcam_camera { /* DMA buffers - DMA modes */ struct mcam_vb_buffer *vb_bufs[MAX_DMA_BUFS]; - struct vb2_alloc_ctx *vb_alloc_ctx; - struct vb2_alloc_ctx *vb_alloc_ctx_sg; /* Mode-specific ops, set at open time */ void (*dma_setup)(struct mcam_camera *cam); diff --git a/drivers/media/platform/mtk-vcodec/Makefile b/drivers/media/platform/mtk-vcodec/Makefile new file mode 100644 index 000000000000..dc5cb006d600 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/Makefile @@ -0,0 +1,19 @@ + + +obj-$(CONFIG_VIDEO_MEDIATEK_VCODEC) += mtk-vcodec-enc.o mtk-vcodec-common.o + + + +mtk-vcodec-enc-y := venc/venc_vp8_if.o \ + venc/venc_h264_if.o \ + mtk_vcodec_enc.o \ + mtk_vcodec_enc_drv.o \ + mtk_vcodec_enc_pm.o \ + venc_drv_if.o \ + venc_vpu_if.o \ + + +mtk-vcodec-common-y := mtk_vcodec_intr.o \ + mtk_vcodec_util.o\ + +ccflags-y += -I$(srctree)/drivers/media/platform/mtk-vpu diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h new file mode 100644 index 000000000000..94f0a425be42 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h @@ -0,0 +1,335 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: PC Chen +* Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#ifndef _MTK_VCODEC_DRV_H_ +#define _MTK_VCODEC_DRV_H_ + +#include +#include +#include +#include +#include +#include + +#include "mtk_vcodec_util.h" + +#define MTK_VCODEC_DRV_NAME "mtk_vcodec_drv" +#define MTK_VCODEC_ENC_NAME "mtk-vcodec-enc" +#define MTK_PLATFORM_STR "platform:mt8173" + + +#define MTK_VCODEC_MAX_PLANES 3 +#define MTK_V4L2_BENCHMARK 0 +#define WAIT_INTR_TIMEOUT_MS 1000 + +/** + * enum mtk_hw_reg_idx - MTK hw register base index + */ +enum mtk_hw_reg_idx { + VDEC_SYS, + VDEC_MISC, + VDEC_LD, + VDEC_TOP, + VDEC_CM, + VDEC_AD, + VDEC_AV, + VDEC_PP, + VDEC_HWD, + VDEC_HWQ, + VDEC_HWB, + VDEC_HWG, + NUM_MAX_VDEC_REG_BASE, + /* h264 encoder */ + VENC_SYS = NUM_MAX_VDEC_REG_BASE, + /* vp8 encoder */ + VENC_LT_SYS, + NUM_MAX_VCODEC_REG_BASE +}; + +/** + * enum mtk_instance_type - The type of an MTK Vcodec instance. + */ +enum mtk_instance_type { + MTK_INST_DECODER = 0, + MTK_INST_ENCODER = 1, +}; + +/** + * enum mtk_instance_state - The state of an MTK Vcodec instance. + * @MTK_STATE_FREE - default state when instance is created + * @MTK_STATE_INIT - vcodec instance is initialized + * @MTK_STATE_HEADER - vdec had sps/pps header parsed or venc + * had sps/pps header encoded + * @MTK_STATE_FLUSH - vdec is flushing. Only used by decoder + * @MTK_STATE_ABORT - vcodec should be aborted + */ +enum mtk_instance_state { + MTK_STATE_FREE = 0, + MTK_STATE_INIT = 1, + MTK_STATE_HEADER = 2, + MTK_STATE_FLUSH = 3, + MTK_STATE_ABORT = 4, +}; + +/** + * struct mtk_encode_param - General encoding parameters type + */ +enum mtk_encode_param { + MTK_ENCODE_PARAM_NONE = 0, + MTK_ENCODE_PARAM_BITRATE = (1 << 0), + MTK_ENCODE_PARAM_FRAMERATE = (1 << 1), + MTK_ENCODE_PARAM_INTRA_PERIOD = (1 << 2), + MTK_ENCODE_PARAM_FORCE_INTRA = (1 << 3), + MTK_ENCODE_PARAM_GOP_SIZE = (1 << 4), +}; + +enum mtk_fmt_type { + MTK_FMT_DEC = 0, + MTK_FMT_ENC = 1, + MTK_FMT_FRAME = 2, +}; + +/** + * struct mtk_video_fmt - Structure used to store information about pixelformats + */ +struct mtk_video_fmt { + u32 fourcc; + enum mtk_fmt_type type; + u32 num_planes; +}; + +/** + * struct mtk_codec_framesizes - Structure used to store information about + * framesizes + */ +struct mtk_codec_framesizes { + u32 fourcc; + struct v4l2_frmsize_stepwise stepwise; +}; + +/** + * struct mtk_q_type - Type of queue + */ +enum mtk_q_type { + MTK_Q_DATA_SRC = 0, + MTK_Q_DATA_DST = 1, +}; + +/** + * struct mtk_q_data - Structure used to store information about queue + */ +struct mtk_q_data { + unsigned int visible_width; + unsigned int visible_height; + unsigned int coded_width; + unsigned int coded_height; + enum v4l2_field field; + unsigned int bytesperline[MTK_VCODEC_MAX_PLANES]; + unsigned int sizeimage[MTK_VCODEC_MAX_PLANES]; + struct mtk_video_fmt *fmt; +}; + +/** + * struct mtk_enc_params - General encoding parameters + * @bitrate: target bitrate in bits per second + * @num_b_frame: number of b frames between p-frame + * @rc_frame: frame based rate control + * @rc_mb: macroblock based rate control + * @seq_hdr_mode: H.264 sequence header is encoded separately or joined + * with the first frame + * @intra_period: I frame period + * @gop_size: group of picture size, it's used as the intra frame period + * @framerate_num: frame rate numerator. ex: framerate_num=30 and + * framerate_denom=1 menas FPS is 30 + * @framerate_denom: frame rate denominator. ex: framerate_num=30 and + * framerate_denom=1 menas FPS is 30 + * @h264_max_qp: Max value for H.264 quantization parameter + * @h264_profile: V4L2 defined H.264 profile + * @h264_level: V4L2 defined H.264 level + * @force_intra: force/insert intra frame + */ +struct mtk_enc_params { + unsigned int bitrate; + unsigned int num_b_frame; + unsigned int rc_frame; + unsigned int rc_mb; + unsigned int seq_hdr_mode; + unsigned int intra_period; + unsigned int gop_size; + unsigned int framerate_num; + unsigned int framerate_denom; + unsigned int h264_max_qp; + unsigned int h264_profile; + unsigned int h264_level; + unsigned int force_intra; +}; + +/** + * struct mtk_vcodec_pm - Power management data structure + */ +struct mtk_vcodec_pm { + struct clk *vcodecpll; + struct clk *univpll_d2; + struct clk *clk_cci400_sel; + struct clk *vdecpll; + struct clk *vdec_sel; + struct clk *vencpll_d2; + struct clk *venc_sel; + struct clk *univpll1_d2; + struct clk *venc_lt_sel; + struct device *larbvdec; + struct device *larbvenc; + struct device *larbvenclt; + struct device *dev; + struct mtk_vcodec_dev *mtkdev; +}; + +/** + * struct mtk_vcodec_ctx - Context (instance) private data. + * + * @type: type of the instance - decoder or encoder + * @dev: pointer to the mtk_vcodec_dev of the device + * @list: link to ctx_list of mtk_vcodec_dev + * @fh: struct v4l2_fh + * @m2m_ctx: pointer to the v4l2_m2m_ctx of the context + * @q_data: store information of input and output queue + * of the context + * @id: index of the context that this structure describes + * @state: state of the context + * @param_change: indicate encode parameter type + * @enc_params: encoding parameters + * @enc_if: hoooked encoder driver interface + * @drv_handle: driver handle for specific decode/encode instance + * + * @int_cond: variable used by the waitqueue + * @int_type: type of the last interrupt + * @queue: waitqueue that can be used to wait for this context to + * finish + * @irq_status: irq status + * + * @ctrl_hdl: handler for v4l2 framework + * @encode_work: worker for the encoding + * + * @colorspace: enum v4l2_colorspace; supplemental to pixelformat + * @ycbcr_enc: enum v4l2_ycbcr_encoding, Y'CbCr encoding + * @quantization: enum v4l2_quantization, colorspace quantization + * @xfer_func: enum v4l2_xfer_func, colorspace transfer function + */ +struct mtk_vcodec_ctx { + enum mtk_instance_type type; + struct mtk_vcodec_dev *dev; + struct list_head list; + + struct v4l2_fh fh; + struct v4l2_m2m_ctx *m2m_ctx; + struct mtk_q_data q_data[2]; + int id; + enum mtk_instance_state state; + enum mtk_encode_param param_change; + struct mtk_enc_params enc_params; + + struct venc_common_if *enc_if; + unsigned long drv_handle; + + int int_cond; + int int_type; + wait_queue_head_t queue; + unsigned int irq_status; + + struct v4l2_ctrl_handler ctrl_hdl; + struct work_struct encode_work; + + enum v4l2_colorspace colorspace; + enum v4l2_ycbcr_encoding ycbcr_enc; + enum v4l2_quantization quantization; + enum v4l2_xfer_func xfer_func; +}; + +/** + * struct mtk_vcodec_dev - driver data + * @v4l2_dev: V4L2 device to register video devices for. + * @vfd_enc: Video device for encoder. + * + * @m2m_dev_enc: m2m device for encoder. + * @plat_dev: platform device + * @vpu_plat_dev: mtk vpu platform device + * @ctx_list: list of struct mtk_vcodec_ctx + * @irqlock: protect data access by irq handler and work thread + * @curr_ctx: The context that is waiting for codec hardware + * + * @reg_base: Mapped address of MTK Vcodec registers. + * + * @id_counter: used to identify current opened instance + * @num_instances: counter of active MTK Vcodec instances + * + * @encode_workqueue: encode work queue + * + * @int_cond: used to identify interrupt condition happen + * @int_type: used to identify what kind of interrupt condition happen + * @dev_mutex: video_device lock + * @queue: waitqueue for waiting for completion of device commands + * + * @enc_irq: h264 encoder irq resource + * @enc_lt_irq: vp8 encoder irq resource + * + * @enc_mutex: encoder hardware lock. + * + * @pm: power management control + * @dec_capability: used to identify decode capability, ex: 4k + * @enc_capability: used to identify encode capability + */ +struct mtk_vcodec_dev { + struct v4l2_device v4l2_dev; + struct video_device *vfd_enc; + + struct v4l2_m2m_dev *m2m_dev_enc; + struct platform_device *plat_dev; + struct platform_device *vpu_plat_dev; + struct list_head ctx_list; + spinlock_t irqlock; + struct mtk_vcodec_ctx *curr_ctx; + void __iomem *reg_base[NUM_MAX_VCODEC_REG_BASE]; + + unsigned long id_counter; + int num_instances; + + struct workqueue_struct *encode_workqueue; + + int int_cond; + int int_type; + struct mutex dev_mutex; + wait_queue_head_t queue; + + int enc_irq; + int enc_lt_irq; + + struct mutex enc_mutex; + + struct mtk_vcodec_pm pm; + unsigned int dec_capability; + unsigned int enc_capability; +}; + +static inline struct mtk_vcodec_ctx *fh_to_ctx(struct v4l2_fh *fh) +{ + return container_of(fh, struct mtk_vcodec_ctx, fh); +} + +static inline struct mtk_vcodec_ctx *ctrl_to_ctx(struct v4l2_ctrl *ctrl) +{ + return container_of(ctrl->handler, struct mtk_vcodec_ctx, ctrl_hdl); +} + +#endif /* _MTK_VCODEC_DRV_H_ */ diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c new file mode 100644 index 000000000000..3ed3f2d31df5 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c @@ -0,0 +1,1292 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: PC Chen +* Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#include +#include +#include +#include + +#include "mtk_vcodec_drv.h" +#include "mtk_vcodec_enc.h" +#include "mtk_vcodec_intr.h" +#include "mtk_vcodec_util.h" +#include "venc_drv_if.h" + +#define MTK_VENC_MIN_W 160U +#define MTK_VENC_MIN_H 128U +#define MTK_VENC_MAX_W 1920U +#define MTK_VENC_MAX_H 1088U +#define DFT_CFG_WIDTH MTK_VENC_MIN_W +#define DFT_CFG_HEIGHT MTK_VENC_MIN_H +#define MTK_MAX_CTRLS_HINT 20 +#define OUT_FMT_IDX 0 +#define CAP_FMT_IDX 4 + + +static void mtk_venc_worker(struct work_struct *work); + +static struct mtk_video_fmt mtk_video_formats[] = { + { + .fourcc = V4L2_PIX_FMT_NV12M, + .type = MTK_FMT_FRAME, + .num_planes = 2, + }, + { + .fourcc = V4L2_PIX_FMT_NV21M, + .type = MTK_FMT_FRAME, + .num_planes = 2, + }, + { + .fourcc = V4L2_PIX_FMT_YUV420M, + .type = MTK_FMT_FRAME, + .num_planes = 3, + }, + { + .fourcc = V4L2_PIX_FMT_YVU420M, + .type = MTK_FMT_FRAME, + .num_planes = 3, + }, + { + .fourcc = V4L2_PIX_FMT_H264, + .type = MTK_FMT_ENC, + .num_planes = 1, + }, + { + .fourcc = V4L2_PIX_FMT_VP8, + .type = MTK_FMT_ENC, + .num_planes = 1, + }, +}; + +#define NUM_FORMATS ARRAY_SIZE(mtk_video_formats) + +static const struct mtk_codec_framesizes mtk_venc_framesizes[] = { + { + .fourcc = V4L2_PIX_FMT_H264, + .stepwise = { MTK_VENC_MIN_W, MTK_VENC_MAX_W, 16, + MTK_VENC_MIN_H, MTK_VENC_MAX_H, 16 }, + }, + { + .fourcc = V4L2_PIX_FMT_VP8, + .stepwise = { MTK_VENC_MIN_W, MTK_VENC_MAX_W, 16, + MTK_VENC_MIN_H, MTK_VENC_MAX_H, 16 }, + }, +}; + +#define NUM_SUPPORTED_FRAMESIZE ARRAY_SIZE(mtk_venc_framesizes) + +static int vidioc_venc_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct mtk_vcodec_ctx *ctx = ctrl_to_ctx(ctrl); + struct mtk_enc_params *p = &ctx->enc_params; + int ret = 0; + + switch (ctrl->id) { + case V4L2_CID_MPEG_VIDEO_BITRATE: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_BITRATE val = %d", + ctrl->val); + p->bitrate = ctrl->val; + ctx->param_change |= MTK_ENCODE_PARAM_BITRATE; + break; + case V4L2_CID_MPEG_VIDEO_B_FRAMES: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_B_FRAMES val = %d", + ctrl->val); + p->num_b_frame = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE val = %d", + ctrl->val); + p->rc_frame = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_MAX_QP: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_H264_MAX_QP val = %d", + ctrl->val); + p->h264_max_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEADER_MODE: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_HEADER_MODE val = %d", + ctrl->val); + p->seq_hdr_mode = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE val = %d", + ctrl->val); + p->rc_mb = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_PROFILE: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_H264_PROFILE val = %d", + ctrl->val); + p->h264_profile = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_LEVEL: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_H264_LEVEL val = %d", + ctrl->val); + p->h264_level = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_I_PERIOD: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_H264_I_PERIOD val = %d", + ctrl->val); + p->intra_period = ctrl->val; + ctx->param_change |= MTK_ENCODE_PARAM_INTRA_PERIOD; + break; + case V4L2_CID_MPEG_VIDEO_GOP_SIZE: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_GOP_SIZE val = %d", + ctrl->val); + p->gop_size = ctrl->val; + ctx->param_change |= MTK_ENCODE_PARAM_GOP_SIZE; + break; + case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME: + mtk_v4l2_debug(2, "V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME"); + p->force_intra = 1; + ctx->param_change |= MTK_ENCODE_PARAM_FORCE_INTRA; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static const struct v4l2_ctrl_ops mtk_vcodec_enc_ctrl_ops = { + .s_ctrl = vidioc_venc_s_ctrl, +}; + +static int vidioc_enum_fmt(struct v4l2_fmtdesc *f, bool output_queue) +{ + struct mtk_video_fmt *fmt; + int i, j = 0; + + for (i = 0; i < NUM_FORMATS; ++i) { + if (output_queue && mtk_video_formats[i].type != MTK_FMT_FRAME) + continue; + if (!output_queue && mtk_video_formats[i].type != MTK_FMT_ENC) + continue; + + if (j == f->index) { + fmt = &mtk_video_formats[i]; + f->pixelformat = fmt->fourcc; + memset(f->reserved, 0, sizeof(f->reserved)); + return 0; + } + ++j; + } + + return -EINVAL; +} + +static int vidioc_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *fsize) +{ + int i = 0; + + if (fsize->index != 0) + return -EINVAL; + + for (i = 0; i < NUM_SUPPORTED_FRAMESIZE; ++i) { + if (fsize->pixel_format != mtk_venc_framesizes[i].fourcc) + continue; + + fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE; + fsize->stepwise = mtk_venc_framesizes[i].stepwise; + return 0; + } + + return -EINVAL; +} + +static int vidioc_enum_fmt_vid_cap_mplane(struct file *file, void *pirv, + struct v4l2_fmtdesc *f) +{ + return vidioc_enum_fmt(f, false); +} + +static int vidioc_enum_fmt_vid_out_mplane(struct file *file, void *prov, + struct v4l2_fmtdesc *f) +{ + return vidioc_enum_fmt(f, true); +} + +static int vidioc_venc_querycap(struct file *file, void *priv, + struct v4l2_capability *cap) +{ + strlcpy(cap->driver, MTK_VCODEC_ENC_NAME, sizeof(cap->driver)); + strlcpy(cap->bus_info, MTK_PLATFORM_STR, sizeof(cap->bus_info)); + strlcpy(cap->card, MTK_PLATFORM_STR, sizeof(cap->card)); + + return 0; +} + +static int vidioc_venc_s_parm(struct file *file, void *priv, + struct v4l2_streamparm *a) +{ + struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv); + + if (a->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + return -EINVAL; + + ctx->enc_params.framerate_num = + a->parm.output.timeperframe.denominator; + ctx->enc_params.framerate_denom = + a->parm.output.timeperframe.numerator; + ctx->param_change |= MTK_ENCODE_PARAM_FRAMERATE; + + return 0; +} + +static int vidioc_venc_g_parm(struct file *file, void *priv, + struct v4l2_streamparm *a) +{ + struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv); + + if (a->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + return -EINVAL; + + a->parm.output.timeperframe.denominator = + ctx->enc_params.framerate_num; + a->parm.output.timeperframe.numerator = + ctx->enc_params.framerate_denom; + + return 0; +} + +static struct mtk_q_data *mtk_venc_get_q_data(struct mtk_vcodec_ctx *ctx, + enum v4l2_buf_type type) +{ + if (V4L2_TYPE_IS_OUTPUT(type)) + return &ctx->q_data[MTK_Q_DATA_SRC]; + + return &ctx->q_data[MTK_Q_DATA_DST]; +} + +static struct mtk_video_fmt *mtk_venc_find_format(struct v4l2_format *f) +{ + struct mtk_video_fmt *fmt; + unsigned int k; + + for (k = 0; k < NUM_FORMATS; k++) { + fmt = &mtk_video_formats[k]; + if (fmt->fourcc == f->fmt.pix.pixelformat) + return fmt; + } + + return NULL; +} + +/* V4L2 specification suggests the driver corrects the format struct if any of + * the dimensions is unsupported + */ +static int vidioc_try_fmt(struct v4l2_format *f, struct mtk_video_fmt *fmt) +{ + struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp; + int i; + + pix_fmt_mp->field = V4L2_FIELD_NONE; + + if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { + pix_fmt_mp->num_planes = 1; + pix_fmt_mp->plane_fmt[0].bytesperline = 0; + } else if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { + int tmp_w, tmp_h; + + pix_fmt_mp->height = clamp(pix_fmt_mp->height, + MTK_VENC_MIN_H, + MTK_VENC_MAX_H); + pix_fmt_mp->width = clamp(pix_fmt_mp->width, + MTK_VENC_MIN_W, + MTK_VENC_MAX_W); + + /* find next closer width align 16, heign align 32, size align + * 64 rectangle + */ + tmp_w = pix_fmt_mp->width; + tmp_h = pix_fmt_mp->height; + v4l_bound_align_image(&pix_fmt_mp->width, + MTK_VENC_MIN_W, + MTK_VENC_MAX_W, 4, + &pix_fmt_mp->height, + MTK_VENC_MIN_H, + MTK_VENC_MAX_H, 5, 6); + + if (pix_fmt_mp->width < tmp_w && + (pix_fmt_mp->width + 16) <= MTK_VENC_MAX_W) + pix_fmt_mp->width += 16; + if (pix_fmt_mp->height < tmp_h && + (pix_fmt_mp->height + 32) <= MTK_VENC_MAX_H) + pix_fmt_mp->height += 32; + + mtk_v4l2_debug(0, + "before resize width=%d, height=%d, after resize width=%d, height=%d, sizeimage=%d %d", + tmp_w, tmp_h, pix_fmt_mp->width, + pix_fmt_mp->height, + pix_fmt_mp->plane_fmt[0].sizeimage, + pix_fmt_mp->plane_fmt[1].sizeimage); + + pix_fmt_mp->num_planes = fmt->num_planes; + pix_fmt_mp->plane_fmt[0].sizeimage = + pix_fmt_mp->width * pix_fmt_mp->height + + ((ALIGN(pix_fmt_mp->width, 16) * 2) * 16); + pix_fmt_mp->plane_fmt[0].bytesperline = pix_fmt_mp->width; + + if (pix_fmt_mp->num_planes == 2) { + pix_fmt_mp->plane_fmt[1].sizeimage = + (pix_fmt_mp->width * pix_fmt_mp->height) / 2 + + (ALIGN(pix_fmt_mp->width, 16) * 16); + pix_fmt_mp->plane_fmt[2].sizeimage = 0; + pix_fmt_mp->plane_fmt[1].bytesperline = + pix_fmt_mp->width; + pix_fmt_mp->plane_fmt[2].bytesperline = 0; + } else if (pix_fmt_mp->num_planes == 3) { + pix_fmt_mp->plane_fmt[1].sizeimage = + pix_fmt_mp->plane_fmt[2].sizeimage = + (pix_fmt_mp->width * pix_fmt_mp->height) / 4 + + ((ALIGN(pix_fmt_mp->width, 16) / 2) * 16); + pix_fmt_mp->plane_fmt[1].bytesperline = + pix_fmt_mp->plane_fmt[2].bytesperline = + pix_fmt_mp->width / 2; + } + } + + for (i = 0; i < pix_fmt_mp->num_planes; i++) + memset(&(pix_fmt_mp->plane_fmt[i].reserved[0]), 0x0, + sizeof(pix_fmt_mp->plane_fmt[0].reserved)); + + pix_fmt_mp->flags = 0; + memset(&pix_fmt_mp->reserved, 0x0, + sizeof(pix_fmt_mp->reserved)); + + return 0; +} + +static void mtk_venc_set_param(struct mtk_vcodec_ctx *ctx, + struct venc_enc_param *param) +{ + struct mtk_q_data *q_data_src = &ctx->q_data[MTK_Q_DATA_SRC]; + struct mtk_enc_params *enc_params = &ctx->enc_params; + + switch (q_data_src->fmt->fourcc) { + case V4L2_PIX_FMT_YUV420M: + param->input_yuv_fmt = VENC_YUV_FORMAT_I420; + break; + case V4L2_PIX_FMT_YVU420M: + param->input_yuv_fmt = VENC_YUV_FORMAT_YV12; + break; + case V4L2_PIX_FMT_NV12M: + param->input_yuv_fmt = VENC_YUV_FORMAT_NV12; + break; + case V4L2_PIX_FMT_NV21M: + param->input_yuv_fmt = VENC_YUV_FORMAT_NV21; + break; + default: + mtk_v4l2_err("Unsupport fourcc =%d", q_data_src->fmt->fourcc); + break; + } + param->h264_profile = enc_params->h264_profile; + param->h264_level = enc_params->h264_level; + + /* Config visible resolution */ + param->width = q_data_src->visible_width; + param->height = q_data_src->visible_height; + /* Config coded resolution */ + param->buf_width = q_data_src->coded_width; + param->buf_height = q_data_src->coded_height; + param->frm_rate = enc_params->framerate_num / + enc_params->framerate_denom; + param->intra_period = enc_params->intra_period; + param->gop_size = enc_params->gop_size; + param->bitrate = enc_params->bitrate; + + mtk_v4l2_debug(0, + "fmt 0x%x, P/L %d/%d, w/h %d/%d, buf %d/%d, fps/bps %d/%d, gop %d, i_period %d", + param->input_yuv_fmt, param->h264_profile, + param->h264_level, param->width, param->height, + param->buf_width, param->buf_height, + param->frm_rate, param->bitrate, + param->gop_size, param->intra_period); +} + +static int vidioc_venc_s_fmt_cap(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv); + struct vb2_queue *vq; + struct mtk_q_data *q_data; + int i, ret; + struct mtk_video_fmt *fmt; + + vq = v4l2_m2m_get_vq(ctx->m2m_ctx, f->type); + if (!vq) { + mtk_v4l2_err("fail to get vq"); + return -EINVAL; + } + + if (vb2_is_busy(vq)) { + mtk_v4l2_err("queue busy"); + return -EBUSY; + } + + q_data = mtk_venc_get_q_data(ctx, f->type); + if (!q_data) { + mtk_v4l2_err("fail to get q data"); + return -EINVAL; + } + + fmt = mtk_venc_find_format(f); + if (!fmt) { + f->fmt.pix.pixelformat = mtk_video_formats[CAP_FMT_IDX].fourcc; + fmt = mtk_venc_find_format(f); + } + + q_data->fmt = fmt; + ret = vidioc_try_fmt(f, q_data->fmt); + if (ret) + return ret; + + q_data->coded_width = f->fmt.pix_mp.width; + q_data->coded_height = f->fmt.pix_mp.height; + q_data->field = f->fmt.pix_mp.field; + + for (i = 0; i < f->fmt.pix_mp.num_planes; i++) { + struct v4l2_plane_pix_format *plane_fmt; + + plane_fmt = &f->fmt.pix_mp.plane_fmt[i]; + q_data->bytesperline[i] = plane_fmt->bytesperline; + q_data->sizeimage[i] = plane_fmt->sizeimage; + } + + if (ctx->state == MTK_STATE_FREE) { + ret = venc_if_init(ctx, q_data->fmt->fourcc); + if (ret) { + mtk_v4l2_err("venc_if_init failed=%d, codec type=%x", + ret, q_data->fmt->fourcc); + return -EBUSY; + } + ctx->state = MTK_STATE_INIT; + } + + return 0; +} + +static int vidioc_venc_s_fmt_out(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv); + struct vb2_queue *vq; + struct mtk_q_data *q_data; + int ret, i; + struct mtk_video_fmt *fmt; + unsigned int pitch_w_div16; + struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp; + + vq = v4l2_m2m_get_vq(ctx->m2m_ctx, f->type); + if (!vq) { + mtk_v4l2_err("fail to get vq"); + return -EINVAL; + } + + if (vb2_is_busy(vq)) { + mtk_v4l2_err("queue busy"); + return -EBUSY; + } + + q_data = mtk_venc_get_q_data(ctx, f->type); + if (!q_data) { + mtk_v4l2_err("fail to get q data"); + return -EINVAL; + } + + fmt = mtk_venc_find_format(f); + if (!fmt) { + f->fmt.pix.pixelformat = mtk_video_formats[OUT_FMT_IDX].fourcc; + fmt = mtk_venc_find_format(f); + } + + pix_fmt_mp->height = clamp(pix_fmt_mp->height, + MTK_VENC_MIN_H, + MTK_VENC_MAX_H); + pix_fmt_mp->width = clamp(pix_fmt_mp->width, + MTK_VENC_MIN_W, + MTK_VENC_MAX_W); + + q_data->visible_width = f->fmt.pix_mp.width; + q_data->visible_height = f->fmt.pix_mp.height; + q_data->fmt = fmt; + ret = vidioc_try_fmt(f, q_data->fmt); + if (ret) + return ret; + + q_data->coded_width = f->fmt.pix_mp.width; + q_data->coded_height = f->fmt.pix_mp.height; + + pitch_w_div16 = DIV_ROUND_UP(q_data->visible_width, 16); + if (pitch_w_div16 % 8 != 0) { + /* Adjust returned width/height, so application could correctly + * allocate hw required memory + */ + q_data->visible_height += 32; + vidioc_try_fmt(f, q_data->fmt); + } + + q_data->field = f->fmt.pix_mp.field; + ctx->colorspace = f->fmt.pix_mp.colorspace; + ctx->ycbcr_enc = f->fmt.pix_mp.ycbcr_enc; + ctx->quantization = f->fmt.pix_mp.quantization; + ctx->xfer_func = f->fmt.pix_mp.xfer_func; + + for (i = 0; i < f->fmt.pix_mp.num_planes; i++) { + struct v4l2_plane_pix_format *plane_fmt; + + plane_fmt = &f->fmt.pix_mp.plane_fmt[i]; + q_data->bytesperline[i] = plane_fmt->bytesperline; + q_data->sizeimage[i] = plane_fmt->sizeimage; + } + + return 0; +} + +static int vidioc_venc_g_fmt(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; + struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv); + struct vb2_queue *vq; + struct mtk_q_data *q_data; + int i; + + vq = v4l2_m2m_get_vq(ctx->m2m_ctx, f->type); + if (!vq) + return -EINVAL; + + q_data = mtk_venc_get_q_data(ctx, f->type); + + pix->width = q_data->coded_width; + pix->height = q_data->coded_height; + pix->pixelformat = q_data->fmt->fourcc; + pix->field = q_data->field; + pix->num_planes = q_data->fmt->num_planes; + for (i = 0; i < pix->num_planes; i++) { + pix->plane_fmt[i].bytesperline = q_data->bytesperline[i]; + pix->plane_fmt[i].sizeimage = q_data->sizeimage[i]; + memset(&(pix->plane_fmt[i].reserved[0]), 0x0, + sizeof(pix->plane_fmt[i].reserved)); + } + + pix->flags = 0; + pix->colorspace = ctx->colorspace; + pix->ycbcr_enc = ctx->ycbcr_enc; + pix->quantization = ctx->quantization; + pix->xfer_func = ctx->xfer_func; + + return 0; +} + +static int vidioc_try_fmt_vid_cap_mplane(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct mtk_video_fmt *fmt; + struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv); + + fmt = mtk_venc_find_format(f); + if (!fmt) { + f->fmt.pix.pixelformat = mtk_video_formats[CAP_FMT_IDX].fourcc; + fmt = mtk_venc_find_format(f); + } + f->fmt.pix_mp.colorspace = ctx->colorspace; + f->fmt.pix_mp.ycbcr_enc = ctx->ycbcr_enc; + f->fmt.pix_mp.quantization = ctx->quantization; + f->fmt.pix_mp.xfer_func = ctx->xfer_func; + + return vidioc_try_fmt(f, fmt); +} + +static int vidioc_try_fmt_vid_out_mplane(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct mtk_video_fmt *fmt; + + fmt = mtk_venc_find_format(f); + if (!fmt) { + f->fmt.pix.pixelformat = mtk_video_formats[OUT_FMT_IDX].fourcc; + fmt = mtk_venc_find_format(f); + } + if (!f->fmt.pix_mp.colorspace) { + f->fmt.pix_mp.colorspace = V4L2_COLORSPACE_REC709; + f->fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + f->fmt.pix_mp.quantization = V4L2_QUANTIZATION_DEFAULT; + f->fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT; + } + + return vidioc_try_fmt(f, fmt); +} + +static int vidioc_venc_qbuf(struct file *file, void *priv, + struct v4l2_buffer *buf) +{ + struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv); + + if (ctx->state == MTK_STATE_ABORT) { + mtk_v4l2_err("[%d] Call on QBUF after unrecoverable error", + ctx->id); + return -EIO; + } + + return v4l2_m2m_qbuf(file, ctx->m2m_ctx, buf); +} + +static int vidioc_venc_dqbuf(struct file *file, void *priv, + struct v4l2_buffer *buf) +{ + struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv); + + if (ctx->state == MTK_STATE_ABORT) { + mtk_v4l2_err("[%d] Call on QBUF after unrecoverable error", + ctx->id); + return -EIO; + } + + return v4l2_m2m_dqbuf(file, ctx->m2m_ctx, buf); +} + +const struct v4l2_ioctl_ops mtk_venc_ioctl_ops = { + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, + + .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs, + .vidioc_querybuf = v4l2_m2m_ioctl_querybuf, + .vidioc_qbuf = vidioc_venc_qbuf, + .vidioc_dqbuf = vidioc_venc_dqbuf, + + .vidioc_querycap = vidioc_venc_querycap, + .vidioc_enum_fmt_vid_cap_mplane = vidioc_enum_fmt_vid_cap_mplane, + .vidioc_enum_fmt_vid_out_mplane = vidioc_enum_fmt_vid_out_mplane, + .vidioc_enum_framesizes = vidioc_enum_framesizes, + + .vidioc_try_fmt_vid_cap_mplane = vidioc_try_fmt_vid_cap_mplane, + .vidioc_try_fmt_vid_out_mplane = vidioc_try_fmt_vid_out_mplane, + .vidioc_expbuf = v4l2_m2m_ioctl_expbuf, + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, + + .vidioc_s_parm = vidioc_venc_s_parm, + .vidioc_g_parm = vidioc_venc_g_parm, + .vidioc_s_fmt_vid_cap_mplane = vidioc_venc_s_fmt_cap, + .vidioc_s_fmt_vid_out_mplane = vidioc_venc_s_fmt_out, + + .vidioc_g_fmt_vid_cap_mplane = vidioc_venc_g_fmt, + .vidioc_g_fmt_vid_out_mplane = vidioc_venc_g_fmt, + + .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, + .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf, +}; + +static int vb2ops_venc_queue_setup(struct vb2_queue *vq, + unsigned int *nbuffers, + unsigned int *nplanes, + unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct mtk_vcodec_ctx *ctx = vb2_get_drv_priv(vq); + struct mtk_q_data *q_data; + unsigned int i; + + q_data = mtk_venc_get_q_data(ctx, vq->type); + + if (q_data == NULL) + return -EINVAL; + + if (*nplanes) { + for (i = 0; i < *nplanes; i++) + if (sizes[i] < q_data->sizeimage[i]) + return -EINVAL; + } else { + *nplanes = q_data->fmt->num_planes; + for (i = 0; i < *nplanes; i++) + sizes[i] = q_data->sizeimage[i]; + } + + return 0; +} + +static int vb2ops_venc_buf_prepare(struct vb2_buffer *vb) +{ + struct mtk_vcodec_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + struct mtk_q_data *q_data; + int i; + + q_data = mtk_venc_get_q_data(ctx, vb->vb2_queue->type); + + for (i = 0; i < q_data->fmt->num_planes; i++) { + if (vb2_plane_size(vb, i) < q_data->sizeimage[i]) { + mtk_v4l2_err("data will not fit into plane %d (%lu < %d)", + i, vb2_plane_size(vb, i), + q_data->sizeimage[i]); + return -EINVAL; + } + } + + return 0; +} + +static void vb2ops_venc_buf_queue(struct vb2_buffer *vb) +{ + struct mtk_vcodec_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + struct vb2_v4l2_buffer *vb2_v4l2 = + container_of(vb, struct vb2_v4l2_buffer, vb2_buf); + + struct mtk_video_enc_buf *mtk_buf = + container_of(vb2_v4l2, struct mtk_video_enc_buf, vb); + + if ((vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) && + (ctx->param_change != MTK_ENCODE_PARAM_NONE)) { + mtk_v4l2_debug(1, "[%d] Before id=%d encode parameter change %x", + ctx->id, + mtk_buf->vb.vb2_buf.index, + ctx->param_change); + mtk_buf->param_change = ctx->param_change; + mtk_buf->enc_params = ctx->enc_params; + ctx->param_change = MTK_ENCODE_PARAM_NONE; + } + + v4l2_m2m_buf_queue(ctx->m2m_ctx, to_vb2_v4l2_buffer(vb)); +} + +static int vb2ops_venc_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct mtk_vcodec_ctx *ctx = vb2_get_drv_priv(q); + struct venc_enc_param param; + int ret; + int i; + + /* Once state turn into MTK_STATE_ABORT, we need stop_streaming + * to clear it + */ + if ((ctx->state == MTK_STATE_ABORT) || (ctx->state == MTK_STATE_FREE)) { + ret = -EIO; + goto err_set_param; + } + + /* Do the initialization when both start_streaming have been called */ + if (V4L2_TYPE_IS_OUTPUT(q->type)) { + if (!vb2_start_streaming_called(&ctx->m2m_ctx->cap_q_ctx.q)) + return 0; + } else { + if (!vb2_start_streaming_called(&ctx->m2m_ctx->out_q_ctx.q)) + return 0; + } + + mtk_venc_set_param(ctx, ¶m); + ret = venc_if_set_param(ctx, VENC_SET_PARAM_ENC, ¶m); + if (ret) { + mtk_v4l2_err("venc_if_set_param failed=%d", ret); + ctx->state = MTK_STATE_ABORT; + goto err_set_param; + } + ctx->param_change = MTK_ENCODE_PARAM_NONE; + + if ((ctx->q_data[MTK_Q_DATA_DST].fmt->fourcc == V4L2_PIX_FMT_H264) && + (ctx->enc_params.seq_hdr_mode != + V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE)) { + ret = venc_if_set_param(ctx, + VENC_SET_PARAM_PREPEND_HEADER, + NULL); + if (ret) { + mtk_v4l2_err("venc_if_set_param failed=%d", ret); + ctx->state = MTK_STATE_ABORT; + goto err_set_param; + } + ctx->state = MTK_STATE_HEADER; + } + + return 0; + +err_set_param: + for (i = 0; i < q->num_buffers; ++i) { + if (q->bufs[i]->state == VB2_BUF_STATE_ACTIVE) { + mtk_v4l2_debug(0, "[%d] id=%d, type=%d, %d -> VB2_BUF_STATE_QUEUED", + ctx->id, i, q->type, + (int)q->bufs[i]->state); + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(q->bufs[i]), + VB2_BUF_STATE_QUEUED); + } + } + + return ret; +} + +static void vb2ops_venc_stop_streaming(struct vb2_queue *q) +{ + struct mtk_vcodec_ctx *ctx = vb2_get_drv_priv(q); + struct vb2_buffer *src_buf, *dst_buf; + int ret; + + mtk_v4l2_debug(2, "[%d]-> type=%d", ctx->id, q->type); + + if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { + while ((dst_buf = v4l2_m2m_dst_buf_remove(ctx->m2m_ctx))) { + dst_buf->planes[0].bytesused = 0; + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(dst_buf), + VB2_BUF_STATE_ERROR); + } + } else { + while ((src_buf = v4l2_m2m_src_buf_remove(ctx->m2m_ctx))) + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(src_buf), + VB2_BUF_STATE_ERROR); + } + + if ((q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE && + vb2_is_streaming(&ctx->m2m_ctx->out_q_ctx.q)) || + (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && + vb2_is_streaming(&ctx->m2m_ctx->cap_q_ctx.q))) { + mtk_v4l2_debug(1, "[%d]-> q type %d out=%d cap=%d", + ctx->id, q->type, + vb2_is_streaming(&ctx->m2m_ctx->out_q_ctx.q), + vb2_is_streaming(&ctx->m2m_ctx->cap_q_ctx.q)); + return; + } + + /* Release the encoder if both streams are stopped. */ + ret = venc_if_deinit(ctx); + if (ret) + mtk_v4l2_err("venc_if_deinit failed=%d", ret); + + ctx->state = MTK_STATE_FREE; +} + +static struct vb2_ops mtk_venc_vb2_ops = { + .queue_setup = vb2ops_venc_queue_setup, + .buf_prepare = vb2ops_venc_buf_prepare, + .buf_queue = vb2ops_venc_buf_queue, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, + .start_streaming = vb2ops_venc_start_streaming, + .stop_streaming = vb2ops_venc_stop_streaming, +}; + +static int mtk_venc_encode_header(void *priv) +{ + struct mtk_vcodec_ctx *ctx = priv; + int ret; + struct vb2_buffer *dst_buf; + struct mtk_vcodec_mem bs_buf; + struct venc_done_result enc_result; + + dst_buf = v4l2_m2m_dst_buf_remove(ctx->m2m_ctx); + if (!dst_buf) { + mtk_v4l2_debug(1, "No dst buffer"); + return -EINVAL; + } + + bs_buf.va = vb2_plane_vaddr(dst_buf, 0); + bs_buf.dma_addr = vb2_dma_contig_plane_dma_addr(dst_buf, 0); + bs_buf.size = (size_t)dst_buf->planes[0].length; + + mtk_v4l2_debug(1, + "[%d] buf id=%d va=0x%p dma_addr=0x%llx size=%zu", + ctx->id, + dst_buf->index, bs_buf.va, + (u64)bs_buf.dma_addr, + bs_buf.size); + + ret = venc_if_encode(ctx, + VENC_START_OPT_ENCODE_SEQUENCE_HEADER, + NULL, &bs_buf, &enc_result); + + if (ret) { + dst_buf->planes[0].bytesused = 0; + ctx->state = MTK_STATE_ABORT; + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(dst_buf), + VB2_BUF_STATE_ERROR); + mtk_v4l2_err("venc_if_encode failed=%d", ret); + return -EINVAL; + } + + ctx->state = MTK_STATE_HEADER; + dst_buf->planes[0].bytesused = enc_result.bs_size; + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(dst_buf), VB2_BUF_STATE_DONE); + + return 0; +} + +static int mtk_venc_param_change(struct mtk_vcodec_ctx *ctx) +{ + struct venc_enc_param enc_prm; + struct vb2_buffer *vb = v4l2_m2m_next_src_buf(ctx->m2m_ctx); + struct vb2_v4l2_buffer *vb2_v4l2 = + container_of(vb, struct vb2_v4l2_buffer, vb2_buf); + struct mtk_video_enc_buf *mtk_buf = + container_of(vb2_v4l2, struct mtk_video_enc_buf, vb); + + int ret = 0; + + memset(&enc_prm, 0, sizeof(enc_prm)); + if (mtk_buf->param_change == MTK_ENCODE_PARAM_NONE) + return 0; + + if (mtk_buf->param_change & MTK_ENCODE_PARAM_BITRATE) { + enc_prm.bitrate = mtk_buf->enc_params.bitrate; + mtk_v4l2_debug(1, "[%d] id=%d, change param br=%d", + ctx->id, + mtk_buf->vb.vb2_buf.index, + enc_prm.bitrate); + ret |= venc_if_set_param(ctx, + VENC_SET_PARAM_ADJUST_BITRATE, + &enc_prm); + } + if (!ret && mtk_buf->param_change & MTK_ENCODE_PARAM_FRAMERATE) { + enc_prm.frm_rate = mtk_buf->enc_params.framerate_num / + mtk_buf->enc_params.framerate_denom; + mtk_v4l2_debug(1, "[%d] id=%d, change param fr=%d", + ctx->id, + mtk_buf->vb.vb2_buf.index, + enc_prm.frm_rate); + ret |= venc_if_set_param(ctx, + VENC_SET_PARAM_ADJUST_FRAMERATE, + &enc_prm); + } + if (!ret && mtk_buf->param_change & MTK_ENCODE_PARAM_GOP_SIZE) { + enc_prm.gop_size = mtk_buf->enc_params.gop_size; + mtk_v4l2_debug(1, "change param intra period=%d", + enc_prm.gop_size); + ret |= venc_if_set_param(ctx, + VENC_SET_PARAM_GOP_SIZE, + &enc_prm); + } + if (!ret && mtk_buf->param_change & MTK_ENCODE_PARAM_FORCE_INTRA) { + mtk_v4l2_debug(1, "[%d] id=%d, change param force I=%d", + ctx->id, + mtk_buf->vb.vb2_buf.index, + mtk_buf->enc_params.force_intra); + if (mtk_buf->enc_params.force_intra) + ret |= venc_if_set_param(ctx, + VENC_SET_PARAM_FORCE_INTRA, + NULL); + } + + mtk_buf->param_change = MTK_ENCODE_PARAM_NONE; + + if (ret) { + ctx->state = MTK_STATE_ABORT; + mtk_v4l2_err("venc_if_set_param %d failed=%d", + mtk_buf->param_change, ret); + return -1; + } + + return 0; +} + +/* + * v4l2_m2m_streamoff() holds dev_mutex and waits mtk_venc_worker() + * to call v4l2_m2m_job_finish(). + * If mtk_venc_worker() tries to acquire dev_mutex, it will deadlock. + * So this function must not try to acquire dev->dev_mutex. + * This means v4l2 ioctls and mtk_venc_worker() can run at the same time. + * mtk_venc_worker() should be carefully implemented to avoid bugs. + */ +static void mtk_venc_worker(struct work_struct *work) +{ + struct mtk_vcodec_ctx *ctx = container_of(work, struct mtk_vcodec_ctx, + encode_work); + struct vb2_buffer *src_buf, *dst_buf; + struct venc_frm_buf frm_buf; + struct mtk_vcodec_mem bs_buf; + struct venc_done_result enc_result; + int ret, i; + struct vb2_v4l2_buffer *vb2_v4l2; + + /* check dst_buf, dst_buf may be removed in device_run + * to stored encdoe header so we need check dst_buf and + * call job_finish here to prevent recursion + */ + dst_buf = v4l2_m2m_dst_buf_remove(ctx->m2m_ctx); + if (!dst_buf) { + v4l2_m2m_job_finish(ctx->dev->m2m_dev_enc, ctx->m2m_ctx); + return; + } + + src_buf = v4l2_m2m_src_buf_remove(ctx->m2m_ctx); + memset(&frm_buf, 0, sizeof(frm_buf)); + for (i = 0; i < src_buf->num_planes ; i++) { + frm_buf.fb_addr[i].va = vb2_plane_vaddr(src_buf, i); + frm_buf.fb_addr[i].dma_addr = + vb2_dma_contig_plane_dma_addr(src_buf, i); + frm_buf.fb_addr[i].size = + (size_t)src_buf->planes[i].length; + } + bs_buf.va = vb2_plane_vaddr(dst_buf, 0); + bs_buf.dma_addr = vb2_dma_contig_plane_dma_addr(dst_buf, 0); + bs_buf.size = (size_t)dst_buf->planes[0].length; + + mtk_v4l2_debug(2, + "Framebuf VA=%p PA=%llx Size=0x%zx;VA=%p PA=0x%llx Size=0x%zx;VA=%p PA=0x%llx Size=%zu", + frm_buf.fb_addr[0].va, + (u64)frm_buf.fb_addr[0].dma_addr, + frm_buf.fb_addr[0].size, + frm_buf.fb_addr[1].va, + (u64)frm_buf.fb_addr[1].dma_addr, + frm_buf.fb_addr[1].size, + frm_buf.fb_addr[2].va, + (u64)frm_buf.fb_addr[2].dma_addr, + frm_buf.fb_addr[2].size); + + ret = venc_if_encode(ctx, VENC_START_OPT_ENCODE_FRAME, + &frm_buf, &bs_buf, &enc_result); + + vb2_v4l2 = container_of(dst_buf, struct vb2_v4l2_buffer, vb2_buf); + if (enc_result.is_key_frm) + vb2_v4l2->flags |= V4L2_BUF_FLAG_KEYFRAME; + + if (ret) { + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(src_buf), + VB2_BUF_STATE_ERROR); + dst_buf->planes[0].bytesused = 0; + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(dst_buf), + VB2_BUF_STATE_ERROR); + mtk_v4l2_err("venc_if_encode failed=%d", ret); + } else { + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(src_buf), + VB2_BUF_STATE_DONE); + dst_buf->planes[0].bytesused = enc_result.bs_size; + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(dst_buf), + VB2_BUF_STATE_DONE); + mtk_v4l2_debug(2, "venc_if_encode bs size=%d", + enc_result.bs_size); + } + + v4l2_m2m_job_finish(ctx->dev->m2m_dev_enc, ctx->m2m_ctx); + + mtk_v4l2_debug(1, "<=== src_buf[%d] dst_buf[%d] venc_if_encode ret=%d Size=%u===>", + src_buf->index, dst_buf->index, ret, + enc_result.bs_size); +} + +static void m2mops_venc_device_run(void *priv) +{ + struct mtk_vcodec_ctx *ctx = priv; + + if ((ctx->q_data[MTK_Q_DATA_DST].fmt->fourcc == V4L2_PIX_FMT_H264) && + (ctx->state != MTK_STATE_HEADER)) { + /* encode h264 sps/pps header */ + mtk_venc_encode_header(ctx); + queue_work(ctx->dev->encode_workqueue, &ctx->encode_work); + return; + } + + mtk_venc_param_change(ctx); + queue_work(ctx->dev->encode_workqueue, &ctx->encode_work); +} + +static int m2mops_venc_job_ready(void *m2m_priv) +{ + struct mtk_vcodec_ctx *ctx = m2m_priv; + + if (ctx->state == MTK_STATE_ABORT || ctx->state == MTK_STATE_FREE) { + mtk_v4l2_debug(3, "[%d]Not ready: state=0x%x.", + ctx->id, ctx->state); + return 0; + } + + return 1; +} + +static void m2mops_venc_job_abort(void *priv) +{ + struct mtk_vcodec_ctx *ctx = priv; + + ctx->state = MTK_STATE_ABORT; +} + +static void m2mops_venc_lock(void *m2m_priv) +{ + struct mtk_vcodec_ctx *ctx = m2m_priv; + + mutex_lock(&ctx->dev->dev_mutex); +} + +static void m2mops_venc_unlock(void *m2m_priv) +{ + struct mtk_vcodec_ctx *ctx = m2m_priv; + + mutex_unlock(&ctx->dev->dev_mutex); +} + +const struct v4l2_m2m_ops mtk_venc_m2m_ops = { + .device_run = m2mops_venc_device_run, + .job_ready = m2mops_venc_job_ready, + .job_abort = m2mops_venc_job_abort, + .lock = m2mops_venc_lock, + .unlock = m2mops_venc_unlock, +}; + +void mtk_vcodec_enc_set_default_params(struct mtk_vcodec_ctx *ctx) +{ + struct mtk_q_data *q_data; + + ctx->m2m_ctx->q_lock = &ctx->dev->dev_mutex; + ctx->fh.m2m_ctx = ctx->m2m_ctx; + ctx->fh.ctrl_handler = &ctx->ctrl_hdl; + INIT_WORK(&ctx->encode_work, mtk_venc_worker); + + ctx->colorspace = V4L2_COLORSPACE_REC709; + ctx->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + ctx->quantization = V4L2_QUANTIZATION_DEFAULT; + ctx->xfer_func = V4L2_XFER_FUNC_DEFAULT; + + q_data = &ctx->q_data[MTK_Q_DATA_SRC]; + memset(q_data, 0, sizeof(struct mtk_q_data)); + q_data->visible_width = DFT_CFG_WIDTH; + q_data->visible_height = DFT_CFG_HEIGHT; + q_data->coded_width = DFT_CFG_WIDTH; + q_data->coded_height = DFT_CFG_HEIGHT; + q_data->field = V4L2_FIELD_NONE; + + q_data->fmt = &mtk_video_formats[OUT_FMT_IDX]; + + v4l_bound_align_image(&q_data->coded_width, + MTK_VENC_MIN_W, + MTK_VENC_MAX_W, 4, + &q_data->coded_height, + MTK_VENC_MIN_H, + MTK_VENC_MAX_H, 5, 6); + + if (q_data->coded_width < DFT_CFG_WIDTH && + (q_data->coded_width + 16) <= MTK_VENC_MAX_W) + q_data->coded_width += 16; + if (q_data->coded_height < DFT_CFG_HEIGHT && + (q_data->coded_height + 32) <= MTK_VENC_MAX_H) + q_data->coded_height += 32; + + q_data->sizeimage[0] = + q_data->coded_width * q_data->coded_height+ + ((ALIGN(q_data->coded_width, 16) * 2) * 16); + q_data->bytesperline[0] = q_data->coded_width; + q_data->sizeimage[1] = + (q_data->coded_width * q_data->coded_height) / 2 + + (ALIGN(q_data->coded_width, 16) * 16); + q_data->bytesperline[1] = q_data->coded_width; + + q_data = &ctx->q_data[MTK_Q_DATA_DST]; + memset(q_data, 0, sizeof(struct mtk_q_data)); + q_data->coded_width = DFT_CFG_WIDTH; + q_data->coded_height = DFT_CFG_HEIGHT; + q_data->fmt = &mtk_video_formats[CAP_FMT_IDX]; + q_data->field = V4L2_FIELD_NONE; + ctx->q_data[MTK_Q_DATA_DST].sizeimage[0] = + DFT_CFG_WIDTH * DFT_CFG_HEIGHT; + ctx->q_data[MTK_Q_DATA_DST].bytesperline[0] = 0; + +} + +int mtk_vcodec_enc_ctrls_setup(struct mtk_vcodec_ctx *ctx) +{ + const struct v4l2_ctrl_ops *ops = &mtk_vcodec_enc_ctrl_ops; + struct v4l2_ctrl_handler *handler = &ctx->ctrl_hdl; + + v4l2_ctrl_handler_init(handler, MTK_MAX_CTRLS_HINT); + + v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_BITRATE, + 1, 4000000, 1, 4000000); + v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_B_FRAMES, + 0, 2, 1, 0); + v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE, + 0, 1, 1, 1); + v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_H264_MAX_QP, + 0, 51, 1, 51); + v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_H264_I_PERIOD, + 0, 65535, 1, 0); + v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE, + 0, 65535, 1, 0); + v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE, + 0, 1, 1, 0); + v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, + 0, 0, 0, 0); + v4l2_ctrl_new_std_menu(handler, ops, + V4L2_CID_MPEG_VIDEO_HEADER_MODE, + V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, + 0, V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE); + v4l2_ctrl_new_std_menu(handler, ops, V4L2_CID_MPEG_VIDEO_H264_PROFILE, + V4L2_MPEG_VIDEO_H264_PROFILE_HIGH, + 0, V4L2_MPEG_VIDEO_H264_PROFILE_MAIN); + v4l2_ctrl_new_std_menu(handler, ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, + V4L2_MPEG_VIDEO_H264_LEVEL_4_2, + 0, V4L2_MPEG_VIDEO_H264_LEVEL_4_0); + if (handler->error) { + mtk_v4l2_err("Init control handler fail %d", + handler->error); + return handler->error; + } + + v4l2_ctrl_handler_setup(&ctx->ctrl_hdl); + + return 0; +} + +int mtk_vcodec_enc_queue_init(void *priv, struct vb2_queue *src_vq, + struct vb2_queue *dst_vq) +{ + struct mtk_vcodec_ctx *ctx = priv; + int ret; + + /* Note: VB2_USERPTR works with dma-contig because mt8173 + * support iommu + * https://patchwork.kernel.org/patch/8335461/ + * https://patchwork.kernel.org/patch/7596181/ + */ + src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + src_vq->io_modes = VB2_DMABUF | VB2_MMAP | VB2_USERPTR; + src_vq->drv_priv = ctx; + src_vq->buf_struct_size = sizeof(struct mtk_video_enc_buf); + src_vq->ops = &mtk_venc_vb2_ops; + src_vq->mem_ops = &vb2_dma_contig_memops; + src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + src_vq->lock = &ctx->dev->dev_mutex; + src_vq->dev = &ctx->dev->plat_dev->dev; + + ret = vb2_queue_init(src_vq); + if (ret) + return ret; + + dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + dst_vq->io_modes = VB2_DMABUF | VB2_MMAP | VB2_USERPTR; + dst_vq->drv_priv = ctx; + dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + dst_vq->ops = &mtk_venc_vb2_ops; + dst_vq->mem_ops = &vb2_dma_contig_memops; + dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + dst_vq->lock = &ctx->dev->dev_mutex; + dst_vq->dev = &ctx->dev->plat_dev->dev; + + return vb2_queue_init(dst_vq); +} + +int mtk_venc_unlock(struct mtk_vcodec_ctx *ctx) +{ + struct mtk_vcodec_dev *dev = ctx->dev; + + mutex_unlock(&dev->enc_mutex); + return 0; +} + +int mtk_venc_lock(struct mtk_vcodec_ctx *ctx) +{ + struct mtk_vcodec_dev *dev = ctx->dev; + + mutex_lock(&dev->enc_mutex); + return 0; +} + +void mtk_vcodec_enc_release(struct mtk_vcodec_ctx *ctx) +{ + venc_if_deinit(ctx); +} diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h new file mode 100644 index 000000000000..d7a154a97510 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h @@ -0,0 +1,58 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: PC Chen +* Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#ifndef _MTK_VCODEC_ENC_H_ +#define _MTK_VCODEC_ENC_H_ + +#include +#include + +#define MTK_VENC_IRQ_STATUS_SPS 0x1 +#define MTK_VENC_IRQ_STATUS_PPS 0x2 +#define MTK_VENC_IRQ_STATUS_FRM 0x4 +#define MTK_VENC_IRQ_STATUS_DRAM 0x8 +#define MTK_VENC_IRQ_STATUS_PAUSE 0x10 +#define MTK_VENC_IRQ_STATUS_SWITCH 0x20 + +#define MTK_VENC_IRQ_STATUS_OFFSET 0x05C +#define MTK_VENC_IRQ_ACK_OFFSET 0x060 + +/** + * struct mtk_video_enc_buf - Private data related to each VB2 buffer. + * @vb: Pointer to related VB2 buffer. + * @list: list that buffer link to + * @param_change: Types of encode parameter change before encoding this + * buffer + * @enc_params: Encode parameters changed before encode this buffer + */ +struct mtk_video_enc_buf { + struct vb2_v4l2_buffer vb; + struct list_head list; + u32 param_change; + struct mtk_enc_params enc_params; +}; + +extern const struct v4l2_ioctl_ops mtk_venc_ioctl_ops; +extern const struct v4l2_m2m_ops mtk_venc_m2m_ops; + +int mtk_venc_unlock(struct mtk_vcodec_ctx *ctx); +int mtk_venc_lock(struct mtk_vcodec_ctx *ctx); +int mtk_vcodec_enc_queue_init(void *priv, struct vb2_queue *src_vq, + struct vb2_queue *dst_vq); +void mtk_vcodec_enc_release(struct mtk_vcodec_ctx *ctx); +int mtk_vcodec_enc_ctrls_setup(struct mtk_vcodec_ctx *ctx); +void mtk_vcodec_enc_set_default_params(struct mtk_vcodec_ctx *ctx); + +#endif /* _MTK_VCODEC_ENC_H_ */ diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c new file mode 100644 index 000000000000..e277b7c23516 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c @@ -0,0 +1,439 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: PC Chen +* Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mtk_vcodec_drv.h" +#include "mtk_vcodec_enc.h" +#include "mtk_vcodec_enc_pm.h" +#include "mtk_vcodec_intr.h" +#include "mtk_vcodec_util.h" +#include "mtk_vpu.h" + +module_param(mtk_v4l2_dbg_level, int, S_IRUGO | S_IWUSR); +module_param(mtk_vcodec_dbg, bool, S_IRUGO | S_IWUSR); + +/* Wake up context wait_queue */ +static void wake_up_ctx(struct mtk_vcodec_ctx *ctx, unsigned int reason) +{ + ctx->int_cond = 1; + ctx->int_type = reason; + wake_up_interruptible(&ctx->queue); +} + +static void clean_irq_status(unsigned int irq_status, void __iomem *addr) +{ + if (irq_status & MTK_VENC_IRQ_STATUS_PAUSE) + writel(MTK_VENC_IRQ_STATUS_PAUSE, addr); + + if (irq_status & MTK_VENC_IRQ_STATUS_SWITCH) + writel(MTK_VENC_IRQ_STATUS_SWITCH, addr); + + if (irq_status & MTK_VENC_IRQ_STATUS_DRAM) + writel(MTK_VENC_IRQ_STATUS_DRAM, addr); + + if (irq_status & MTK_VENC_IRQ_STATUS_SPS) + writel(MTK_VENC_IRQ_STATUS_SPS, addr); + + if (irq_status & MTK_VENC_IRQ_STATUS_PPS) + writel(MTK_VENC_IRQ_STATUS_PPS, addr); + + if (irq_status & MTK_VENC_IRQ_STATUS_FRM) + writel(MTK_VENC_IRQ_STATUS_FRM, addr); + +} +static irqreturn_t mtk_vcodec_enc_irq_handler(int irq, void *priv) +{ + struct mtk_vcodec_dev *dev = priv; + struct mtk_vcodec_ctx *ctx; + unsigned long flags; + void __iomem *addr; + + spin_lock_irqsave(&dev->irqlock, flags); + ctx = dev->curr_ctx; + spin_unlock_irqrestore(&dev->irqlock, flags); + + mtk_v4l2_debug(1, "id=%d", ctx->id); + addr = dev->reg_base[VENC_SYS] + MTK_VENC_IRQ_ACK_OFFSET; + + ctx->irq_status = readl(dev->reg_base[VENC_SYS] + + (MTK_VENC_IRQ_STATUS_OFFSET)); + + clean_irq_status(ctx->irq_status, addr); + + wake_up_ctx(ctx, MTK_INST_IRQ_RECEIVED); + return IRQ_HANDLED; +} + +static irqreturn_t mtk_vcodec_enc_lt_irq_handler(int irq, void *priv) +{ + struct mtk_vcodec_dev *dev = priv; + struct mtk_vcodec_ctx *ctx; + unsigned long flags; + void __iomem *addr; + + spin_lock_irqsave(&dev->irqlock, flags); + ctx = dev->curr_ctx; + spin_unlock_irqrestore(&dev->irqlock, flags); + + mtk_v4l2_debug(1, "id=%d", ctx->id); + ctx->irq_status = readl(dev->reg_base[VENC_LT_SYS] + + (MTK_VENC_IRQ_STATUS_OFFSET)); + + addr = dev->reg_base[VENC_LT_SYS] + MTK_VENC_IRQ_ACK_OFFSET; + + clean_irq_status(ctx->irq_status, addr); + + wake_up_ctx(ctx, MTK_INST_IRQ_RECEIVED); + return IRQ_HANDLED; +} + +static void mtk_vcodec_enc_reset_handler(void *priv) +{ + struct mtk_vcodec_dev *dev = priv; + struct mtk_vcodec_ctx *ctx; + + mtk_v4l2_debug(0, "Watchdog timeout!!"); + + mutex_lock(&dev->dev_mutex); + list_for_each_entry(ctx, &dev->ctx_list, list) { + ctx->state = MTK_STATE_ABORT; + mtk_v4l2_debug(0, "[%d] Change to state MTK_STATE_ABORT", + ctx->id); + } + mutex_unlock(&dev->dev_mutex); +} + +static int fops_vcodec_open(struct file *file) +{ + struct mtk_vcodec_dev *dev = video_drvdata(file); + struct mtk_vcodec_ctx *ctx = NULL; + int ret = 0; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + mutex_lock(&dev->dev_mutex); + /* + * Use simple counter to uniquely identify this context. Only + * used for logging. + */ + ctx->id = dev->id_counter++; + v4l2_fh_init(&ctx->fh, video_devdata(file)); + file->private_data = &ctx->fh; + v4l2_fh_add(&ctx->fh); + INIT_LIST_HEAD(&ctx->list); + ctx->dev = dev; + init_waitqueue_head(&ctx->queue); + + ctx->type = MTK_INST_ENCODER; + ret = mtk_vcodec_enc_ctrls_setup(ctx); + if (ret) { + mtk_v4l2_err("Failed to setup controls() (%d)", + ret); + goto err_ctrls_setup; + } + ctx->m2m_ctx = v4l2_m2m_ctx_init(dev->m2m_dev_enc, ctx, + &mtk_vcodec_enc_queue_init); + if (IS_ERR((__force void *)ctx->m2m_ctx)) { + ret = PTR_ERR((__force void *)ctx->m2m_ctx); + mtk_v4l2_err("Failed to v4l2_m2m_ctx_init() (%d)", + ret); + goto err_m2m_ctx_init; + } + mtk_vcodec_enc_set_default_params(ctx); + + if (v4l2_fh_is_singular(&ctx->fh)) { + /* + * vpu_load_firmware checks if it was loaded already and + * does nothing in that case + */ + ret = vpu_load_firmware(dev->vpu_plat_dev); + if (ret < 0) { + /* + * Return 0 if downloading firmware successfully, + * otherwise it is failed + */ + mtk_v4l2_err("vpu_load_firmware failed!"); + goto err_load_fw; + } + + dev->enc_capability = + vpu_get_venc_hw_capa(dev->vpu_plat_dev); + mtk_v4l2_debug(0, "encoder capability %x", dev->enc_capability); + } + + mtk_v4l2_debug(2, "Create instance [%d]@%p m2m_ctx=%p ", + ctx->id, ctx, ctx->m2m_ctx); + + dev->num_instances++; + list_add(&ctx->list, &dev->ctx_list); + + mutex_unlock(&dev->dev_mutex); + mtk_v4l2_debug(0, "%s encoder [%d]", dev_name(&dev->plat_dev->dev), + ctx->id); + return ret; + + /* Deinit when failure occurred */ +err_load_fw: + v4l2_m2m_ctx_release(ctx->m2m_ctx); +err_m2m_ctx_init: + v4l2_ctrl_handler_free(&ctx->ctrl_hdl); +err_ctrls_setup: + v4l2_fh_del(&ctx->fh); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + mutex_unlock(&dev->dev_mutex); + + return ret; +} + +static int fops_vcodec_release(struct file *file) +{ + struct mtk_vcodec_dev *dev = video_drvdata(file); + struct mtk_vcodec_ctx *ctx = fh_to_ctx(file->private_data); + + mtk_v4l2_debug(1, "[%d] encoder", ctx->id); + mutex_lock(&dev->dev_mutex); + + mtk_vcodec_enc_release(ctx); + v4l2_fh_del(&ctx->fh); + v4l2_fh_exit(&ctx->fh); + v4l2_ctrl_handler_free(&ctx->ctrl_hdl); + v4l2_m2m_ctx_release(ctx->m2m_ctx); + + list_del_init(&ctx->list); + dev->num_instances--; + kfree(ctx); + mutex_unlock(&dev->dev_mutex); + return 0; +} + +static const struct v4l2_file_operations mtk_vcodec_fops = { + .owner = THIS_MODULE, + .open = fops_vcodec_open, + .release = fops_vcodec_release, + .poll = v4l2_m2m_fop_poll, + .unlocked_ioctl = video_ioctl2, + .mmap = v4l2_m2m_fop_mmap, +}; + +static int mtk_vcodec_probe(struct platform_device *pdev) +{ + struct mtk_vcodec_dev *dev; + struct video_device *vfd_enc; + struct resource *res; + int i, j, ret; + DEFINE_DMA_ATTRS(attrs); + + dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + INIT_LIST_HEAD(&dev->ctx_list); + dev->plat_dev = pdev; + + dev->vpu_plat_dev = vpu_get_plat_device(dev->plat_dev); + if (dev->vpu_plat_dev == NULL) { + mtk_v4l2_err("[VPU] vpu device in not ready"); + return -EPROBE_DEFER; + } + + vpu_wdt_reg_handler(dev->vpu_plat_dev, mtk_vcodec_enc_reset_handler, + dev, VPU_RST_ENC); + + ret = mtk_vcodec_init_enc_pm(dev); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to get mt vcodec clock source!"); + return ret; + } + + for (i = VENC_SYS, j = 0; i < NUM_MAX_VCODEC_REG_BASE; i++, j++) { + res = platform_get_resource(pdev, IORESOURCE_MEM, j); + if (res == NULL) { + dev_err(&pdev->dev, "get memory resource failed."); + ret = -ENXIO; + goto err_res; + } + dev->reg_base[i] = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR((__force void *)dev->reg_base[i])) { + ret = PTR_ERR((__force void *)dev->reg_base[i]); + goto err_res; + } + mtk_v4l2_debug(2, "reg[%d] base=0x%p", i, dev->reg_base[i]); + } + + res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (res == NULL) { + dev_err(&pdev->dev, "failed to get irq resource"); + ret = -ENOENT; + goto err_res; + } + + dev->enc_irq = platform_get_irq(pdev, 0); + ret = devm_request_irq(&pdev->dev, dev->enc_irq, + mtk_vcodec_enc_irq_handler, + 0, pdev->name, dev); + if (ret) { + dev_err(&pdev->dev, "Failed to install dev->enc_irq %d (%d)", + dev->enc_irq, + ret); + ret = -EINVAL; + goto err_res; + } + + dev->enc_lt_irq = platform_get_irq(pdev, 1); + ret = devm_request_irq(&pdev->dev, + dev->enc_lt_irq, mtk_vcodec_enc_lt_irq_handler, + 0, pdev->name, dev); + if (ret) { + dev_err(&pdev->dev, + "Failed to install dev->enc_lt_irq %d (%d)", + dev->enc_lt_irq, ret); + ret = -EINVAL; + goto err_res; + } + + disable_irq(dev->enc_irq); + disable_irq(dev->enc_lt_irq); /* VENC_LT */ + mutex_init(&dev->enc_mutex); + mutex_init(&dev->dev_mutex); + spin_lock_init(&dev->irqlock); + + snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name), "%s", + "[MTK_V4L2_VENC]"); + + ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev); + if (ret) { + mtk_v4l2_err("v4l2_device_register err=%d", ret); + goto err_res; + } + + init_waitqueue_head(&dev->queue); + + /* allocate video device for encoder and register it */ + vfd_enc = video_device_alloc(); + if (!vfd_enc) { + mtk_v4l2_err("Failed to allocate video device"); + ret = -ENOMEM; + goto err_enc_alloc; + } + vfd_enc->fops = &mtk_vcodec_fops; + vfd_enc->ioctl_ops = &mtk_venc_ioctl_ops; + vfd_enc->release = video_device_release; + vfd_enc->lock = &dev->dev_mutex; + vfd_enc->v4l2_dev = &dev->v4l2_dev; + vfd_enc->vfl_dir = VFL_DIR_M2M; + vfd_enc->device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | + V4L2_CAP_STREAMING; + + snprintf(vfd_enc->name, sizeof(vfd_enc->name), "%s", + MTK_VCODEC_ENC_NAME); + video_set_drvdata(vfd_enc, dev); + dev->vfd_enc = vfd_enc; + platform_set_drvdata(pdev, dev); + + dev->m2m_dev_enc = v4l2_m2m_init(&mtk_venc_m2m_ops); + if (IS_ERR((__force void *)dev->m2m_dev_enc)) { + mtk_v4l2_err("Failed to init mem2mem enc device"); + ret = PTR_ERR((__force void *)dev->m2m_dev_enc); + goto err_enc_mem_init; + } + + dev->encode_workqueue = + alloc_ordered_workqueue(MTK_VCODEC_ENC_NAME, + WQ_MEM_RECLAIM | + WQ_FREEZABLE); + if (!dev->encode_workqueue) { + mtk_v4l2_err("Failed to create encode workqueue"); + ret = -EINVAL; + goto err_event_workq; + } + + ret = video_register_device(vfd_enc, VFL_TYPE_GRABBER, 1); + if (ret) { + mtk_v4l2_err("Failed to register video device"); + goto err_enc_reg; + } + + /* Avoid the iommu eat big hunks */ + dma_set_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, &attrs); + + mtk_v4l2_debug(0, "encoder registered as /dev/video%d", + vfd_enc->num); + + return 0; + +err_enc_reg: + destroy_workqueue(dev->encode_workqueue); +err_event_workq: + v4l2_m2m_release(dev->m2m_dev_enc); +err_enc_mem_init: + video_unregister_device(vfd_enc); +err_enc_alloc: + v4l2_device_unregister(&dev->v4l2_dev); +err_res: + mtk_vcodec_release_enc_pm(dev); + return ret; +} + +static const struct of_device_id mtk_vcodec_enc_match[] = { + {.compatible = "mediatek,mt8173-vcodec-enc",}, + {}, +}; +MODULE_DEVICE_TABLE(of, mtk_vcodec_enc_match); + +static int mtk_vcodec_enc_remove(struct platform_device *pdev) +{ + struct mtk_vcodec_dev *dev = platform_get_drvdata(pdev); + + mtk_v4l2_debug_enter(); + flush_workqueue(dev->encode_workqueue); + destroy_workqueue(dev->encode_workqueue); + if (dev->m2m_dev_enc) + v4l2_m2m_release(dev->m2m_dev_enc); + + if (dev->vfd_enc) + video_unregister_device(dev->vfd_enc); + + v4l2_device_unregister(&dev->v4l2_dev); + mtk_vcodec_release_enc_pm(dev); + return 0; +} + +static struct platform_driver mtk_vcodec_enc_driver = { + .probe = mtk_vcodec_probe, + .remove = mtk_vcodec_enc_remove, + .driver = { + .name = MTK_VCODEC_ENC_NAME, + .of_match_table = mtk_vcodec_enc_match, + }, +}; + +module_platform_driver(mtk_vcodec_enc_driver); + + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Mediatek video codec V4L2 encoder driver"); diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.c new file mode 100644 index 000000000000..3e73e9db781f --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.c @@ -0,0 +1,137 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include + +#include "mtk_vcodec_enc_pm.h" +#include "mtk_vcodec_util.h" +#include "mtk_vpu.h" + + +int mtk_vcodec_init_enc_pm(struct mtk_vcodec_dev *mtkdev) +{ + struct device_node *node; + struct platform_device *pdev; + struct device *dev; + struct mtk_vcodec_pm *pm; + int ret = 0; + + pdev = mtkdev->plat_dev; + pm = &mtkdev->pm; + memset(pm, 0, sizeof(struct mtk_vcodec_pm)); + pm->mtkdev = mtkdev; + pm->dev = &pdev->dev; + dev = &pdev->dev; + + node = of_parse_phandle(dev->of_node, "mediatek,larb", 0); + if (!node) { + mtk_v4l2_err("no mediatek,larb found"); + return -1; + } + pdev = of_find_device_by_node(node); + if (!pdev) { + mtk_v4l2_err("no mediatek,larb device found"); + return -1; + } + pm->larbvenc = &pdev->dev; + + node = of_parse_phandle(dev->of_node, "mediatek,larb", 1); + if (!node) { + mtk_v4l2_err("no mediatek,larb found"); + return -1; + } + + pdev = of_find_device_by_node(node); + if (!pdev) { + mtk_v4l2_err("no mediatek,larb device found"); + return -1; + } + + pm->larbvenclt = &pdev->dev; + pdev = mtkdev->plat_dev; + pm->dev = &pdev->dev; + + pm->vencpll_d2 = devm_clk_get(&pdev->dev, "venc_sel_src"); + if (IS_ERR(pm->vencpll_d2)) { + mtk_v4l2_err("devm_clk_get vencpll_d2 fail"); + ret = PTR_ERR(pm->vencpll_d2); + } + + pm->venc_sel = devm_clk_get(&pdev->dev, "venc_sel"); + if (IS_ERR(pm->venc_sel)) { + mtk_v4l2_err("devm_clk_get venc_sel fail"); + ret = PTR_ERR(pm->venc_sel); + } + + pm->univpll1_d2 = devm_clk_get(&pdev->dev, "venc_lt_sel_src"); + if (IS_ERR(pm->univpll1_d2)) { + mtk_v4l2_err("devm_clk_get univpll1_d2 fail"); + ret = PTR_ERR(pm->univpll1_d2); + } + + pm->venc_lt_sel = devm_clk_get(&pdev->dev, "venc_lt_sel"); + if (IS_ERR(pm->venc_lt_sel)) { + mtk_v4l2_err("devm_clk_get venc_lt_sel fail"); + ret = PTR_ERR(pm->venc_lt_sel); + } + + return ret; +} + +void mtk_vcodec_release_enc_pm(struct mtk_vcodec_dev *mtkdev) +{ +} + + +void mtk_vcodec_enc_clock_on(struct mtk_vcodec_pm *pm) +{ + int ret; + + ret = clk_prepare_enable(pm->venc_sel); + if (ret) + mtk_v4l2_err("clk_prepare_enable fail %d", ret); + + ret = clk_set_parent(pm->venc_sel, pm->vencpll_d2); + if (ret) + mtk_v4l2_err("clk_set_parent fail %d", ret); + + ret = clk_prepare_enable(pm->venc_lt_sel); + if (ret) + mtk_v4l2_err("clk_prepare_enable fail %d", ret); + + ret = clk_set_parent(pm->venc_lt_sel, pm->univpll1_d2); + if (ret) + mtk_v4l2_err("clk_set_parent fail %d", ret); + + ret = mtk_smi_larb_get(pm->larbvenc); + if (ret) + mtk_v4l2_err("mtk_smi_larb_get larb3 fail %d", ret); + + ret = mtk_smi_larb_get(pm->larbvenclt); + if (ret) + mtk_v4l2_err("mtk_smi_larb_get larb4 fail %d", ret); + +} + +void mtk_vcodec_enc_clock_off(struct mtk_vcodec_pm *pm) +{ + mtk_smi_larb_put(pm->larbvenc); + mtk_smi_larb_put(pm->larbvenclt); + clk_disable_unprepare(pm->venc_lt_sel); + clk_disable_unprepare(pm->venc_sel); +} diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.h new file mode 100644 index 000000000000..f32167138976 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.h @@ -0,0 +1,26 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#ifndef _MTK_VCODEC_ENC_PM_H_ +#define _MTK_VCODEC_ENC_PM_H_ + +#include "mtk_vcodec_drv.h" + +int mtk_vcodec_init_enc_pm(struct mtk_vcodec_dev *dev); +void mtk_vcodec_release_enc_pm(struct mtk_vcodec_dev *dev); + +void mtk_vcodec_enc_clock_on(struct mtk_vcodec_pm *pm); +void mtk_vcodec_enc_clock_off(struct mtk_vcodec_pm *pm); + +#endif /* _MTK_VCODEC_ENC_PM_H_ */ diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.c new file mode 100644 index 000000000000..52e7e5c9afa0 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.c @@ -0,0 +1,54 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#include +#include + +#include "mtk_vcodec_drv.h" +#include "mtk_vcodec_intr.h" +#include "mtk_vcodec_util.h" + +int mtk_vcodec_wait_for_done_ctx(struct mtk_vcodec_ctx *ctx, int command, + unsigned int timeout_ms) +{ + wait_queue_head_t *waitqueue; + long timeout_jiff, ret; + int status = 0; + + waitqueue = (wait_queue_head_t *)&ctx->queue; + timeout_jiff = msecs_to_jiffies(timeout_ms); + + ret = wait_event_interruptible_timeout(*waitqueue, + (ctx->int_cond && + (ctx->int_type == command)), + timeout_jiff); + + if (!ret) { + status = -1; /* timeout */ + mtk_v4l2_err("[%d] cmd=%d, ctx->type=%d, wait_event_interruptible_timeout time=%ums out %d %d!", + ctx->id, ctx->type, command, timeout_ms, + ctx->int_cond, ctx->int_type); + } else if (-ERESTARTSYS == ret) { + mtk_v4l2_err("[%d] cmd=%d, ctx->type=%d, wait_event_interruptible_timeout interrupted by a signal %d %d", + ctx->id, ctx->type, command, ctx->int_cond, + ctx->int_type); + status = -1; + } + + ctx->int_cond = 0; + ctx->int_type = 0; + + return status; +} +EXPORT_SYMBOL(mtk_vcodec_wait_for_done_ctx); diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h new file mode 100644 index 000000000000..33e890f5aa9c --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h @@ -0,0 +1,27 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#ifndef _MTK_VCODEC_INTR_H_ +#define _MTK_VCODEC_INTR_H_ + +#define MTK_INST_IRQ_RECEIVED 0x1 +#define MTK_INST_WORK_THREAD_ABORT_DONE 0x2 + +struct mtk_vcodec_ctx; + +/* timeout is ms */ +int mtk_vcodec_wait_for_done_ctx(struct mtk_vcodec_ctx *data, int command, + unsigned int timeout_ms); + +#endif /* _MTK_VCODEC_INTR_H_ */ diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c new file mode 100644 index 000000000000..5e3651372a3c --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c @@ -0,0 +1,94 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: PC Chen +* Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#include + +#include "mtk_vcodec_drv.h" +#include "mtk_vcodec_util.h" +#include "mtk_vpu.h" + +/* For encoder, this will enable logs in venc/*/ +bool mtk_vcodec_dbg; +EXPORT_SYMBOL(mtk_vcodec_dbg); + +/* The log level of v4l2 encoder or decoder driver. + * That is, files under mtk-vcodec/. + */ +int mtk_v4l2_dbg_level; +EXPORT_SYMBOL(mtk_v4l2_dbg_level); + +void __iomem *mtk_vcodec_get_reg_addr(struct mtk_vcodec_ctx *data, + unsigned int reg_idx) +{ + struct mtk_vcodec_ctx *ctx = (struct mtk_vcodec_ctx *)data; + + if (!data || reg_idx >= NUM_MAX_VCODEC_REG_BASE) { + mtk_v4l2_err("Invalid arguments, reg_idx=%d", reg_idx); + return NULL; + } + return ctx->dev->reg_base[reg_idx]; +} +EXPORT_SYMBOL(mtk_vcodec_get_reg_addr); + +int mtk_vcodec_mem_alloc(struct mtk_vcodec_ctx *data, + struct mtk_vcodec_mem *mem) +{ + unsigned long size = mem->size; + struct mtk_vcodec_ctx *ctx = (struct mtk_vcodec_ctx *)data; + struct device *dev = &ctx->dev->plat_dev->dev; + + mem->va = dma_alloc_coherent(dev, size, &mem->dma_addr, GFP_KERNEL); + + if (!mem->va) { + mtk_v4l2_err("%s dma_alloc size=%ld failed!", dev_name(dev), + size); + return -ENOMEM; + } + + memset(mem->va, 0, size); + + mtk_v4l2_debug(3, "[%d] - va = %p", ctx->id, mem->va); + mtk_v4l2_debug(3, "[%d] - dma = 0x%lx", ctx->id, + (unsigned long)mem->dma_addr); + mtk_v4l2_debug(3, "[%d] size = 0x%lx", ctx->id, size); + + return 0; +} +EXPORT_SYMBOL(mtk_vcodec_mem_alloc); + +void mtk_vcodec_mem_free(struct mtk_vcodec_ctx *data, + struct mtk_vcodec_mem *mem) +{ + unsigned long size = mem->size; + struct mtk_vcodec_ctx *ctx = (struct mtk_vcodec_ctx *)data; + struct device *dev = &ctx->dev->plat_dev->dev; + + if (!mem->va) { + mtk_v4l2_err("%s dma_free size=%ld failed!", dev_name(dev), + size); + return; + } + + dma_free_coherent(dev, size, mem->va, mem->dma_addr); + mem->va = NULL; + mem->dma_addr = 0; + mem->size = 0; + + mtk_v4l2_debug(3, "[%d] - va = %p", ctx->id, mem->va); + mtk_v4l2_debug(3, "[%d] - dma = 0x%lx", ctx->id, + (unsigned long)mem->dma_addr); + mtk_v4l2_debug(3, "[%d] size = 0x%lx", ctx->id, size); +} +EXPORT_SYMBOL(mtk_vcodec_mem_free); diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.h new file mode 100644 index 000000000000..d6345fc04840 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.h @@ -0,0 +1,87 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: PC Chen +* Tiffany Lin +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#ifndef _MTK_VCODEC_UTIL_H_ +#define _MTK_VCODEC_UTIL_H_ + +#include +#include + +struct mtk_vcodec_mem { + size_t size; + void *va; + dma_addr_t dma_addr; +}; + +struct mtk_vcodec_ctx; + +extern int mtk_v4l2_dbg_level; +extern bool mtk_vcodec_dbg; + +#define DEBUG 1 + +#if defined(DEBUG) + +#define mtk_v4l2_debug(level, fmt, args...) \ + do { \ + if (mtk_v4l2_dbg_level >= level) \ + pr_info("[MTK_V4L2] level=%d %s(),%d: " fmt "\n",\ + level, __func__, __LINE__, ##args); \ + } while (0) + +#define mtk_v4l2_err(fmt, args...) \ + pr_err("[MTK_V4L2][ERROR] %s:%d: " fmt "\n", __func__, __LINE__, \ + ##args) + + +#define mtk_v4l2_debug_enter() mtk_v4l2_debug(3, "+") +#define mtk_v4l2_debug_leave() mtk_v4l2_debug(3, "-") + +#define mtk_vcodec_debug(h, fmt, args...) \ + do { \ + if (mtk_vcodec_dbg) \ + pr_info("[MTK_VCODEC][%d]: %s() " fmt "\n", \ + ((struct mtk_vcodec_ctx *)h->ctx)->id, \ + __func__, ##args); \ + } while (0) + +#define mtk_vcodec_err(h, fmt, args...) \ + pr_err("[MTK_VCODEC][ERROR][%d]: %s() " fmt "\n", \ + ((struct mtk_vcodec_ctx *)h->ctx)->id, __func__, ##args) + +#define mtk_vcodec_debug_enter(h) mtk_vcodec_debug(h, "+") +#define mtk_vcodec_debug_leave(h) mtk_vcodec_debug(h, "-") + +#else + +#define mtk_v4l2_debug(level, fmt, args...) +#define mtk_v4l2_err(fmt, args...) +#define mtk_v4l2_debug_enter() +#define mtk_v4l2_debug_leave() + +#define mtk_vcodec_debug(h, fmt, args...) +#define mtk_vcodec_err(h, fmt, args...) +#define mtk_vcodec_debug_enter(h) +#define mtk_vcodec_debug_leave(h) + +#endif + +void __iomem *mtk_vcodec_get_reg_addr(struct mtk_vcodec_ctx *data, + unsigned int reg_idx); +int mtk_vcodec_mem_alloc(struct mtk_vcodec_ctx *data, + struct mtk_vcodec_mem *mem); +void mtk_vcodec_mem_free(struct mtk_vcodec_ctx *data, + struct mtk_vcodec_mem *mem); +#endif /* _MTK_VCODEC_UTIL_H_ */ diff --git a/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c b/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c new file mode 100644 index 000000000000..9a600525b3c1 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c @@ -0,0 +1,679 @@ +/* + * Copyright (c) 2016 MediaTek Inc. + * Author: Jungchang Tsao + * Daniel Hsiao + * PoChun Lin + * + * This program is free software; you can redistribute it and/or + * modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include + +#include "../mtk_vcodec_drv.h" +#include "../mtk_vcodec_util.h" +#include "../mtk_vcodec_intr.h" +#include "../mtk_vcodec_enc.h" +#include "../mtk_vcodec_enc_pm.h" +#include "../venc_drv_base.h" +#include "../venc_ipi_msg.h" +#include "../venc_vpu_if.h" +#include "mtk_vpu.h" + +static const char h264_filler_marker[] = {0x0, 0x0, 0x0, 0x1, 0xc}; + +#define H264_FILLER_MARKER_SIZE ARRAY_SIZE(h264_filler_marker) +#define VENC_PIC_BITSTREAM_BYTE_CNT 0x0098 + +/** + * enum venc_h264_vpu_work_buf - h264 encoder buffer index + */ +enum venc_h264_vpu_work_buf { + VENC_H264_VPU_WORK_BUF_RC_INFO, + VENC_H264_VPU_WORK_BUF_RC_CODE, + VENC_H264_VPU_WORK_BUF_REC_LUMA, + VENC_H264_VPU_WORK_BUF_REC_CHROMA, + VENC_H264_VPU_WORK_BUF_REF_LUMA, + VENC_H264_VPU_WORK_BUF_REF_CHROMA, + VENC_H264_VPU_WORK_BUF_MV_INFO_1, + VENC_H264_VPU_WORK_BUF_MV_INFO_2, + VENC_H264_VPU_WORK_BUF_SKIP_FRAME, + VENC_H264_VPU_WORK_BUF_MAX, +}; + +/** + * enum venc_h264_bs_mode - for bs_mode argument in h264_enc_vpu_encode + */ +enum venc_h264_bs_mode { + H264_BS_MODE_SPS, + H264_BS_MODE_PPS, + H264_BS_MODE_FRAME, +}; + +/* + * struct venc_h264_vpu_config - Structure for h264 encoder configuration + * @input_fourcc: input fourcc + * @bitrate: target bitrate (in bps) + * @pic_w: picture width. Picture size is visible stream resolution, in pixels, + * to be used for display purposes; must be smaller or equal to buffer + * size. + * @pic_h: picture height + * @buf_w: buffer width. Buffer size is stream resolution in pixels aligned to + * hardware requirements. + * @buf_h: buffer height + * @gop_size: group of picture size (idr frame) + * @intra_period: intra frame period + * @framerate: frame rate in fps + * @profile: as specified in standard + * @level: as specified in standard + * @wfd: WFD mode 1:on, 0:off + */ +struct venc_h264_vpu_config { + u32 input_fourcc; + u32 bitrate; + u32 pic_w; + u32 pic_h; + u32 buf_w; + u32 buf_h; + u32 gop_size; + u32 intra_period; + u32 framerate; + u32 profile; + u32 level; + u32 wfd; +}; + +/* + * struct venc_h264_vpu_buf - Structure for buffer information + * @align: buffer alignment (in bytes) + * @iova: IO virtual address + * @vpua: VPU side memory addr which is used by RC_CODE + * @size: buffer size (in bytes) + */ +struct venc_h264_vpu_buf { + u32 align; + u32 iova; + u32 vpua; + u32 size; +}; + +/* + * struct venc_h264_vsi - Structure for VPU driver control and info share + * This structure is allocated in VPU side and shared to AP side. + * @config: h264 encoder configuration + * @work_bufs: working buffer information in VPU side + * The work_bufs here is for storing the 'size' info shared to AP side. + * The similar item in struct venc_h264_inst is for memory allocation + * in AP side. The AP driver will copy the 'size' from here to the one in + * struct mtk_vcodec_mem, then invoke mtk_vcodec_mem_alloc to allocate + * the buffer. After that, bypass the 'dma_addr' to the 'iova' field here for + * register setting in VPU side. + */ +struct venc_h264_vsi { + struct venc_h264_vpu_config config; + struct venc_h264_vpu_buf work_bufs[VENC_H264_VPU_WORK_BUF_MAX]; +}; + +/* + * struct venc_h264_inst - h264 encoder AP driver instance + * @hw_base: h264 encoder hardware register base + * @work_bufs: working buffer + * @pps_buf: buffer to store the pps bitstream + * @work_buf_allocated: working buffer allocated flag + * @frm_cnt: encoded frame count + * @prepend_hdr: when the v4l2 layer send VENC_SET_PARAM_PREPEND_HEADER cmd + * through h264_enc_set_param interface, it will set this flag and prepend the + * sps/pps in h264_enc_encode function. + * @vpu_inst: VPU instance to exchange information between AP and VPU + * @vsi: driver structure allocated by VPU side and shared to AP side for + * control and info share + * @ctx: context for v4l2 layer integration + */ +struct venc_h264_inst { + void __iomem *hw_base; + struct mtk_vcodec_mem work_bufs[VENC_H264_VPU_WORK_BUF_MAX]; + struct mtk_vcodec_mem pps_buf; + bool work_buf_allocated; + unsigned int frm_cnt; + unsigned int prepend_hdr; + struct venc_vpu_inst vpu_inst; + struct venc_h264_vsi *vsi; + struct mtk_vcodec_ctx *ctx; +}; + +static inline void h264_write_reg(struct venc_h264_inst *inst, u32 addr, + u32 val) +{ + writel(val, inst->hw_base + addr); +} + +static inline u32 h264_read_reg(struct venc_h264_inst *inst, u32 addr) +{ + return readl(inst->hw_base + addr); +} + +static unsigned int h264_get_profile(struct venc_h264_inst *inst, + unsigned int profile) +{ + switch (profile) { + case V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE: + return 66; + case V4L2_MPEG_VIDEO_H264_PROFILE_MAIN: + return 77; + case V4L2_MPEG_VIDEO_H264_PROFILE_HIGH: + return 100; + case V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE: + mtk_vcodec_err(inst, "unsupported CONSTRAINED_BASELINE"); + return 0; + case V4L2_MPEG_VIDEO_H264_PROFILE_EXTENDED: + mtk_vcodec_err(inst, "unsupported EXTENDED"); + return 0; + default: + mtk_vcodec_debug(inst, "unsupported profile %d", profile); + return 100; + } +} + +static unsigned int h264_get_level(struct venc_h264_inst *inst, + unsigned int level) +{ + switch (level) { + case V4L2_MPEG_VIDEO_H264_LEVEL_1B: + mtk_vcodec_err(inst, "unsupported 1B"); + return 0; + case V4L2_MPEG_VIDEO_H264_LEVEL_1_0: + return 10; + case V4L2_MPEG_VIDEO_H264_LEVEL_1_1: + return 11; + case V4L2_MPEG_VIDEO_H264_LEVEL_1_2: + return 12; + case V4L2_MPEG_VIDEO_H264_LEVEL_1_3: + return 13; + case V4L2_MPEG_VIDEO_H264_LEVEL_2_0: + return 20; + case V4L2_MPEG_VIDEO_H264_LEVEL_2_1: + return 21; + case V4L2_MPEG_VIDEO_H264_LEVEL_2_2: + return 22; + case V4L2_MPEG_VIDEO_H264_LEVEL_3_0: + return 30; + case V4L2_MPEG_VIDEO_H264_LEVEL_3_1: + return 31; + case V4L2_MPEG_VIDEO_H264_LEVEL_3_2: + return 32; + case V4L2_MPEG_VIDEO_H264_LEVEL_4_0: + return 40; + case V4L2_MPEG_VIDEO_H264_LEVEL_4_1: + return 41; + default: + mtk_vcodec_debug(inst, "unsupported level %d", level); + return 31; + } +} + +static void h264_enc_free_work_buf(struct venc_h264_inst *inst) +{ + int i; + + mtk_vcodec_debug_enter(inst); + + /* Except the SKIP_FRAME buffers, + * other buffers need to be freed by AP. + */ + for (i = 0; i < VENC_H264_VPU_WORK_BUF_MAX; i++) { + if (i != VENC_H264_VPU_WORK_BUF_SKIP_FRAME) + mtk_vcodec_mem_free(inst->ctx, &inst->work_bufs[i]); + } + + mtk_vcodec_mem_free(inst->ctx, &inst->pps_buf); + + mtk_vcodec_debug_leave(inst); +} + +static int h264_enc_alloc_work_buf(struct venc_h264_inst *inst) +{ + int i; + int ret = 0; + struct venc_h264_vpu_buf *wb = inst->vsi->work_bufs; + + mtk_vcodec_debug_enter(inst); + + for (i = 0; i < VENC_H264_VPU_WORK_BUF_MAX; i++) { + /* + * This 'wb' structure is set by VPU side and shared to AP for + * buffer allocation and IO virtual addr mapping. For most of + * the buffers, AP will allocate the buffer according to 'size' + * field and store the IO virtual addr in 'iova' field. There + * are two exceptions: + * (1) RC_CODE buffer, it's pre-allocated in the VPU side, and + * save the VPU addr in the 'vpua' field. The AP will translate + * the VPU addr to the corresponding IO virtual addr and store + * in 'iova' field for reg setting in VPU side. + * (2) SKIP_FRAME buffer, it's pre-allocated in the VPU side, + * and save the VPU addr in the 'vpua' field. The AP will + * translate the VPU addr to the corresponding AP side virtual + * address and do some memcpy access to move to bitstream buffer + * assigned by v4l2 layer. + */ + inst->work_bufs[i].size = wb[i].size; + if (i == VENC_H264_VPU_WORK_BUF_SKIP_FRAME) { + inst->work_bufs[i].va = vpu_mapping_dm_addr( + inst->vpu_inst.dev, wb[i].vpua); + inst->work_bufs[i].dma_addr = 0; + } else { + ret = mtk_vcodec_mem_alloc(inst->ctx, + &inst->work_bufs[i]); + if (ret) { + mtk_vcodec_err(inst, + "cannot allocate buf %d", i); + goto err_alloc; + } + /* + * This RC_CODE is pre-allocated by VPU and saved in VPU + * addr. So we need use memcpy to copy RC_CODE from VPU + * addr into IO virtual addr in 'iova' field for reg + * setting in VPU side. + */ + if (i == VENC_H264_VPU_WORK_BUF_RC_CODE) { + void *tmp_va; + + tmp_va = vpu_mapping_dm_addr(inst->vpu_inst.dev, + wb[i].vpua); + memcpy(inst->work_bufs[i].va, tmp_va, + wb[i].size); + } + } + wb[i].iova = inst->work_bufs[i].dma_addr; + + mtk_vcodec_debug(inst, + "work_buf[%d] va=0x%p iova=%pad size=%zu", + i, inst->work_bufs[i].va, + &inst->work_bufs[i].dma_addr, + inst->work_bufs[i].size); + } + + /* the pps_buf is used by AP side only */ + inst->pps_buf.size = 128; + ret = mtk_vcodec_mem_alloc(inst->ctx, &inst->pps_buf); + if (ret) { + mtk_vcodec_err(inst, "cannot allocate pps_buf"); + goto err_alloc; + } + + mtk_vcodec_debug_leave(inst); + + return ret; + +err_alloc: + h264_enc_free_work_buf(inst); + + return ret; +} + +static unsigned int h264_enc_wait_venc_done(struct venc_h264_inst *inst) +{ + unsigned int irq_status = 0; + struct mtk_vcodec_ctx *ctx = (struct mtk_vcodec_ctx *)inst->ctx; + + if (!mtk_vcodec_wait_for_done_ctx(ctx, MTK_INST_IRQ_RECEIVED, + WAIT_INTR_TIMEOUT_MS)) { + irq_status = ctx->irq_status; + mtk_vcodec_debug(inst, "irq_status %x <-", irq_status); + } + return irq_status; +} + +static int h264_encode_sps(struct venc_h264_inst *inst, + struct mtk_vcodec_mem *bs_buf, + unsigned int *bs_size) +{ + int ret = 0; + unsigned int irq_status; + + mtk_vcodec_debug_enter(inst); + + ret = vpu_enc_encode(&inst->vpu_inst, H264_BS_MODE_SPS, NULL, + bs_buf, bs_size); + if (ret) + return ret; + + irq_status = h264_enc_wait_venc_done(inst); + if (irq_status != MTK_VENC_IRQ_STATUS_SPS) { + mtk_vcodec_err(inst, "expect irq status %d", + MTK_VENC_IRQ_STATUS_SPS); + return -EINVAL; + } + + *bs_size = h264_read_reg(inst, VENC_PIC_BITSTREAM_BYTE_CNT); + mtk_vcodec_debug(inst, "bs size %d <-", *bs_size); + + return ret; +} + +static int h264_encode_pps(struct venc_h264_inst *inst, + struct mtk_vcodec_mem *bs_buf, + unsigned int *bs_size) +{ + int ret = 0; + unsigned int irq_status; + + mtk_vcodec_debug_enter(inst); + + ret = vpu_enc_encode(&inst->vpu_inst, H264_BS_MODE_PPS, NULL, + bs_buf, bs_size); + if (ret) + return ret; + + irq_status = h264_enc_wait_venc_done(inst); + if (irq_status != MTK_VENC_IRQ_STATUS_PPS) { + mtk_vcodec_err(inst, "expect irq status %d", + MTK_VENC_IRQ_STATUS_PPS); + return -EINVAL; + } + + *bs_size = h264_read_reg(inst, VENC_PIC_BITSTREAM_BYTE_CNT); + mtk_vcodec_debug(inst, "bs size %d <-", *bs_size); + + return ret; +} + +static int h264_encode_header(struct venc_h264_inst *inst, + struct mtk_vcodec_mem *bs_buf, + unsigned int *bs_size) +{ + int ret = 0; + unsigned int bs_size_sps; + unsigned int bs_size_pps; + + ret = h264_encode_sps(inst, bs_buf, &bs_size_sps); + if (ret) + return ret; + + ret = h264_encode_pps(inst, &inst->pps_buf, &bs_size_pps); + if (ret) + return ret; + + memcpy(bs_buf->va + bs_size_sps, inst->pps_buf.va, bs_size_pps); + *bs_size = bs_size_sps + bs_size_pps; + + return ret; +} + +static int h264_encode_frame(struct venc_h264_inst *inst, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + unsigned int *bs_size) +{ + int ret = 0; + unsigned int irq_status; + + mtk_vcodec_debug_enter(inst); + + ret = vpu_enc_encode(&inst->vpu_inst, H264_BS_MODE_FRAME, frm_buf, + bs_buf, bs_size); + if (ret) + return ret; + + /* + * skip frame case: The skip frame buffer is composed by vpu side only, + * it does not trigger the hw, so skip the wait interrupt operation. + */ + if (inst->vpu_inst.state == VEN_IPI_MSG_ENC_STATE_SKIP) { + *bs_size = inst->vpu_inst.bs_size; + memcpy(bs_buf->va, + inst->work_bufs[VENC_H264_VPU_WORK_BUF_SKIP_FRAME].va, + *bs_size); + ++inst->frm_cnt; + return ret; + } + + irq_status = h264_enc_wait_venc_done(inst); + if (irq_status != MTK_VENC_IRQ_STATUS_FRM) { + mtk_vcodec_err(inst, "irq_status=%d failed", irq_status); + return -EIO; + } + + *bs_size = h264_read_reg(inst, VENC_PIC_BITSTREAM_BYTE_CNT); + + ++inst->frm_cnt; + mtk_vcodec_debug(inst, "frm %d bs_size %d key_frm %d <-", + inst->frm_cnt, *bs_size, inst->vpu_inst.is_key_frm); + + return ret; +} + +static void h264_encode_filler(struct venc_h264_inst *inst, void *buf, + int size) +{ + unsigned char *p = buf; + + if (size < H264_FILLER_MARKER_SIZE) { + mtk_vcodec_err(inst, "filler size too small %d", size); + return; + } + + memcpy(p, h264_filler_marker, ARRAY_SIZE(h264_filler_marker)); + size -= H264_FILLER_MARKER_SIZE; + p += H264_FILLER_MARKER_SIZE; + memset(p, 0xff, size); +} + +static int h264_enc_init(struct mtk_vcodec_ctx *ctx, unsigned long *handle) +{ + int ret = 0; + struct venc_h264_inst *inst; + + inst = kzalloc(sizeof(*inst), GFP_KERNEL); + if (!inst) + return -ENOMEM; + + inst->ctx = ctx; + inst->vpu_inst.ctx = ctx; + inst->vpu_inst.dev = ctx->dev->vpu_plat_dev; + inst->vpu_inst.id = IPI_VENC_H264; + inst->hw_base = mtk_vcodec_get_reg_addr(inst->ctx, VENC_SYS); + + mtk_vcodec_debug_enter(inst); + + ret = vpu_enc_init(&inst->vpu_inst); + + inst->vsi = (struct venc_h264_vsi *)inst->vpu_inst.vsi; + + mtk_vcodec_debug_leave(inst); + + if (ret) + kfree(inst); + else + (*handle) = (unsigned long)inst; + + return ret; +} + +static int h264_enc_encode(unsigned long handle, + enum venc_start_opt opt, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + struct venc_done_result *result) +{ + int ret = 0; + struct venc_h264_inst *inst = (struct venc_h264_inst *)handle; + struct mtk_vcodec_ctx *ctx = inst->ctx; + + mtk_vcodec_debug(inst, "opt %d ->", opt); + + enable_irq(ctx->dev->enc_irq); + + switch (opt) { + case VENC_START_OPT_ENCODE_SEQUENCE_HEADER: { + unsigned int bs_size_hdr; + + ret = h264_encode_header(inst, bs_buf, &bs_size_hdr); + if (ret) + goto encode_err; + + result->bs_size = bs_size_hdr; + result->is_key_frm = false; + break; + } + + case VENC_START_OPT_ENCODE_FRAME: { + int hdr_sz; + int hdr_sz_ext; + int filler_sz = 0; + const int bs_alignment = 128; + struct mtk_vcodec_mem tmp_bs_buf; + unsigned int bs_size_hdr; + unsigned int bs_size_frm; + + if (!inst->prepend_hdr) { + ret = h264_encode_frame(inst, frm_buf, bs_buf, + &result->bs_size); + if (ret) + goto encode_err; + result->is_key_frm = inst->vpu_inst.is_key_frm; + break; + } + + mtk_vcodec_debug(inst, "h264_encode_frame prepend SPS/PPS"); + + ret = h264_encode_header(inst, bs_buf, &bs_size_hdr); + if (ret) + goto encode_err; + + hdr_sz = bs_size_hdr; + hdr_sz_ext = (hdr_sz & (bs_alignment - 1)); + if (hdr_sz_ext) { + filler_sz = bs_alignment - hdr_sz_ext; + if (hdr_sz_ext + H264_FILLER_MARKER_SIZE > bs_alignment) + filler_sz += bs_alignment; + h264_encode_filler(inst, bs_buf->va + hdr_sz, + filler_sz); + } + + tmp_bs_buf.va = bs_buf->va + hdr_sz + filler_sz; + tmp_bs_buf.dma_addr = bs_buf->dma_addr + hdr_sz + filler_sz; + tmp_bs_buf.size = bs_buf->size - (hdr_sz + filler_sz); + + ret = h264_encode_frame(inst, frm_buf, &tmp_bs_buf, + &bs_size_frm); + if (ret) + goto encode_err; + + result->bs_size = hdr_sz + filler_sz + bs_size_frm; + + mtk_vcodec_debug(inst, "hdr %d filler %d frame %d bs %d", + hdr_sz, filler_sz, bs_size_frm, + result->bs_size); + + inst->prepend_hdr = 0; + result->is_key_frm = inst->vpu_inst.is_key_frm; + break; + } + + default: + mtk_vcodec_err(inst, "venc_start_opt %d not supported", opt); + ret = -EINVAL; + break; + } + +encode_err: + + disable_irq(ctx->dev->enc_irq); + mtk_vcodec_debug(inst, "opt %d <-", opt); + + return ret; +} + +static int h264_enc_set_param(unsigned long handle, + enum venc_set_param_type type, + struct venc_enc_param *enc_prm) +{ + int ret = 0; + struct venc_h264_inst *inst = (struct venc_h264_inst *)handle; + + mtk_vcodec_debug(inst, "->type=%d", type); + + switch (type) { + case VENC_SET_PARAM_ENC: + inst->vsi->config.input_fourcc = enc_prm->input_yuv_fmt; + inst->vsi->config.bitrate = enc_prm->bitrate; + inst->vsi->config.pic_w = enc_prm->width; + inst->vsi->config.pic_h = enc_prm->height; + inst->vsi->config.buf_w = enc_prm->buf_width; + inst->vsi->config.buf_h = enc_prm->buf_height; + inst->vsi->config.gop_size = enc_prm->gop_size; + inst->vsi->config.framerate = enc_prm->frm_rate; + inst->vsi->config.intra_period = enc_prm->intra_period; + inst->vsi->config.profile = + h264_get_profile(inst, enc_prm->h264_profile); + inst->vsi->config.level = + h264_get_level(inst, enc_prm->h264_level); + inst->vsi->config.wfd = 0; + ret = vpu_enc_set_param(&inst->vpu_inst, type, enc_prm); + if (ret) + break; + if (inst->work_buf_allocated) { + h264_enc_free_work_buf(inst); + inst->work_buf_allocated = false; + } + ret = h264_enc_alloc_work_buf(inst); + if (ret) + break; + inst->work_buf_allocated = true; + break; + + case VENC_SET_PARAM_PREPEND_HEADER: + inst->prepend_hdr = 1; + mtk_vcodec_debug(inst, "set prepend header mode"); + break; + + default: + ret = vpu_enc_set_param(&inst->vpu_inst, type, enc_prm); + break; + } + + mtk_vcodec_debug_leave(inst); + + return ret; +} + +static int h264_enc_deinit(unsigned long handle) +{ + int ret = 0; + struct venc_h264_inst *inst = (struct venc_h264_inst *)handle; + + mtk_vcodec_debug_enter(inst); + + ret = vpu_enc_deinit(&inst->vpu_inst); + + if (inst->work_buf_allocated) + h264_enc_free_work_buf(inst); + + mtk_vcodec_debug_leave(inst); + kfree(inst); + + return ret; +} + +static struct venc_common_if venc_h264_if = { + h264_enc_init, + h264_enc_encode, + h264_enc_set_param, + h264_enc_deinit, +}; + +struct venc_common_if *get_h264_enc_comm_if(void); + +struct venc_common_if *get_h264_enc_comm_if(void) +{ + return &venc_h264_if; +} diff --git a/drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c b/drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c new file mode 100644 index 000000000000..60bbcd2a0510 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c @@ -0,0 +1,486 @@ +/* + * Copyright (c) 2016 MediaTek Inc. + * Author: Daniel Hsiao + * PoChun Lin + * + * This program is free software; you can redistribute it and/or + * modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include + +#include "../mtk_vcodec_drv.h" +#include "../mtk_vcodec_util.h" +#include "../mtk_vcodec_intr.h" +#include "../mtk_vcodec_enc.h" +#include "../mtk_vcodec_enc_pm.h" +#include "../venc_drv_base.h" +#include "../venc_ipi_msg.h" +#include "../venc_vpu_if.h" +#include "mtk_vpu.h" + +#define VENC_BITSTREAM_FRAME_SIZE 0x0098 +#define VENC_BITSTREAM_HEADER_LEN 0x00e8 + +/* This ac_tag is vp8 frame tag. */ +#define MAX_AC_TAG_SIZE 10 + +/** + * enum venc_vp8_vpu_work_buf - vp8 encoder buffer index + */ +enum venc_vp8_vpu_work_buf { + VENC_VP8_VPU_WORK_BUF_LUMA, + VENC_VP8_VPU_WORK_BUF_LUMA2, + VENC_VP8_VPU_WORK_BUF_LUMA3, + VENC_VP8_VPU_WORK_BUF_CHROMA, + VENC_VP8_VPU_WORK_BUF_CHROMA2, + VENC_VP8_VPU_WORK_BUF_CHROMA3, + VENC_VP8_VPU_WORK_BUF_MV_INFO, + VENC_VP8_VPU_WORK_BUF_BS_HEADER, + VENC_VP8_VPU_WORK_BUF_PROB_BUF, + VENC_VP8_VPU_WORK_BUF_RC_INFO, + VENC_VP8_VPU_WORK_BUF_RC_CODE, + VENC_VP8_VPU_WORK_BUF_RC_CODE2, + VENC_VP8_VPU_WORK_BUF_RC_CODE3, + VENC_VP8_VPU_WORK_BUF_MAX, +}; + +/* + * struct venc_vp8_vpu_config - Structure for vp8 encoder configuration + * @input_fourcc: input fourcc + * @bitrate: target bitrate (in bps) + * @pic_w: picture width. Picture size is visible stream resolution, in pixels, + * to be used for display purposes; must be smaller or equal to buffer + * size. + * @pic_h: picture height + * @buf_w: buffer width (with 16 alignment). Buffer size is stream resolution + * in pixels aligned to hardware requirements. + * @buf_h: buffer height (with 16 alignment) + * @gop_size: group of picture size (key frame) + * @framerate: frame rate in fps + * @ts_mode: temporal scalability mode (0: disable, 1: enable) + * support three temporal layers - 0: 7.5fps 1: 7.5fps 2: 15fps. + */ +struct venc_vp8_vpu_config { + u32 input_fourcc; + u32 bitrate; + u32 pic_w; + u32 pic_h; + u32 buf_w; + u32 buf_h; + u32 gop_size; + u32 framerate; + u32 ts_mode; +}; + +/* + * struct venc_vp8_vpu_buf -Structure for buffer information + * @align: buffer alignment (in bytes) + * @iova: IO virtual address + * @vpua: VPU side memory addr which is used by RC_CODE + * @size: buffer size (in bytes) + */ +struct venc_vp8_vpu_buf { + u32 align; + u32 iova; + u32 vpua; + u32 size; +}; + +/* + * struct venc_vp8_vsi - Structure for VPU driver control and info share + * This structure is allocated in VPU side and shared to AP side. + * @config: vp8 encoder configuration + * @work_bufs: working buffer information in VPU side + * The work_bufs here is for storing the 'size' info shared to AP side. + * The similar item in struct venc_vp8_inst is for memory allocation + * in AP side. The AP driver will copy the 'size' from here to the one in + * struct mtk_vcodec_mem, then invoke mtk_vcodec_mem_alloc to allocate + * the buffer. After that, bypass the 'dma_addr' to the 'iova' field here for + * register setting in VPU side. + */ +struct venc_vp8_vsi { + struct venc_vp8_vpu_config config; + struct venc_vp8_vpu_buf work_bufs[VENC_VP8_VPU_WORK_BUF_MAX]; +}; + +/* + * struct venc_vp8_inst - vp8 encoder AP driver instance + * @hw_base: vp8 encoder hardware register base + * @work_bufs: working buffer + * @work_buf_allocated: working buffer allocated flag + * @frm_cnt: encoded frame count, it's used for I-frame judgement and + * reset when force intra cmd received. + * @ts_mode: temporal scalability mode (0: disable, 1: enable) + * support three temporal layers - 0: 7.5fps 1: 7.5fps 2: 15fps. + * @vpu_inst: VPU instance to exchange information between AP and VPU + * @vsi: driver structure allocated by VPU side and shared to AP side for + * control and info share + * @ctx: context for v4l2 layer integration + */ +struct venc_vp8_inst { + void __iomem *hw_base; + struct mtk_vcodec_mem work_bufs[VENC_VP8_VPU_WORK_BUF_MAX]; + bool work_buf_allocated; + unsigned int frm_cnt; + unsigned int ts_mode; + struct venc_vpu_inst vpu_inst; + struct venc_vp8_vsi *vsi; + struct mtk_vcodec_ctx *ctx; +}; + +static inline void vp8_enc_write_reg(struct venc_vp8_inst *inst, u32 addr, + u32 val) +{ + writel(val, inst->hw_base + addr); +} + +static inline u32 vp8_enc_read_reg(struct venc_vp8_inst *inst, u32 addr) +{ + return readl(inst->hw_base + addr); +} + +static void vp8_enc_free_work_buf(struct venc_vp8_inst *inst) +{ + int i; + + mtk_vcodec_debug_enter(inst); + + /* Buffers need to be freed by AP. */ + for (i = 0; i < VENC_VP8_VPU_WORK_BUF_MAX; i++) { + if ((inst->work_bufs[i].size == 0)) + continue; + mtk_vcodec_mem_free(inst->ctx, &inst->work_bufs[i]); + } + + mtk_vcodec_debug_leave(inst); +} + +static int vp8_enc_alloc_work_buf(struct venc_vp8_inst *inst) +{ + int i; + int ret = 0; + struct venc_vp8_vpu_buf *wb = inst->vsi->work_bufs; + + mtk_vcodec_debug_enter(inst); + + for (i = 0; i < VENC_VP8_VPU_WORK_BUF_MAX; i++) { + if ((wb[i].size == 0)) + continue; + /* + * This 'wb' structure is set by VPU side and shared to AP for + * buffer allocation and IO virtual addr mapping. For most of + * the buffers, AP will allocate the buffer according to 'size' + * field and store the IO virtual addr in 'iova' field. For the + * RC_CODEx buffers, they are pre-allocated in the VPU side + * because they are inside VPU SRAM, and save the VPU addr in + * the 'vpua' field. The AP will translate the VPU addr to the + * corresponding IO virtual addr and store in 'iova' field. + */ + inst->work_bufs[i].size = wb[i].size; + ret = mtk_vcodec_mem_alloc(inst->ctx, &inst->work_bufs[i]); + if (ret) { + mtk_vcodec_err(inst, + "cannot alloc work_bufs[%d]", i); + goto err_alloc; + } + /* + * This RC_CODEx is pre-allocated by VPU and saved in VPU addr. + * So we need use memcpy to copy RC_CODEx from VPU addr into IO + * virtual addr in 'iova' field for reg setting in VPU side. + */ + if (i == VENC_VP8_VPU_WORK_BUF_RC_CODE || + i == VENC_VP8_VPU_WORK_BUF_RC_CODE2 || + i == VENC_VP8_VPU_WORK_BUF_RC_CODE3) { + void *tmp_va; + + tmp_va = vpu_mapping_dm_addr(inst->vpu_inst.dev, + wb[i].vpua); + memcpy(inst->work_bufs[i].va, tmp_va, wb[i].size); + } + wb[i].iova = inst->work_bufs[i].dma_addr; + + mtk_vcodec_debug(inst, + "work_bufs[%d] va=0x%p,iova=%pad,size=%zu", + i, inst->work_bufs[i].va, + &inst->work_bufs[i].dma_addr, + inst->work_bufs[i].size); + } + + mtk_vcodec_debug_leave(inst); + + return ret; + +err_alloc: + vp8_enc_free_work_buf(inst); + + return ret; +} + +static unsigned int vp8_enc_wait_venc_done(struct venc_vp8_inst *inst) +{ + unsigned int irq_status = 0; + struct mtk_vcodec_ctx *ctx = (struct mtk_vcodec_ctx *)inst->ctx; + + if (!mtk_vcodec_wait_for_done_ctx(ctx, MTK_INST_IRQ_RECEIVED, + WAIT_INTR_TIMEOUT_MS)) { + irq_status = ctx->irq_status; + mtk_vcodec_debug(inst, "isr return %x", irq_status); + } + return irq_status; +} + +/* + * Compose ac_tag, bitstream header and bitstream payload into + * one bitstream buffer. + */ +static int vp8_enc_compose_one_frame(struct venc_vp8_inst *inst, + struct mtk_vcodec_mem *bs_buf, + unsigned int *bs_size) +{ + unsigned int not_key; + u32 bs_frm_size; + u32 bs_hdr_len; + unsigned int ac_tag_size; + u8 ac_tag[MAX_AC_TAG_SIZE]; + u32 tag; + + bs_frm_size = vp8_enc_read_reg(inst, VENC_BITSTREAM_FRAME_SIZE); + bs_hdr_len = vp8_enc_read_reg(inst, VENC_BITSTREAM_HEADER_LEN); + + /* if a frame is key frame, not_key is 0 */ + not_key = !inst->vpu_inst.is_key_frm; + tag = (bs_hdr_len << 5) | 0x10 | not_key; + ac_tag[0] = tag & 0xff; + ac_tag[1] = (tag >> 8) & 0xff; + ac_tag[2] = (tag >> 16) & 0xff; + + /* key frame */ + if (not_key == 0) { + ac_tag_size = MAX_AC_TAG_SIZE; + ac_tag[3] = 0x9d; + ac_tag[4] = 0x01; + ac_tag[5] = 0x2a; + ac_tag[6] = inst->vsi->config.pic_w; + ac_tag[7] = inst->vsi->config.pic_w >> 8; + ac_tag[8] = inst->vsi->config.pic_h; + ac_tag[9] = inst->vsi->config.pic_h >> 8; + } else { + ac_tag_size = 3; + } + + if (bs_buf->size < bs_hdr_len + bs_frm_size + ac_tag_size) { + mtk_vcodec_err(inst, "bitstream buf size is too small(%zu)", + bs_buf->size); + return -EINVAL; + } + + /* + * (1) The vp8 bitstream header and body are generated by the HW vp8 + * encoder separately at the same time. We cannot know the bitstream + * header length in advance. + * (2) From the vp8 spec, there is no stuffing byte allowed between the + * ac tag, bitstream header and bitstream body. + */ + memmove(bs_buf->va + bs_hdr_len + ac_tag_size, + bs_buf->va, bs_frm_size); + memcpy(bs_buf->va + ac_tag_size, + inst->work_bufs[VENC_VP8_VPU_WORK_BUF_BS_HEADER].va, + bs_hdr_len); + memcpy(bs_buf->va, ac_tag, ac_tag_size); + *bs_size = bs_frm_size + bs_hdr_len + ac_tag_size; + + return 0; +} + +static int vp8_enc_encode_frame(struct venc_vp8_inst *inst, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + unsigned int *bs_size) +{ + int ret = 0; + unsigned int irq_status; + + mtk_vcodec_debug(inst, "->frm_cnt=%d", inst->frm_cnt); + + ret = vpu_enc_encode(&inst->vpu_inst, 0, frm_buf, bs_buf, bs_size); + if (ret) + return ret; + + irq_status = vp8_enc_wait_venc_done(inst); + if (irq_status != MTK_VENC_IRQ_STATUS_FRM) { + mtk_vcodec_err(inst, "irq_status=%d failed", irq_status); + return -EIO; + } + + if (vp8_enc_compose_one_frame(inst, bs_buf, bs_size)) { + mtk_vcodec_err(inst, "vp8_enc_compose_one_frame failed"); + return -EINVAL; + } + + inst->frm_cnt++; + mtk_vcodec_debug(inst, "<-size=%d key_frm=%d", *bs_size, + inst->vpu_inst.is_key_frm); + + return ret; +} + +static int vp8_enc_init(struct mtk_vcodec_ctx *ctx, unsigned long *handle) +{ + int ret = 0; + struct venc_vp8_inst *inst; + + inst = kzalloc(sizeof(*inst), GFP_KERNEL); + if (!inst) + return -ENOMEM; + + inst->ctx = ctx; + inst->vpu_inst.ctx = ctx; + inst->vpu_inst.dev = ctx->dev->vpu_plat_dev; + inst->vpu_inst.id = IPI_VENC_VP8; + inst->hw_base = mtk_vcodec_get_reg_addr(inst->ctx, VENC_LT_SYS); + + mtk_vcodec_debug_enter(inst); + + ret = vpu_enc_init(&inst->vpu_inst); + + inst->vsi = (struct venc_vp8_vsi *)inst->vpu_inst.vsi; + + mtk_vcodec_debug_leave(inst); + + if (ret) + kfree(inst); + else + (*handle) = (unsigned long)inst; + + return ret; +} + +static int vp8_enc_encode(unsigned long handle, + enum venc_start_opt opt, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + struct venc_done_result *result) +{ + int ret = 0; + struct venc_vp8_inst *inst = (struct venc_vp8_inst *)handle; + struct mtk_vcodec_ctx *ctx = inst->ctx; + + mtk_vcodec_debug_enter(inst); + + enable_irq(ctx->dev->enc_lt_irq); + + switch (opt) { + case VENC_START_OPT_ENCODE_FRAME: + ret = vp8_enc_encode_frame(inst, frm_buf, bs_buf, + &result->bs_size); + if (ret) + goto encode_err; + result->is_key_frm = inst->vpu_inst.is_key_frm; + break; + + default: + mtk_vcodec_err(inst, "opt not support:%d", opt); + ret = -EINVAL; + break; + } + +encode_err: + + disable_irq(ctx->dev->enc_lt_irq); + mtk_vcodec_debug_leave(inst); + + return ret; +} + +static int vp8_enc_set_param(unsigned long handle, + enum venc_set_param_type type, + struct venc_enc_param *enc_prm) +{ + int ret = 0; + struct venc_vp8_inst *inst = (struct venc_vp8_inst *)handle; + + mtk_vcodec_debug(inst, "->type=%d", type); + + switch (type) { + case VENC_SET_PARAM_ENC: + inst->vsi->config.input_fourcc = enc_prm->input_yuv_fmt; + inst->vsi->config.bitrate = enc_prm->bitrate; + inst->vsi->config.pic_w = enc_prm->width; + inst->vsi->config.pic_h = enc_prm->height; + inst->vsi->config.buf_w = enc_prm->buf_width; + inst->vsi->config.buf_h = enc_prm->buf_height; + inst->vsi->config.gop_size = enc_prm->gop_size; + inst->vsi->config.framerate = enc_prm->frm_rate; + inst->vsi->config.ts_mode = inst->ts_mode; + ret = vpu_enc_set_param(&inst->vpu_inst, type, enc_prm); + if (ret) + break; + if (inst->work_buf_allocated) { + vp8_enc_free_work_buf(inst); + inst->work_buf_allocated = false; + } + ret = vp8_enc_alloc_work_buf(inst); + if (ret) + break; + inst->work_buf_allocated = true; + break; + + /* + * VENC_SET_PARAM_TS_MODE must be called before VENC_SET_PARAM_ENC + */ + case VENC_SET_PARAM_TS_MODE: + inst->ts_mode = 1; + mtk_vcodec_debug(inst, "set ts_mode"); + break; + + default: + ret = vpu_enc_set_param(&inst->vpu_inst, type, enc_prm); + break; + } + + mtk_vcodec_debug_leave(inst); + + return ret; +} + +static int vp8_enc_deinit(unsigned long handle) +{ + int ret = 0; + struct venc_vp8_inst *inst = (struct venc_vp8_inst *)handle; + + mtk_vcodec_debug_enter(inst); + + ret = vpu_enc_deinit(&inst->vpu_inst); + + if (inst->work_buf_allocated) + vp8_enc_free_work_buf(inst); + + mtk_vcodec_debug_leave(inst); + kfree(inst); + + return ret; +} + +static struct venc_common_if venc_vp8_if = { + vp8_enc_init, + vp8_enc_encode, + vp8_enc_set_param, + vp8_enc_deinit, +}; + +struct venc_common_if *get_vp8_enc_comm_if(void); + +struct venc_common_if *get_vp8_enc_comm_if(void) +{ + return &venc_vp8_if; +} diff --git a/drivers/media/platform/mtk-vcodec/venc_drv_base.h b/drivers/media/platform/mtk-vcodec/venc_drv_base.h new file mode 100644 index 000000000000..6308d44dedf6 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/venc_drv_base.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016 MediaTek Inc. + * Author: Daniel Hsiao + * Jungchang Tsao + * Tiffany Lin + * + * This program is free software; you can redistribute it and/or + * modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _VENC_DRV_BASE_ +#define _VENC_DRV_BASE_ + +#include "mtk_vcodec_drv.h" + +#include "venc_drv_if.h" + +struct venc_common_if { + /** + * (*init)() - initialize driver + * @ctx: [in] mtk v4l2 context + * @handle: [out] driver handle + */ + int (*init)(struct mtk_vcodec_ctx *ctx, unsigned long *handle); + + /** + * (*encode)() - trigger encode + * @handle: [in] driver handle + * @opt: [in] encode option + * @frm_buf: [in] frame buffer to store input frame + * @bs_buf: [in] bitstream buffer to store output bitstream + * @result: [out] encode result + */ + int (*encode)(unsigned long handle, enum venc_start_opt opt, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + struct venc_done_result *result); + + /** + * (*set_param)() - set driver's parameter + * @handle: [in] driver handle + * @type: [in] parameter type + * @in: [in] buffer to store the parameter + */ + int (*set_param)(unsigned long handle, enum venc_set_param_type type, + struct venc_enc_param *in); + + /** + * (*deinit)() - deinitialize driver. + * @handle: [in] driver handle + */ + int (*deinit)(unsigned long handle); +}; + +#endif diff --git a/drivers/media/platform/mtk-vcodec/venc_drv_if.c b/drivers/media/platform/mtk-vcodec/venc_drv_if.c new file mode 100644 index 000000000000..c4c83e7189c3 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/venc_drv_if.c @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2016 MediaTek Inc. + * Author: Daniel Hsiao + * Jungchang Tsao + * Tiffany Lin + * + * This program is free software; you can redistribute it and/or + * modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include + +#include "venc_drv_base.h" +#include "venc_drv_if.h" + +#include "mtk_vcodec_enc.h" +#include "mtk_vcodec_enc_pm.h" +#include "mtk_vpu.h" + +struct venc_common_if *get_h264_enc_comm_if(void); +struct venc_common_if *get_vp8_enc_comm_if(void); + +int venc_if_init(struct mtk_vcodec_ctx *ctx, unsigned int fourcc) +{ + int ret = 0; + + switch (fourcc) { + case V4L2_PIX_FMT_VP8: + ctx->enc_if = get_vp8_enc_comm_if(); + break; + case V4L2_PIX_FMT_H264: + ctx->enc_if = get_h264_enc_comm_if(); + break; + default: + return -EINVAL; + } + + mtk_venc_lock(ctx); + mtk_vcodec_enc_clock_on(&ctx->dev->pm); + ret = ctx->enc_if->init(ctx, (unsigned long *)&ctx->drv_handle); + mtk_vcodec_enc_clock_off(&ctx->dev->pm); + mtk_venc_unlock(ctx); + + return ret; +} + +int venc_if_set_param(struct mtk_vcodec_ctx *ctx, + enum venc_set_param_type type, struct venc_enc_param *in) +{ + int ret = 0; + + mtk_venc_lock(ctx); + mtk_vcodec_enc_clock_on(&ctx->dev->pm); + ret = ctx->enc_if->set_param(ctx->drv_handle, type, in); + mtk_vcodec_enc_clock_off(&ctx->dev->pm); + mtk_venc_unlock(ctx); + + return ret; +} + +int venc_if_encode(struct mtk_vcodec_ctx *ctx, + enum venc_start_opt opt, struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + struct venc_done_result *result) +{ + int ret = 0; + unsigned long flags; + + mtk_venc_lock(ctx); + + spin_lock_irqsave(&ctx->dev->irqlock, flags); + ctx->dev->curr_ctx = ctx; + spin_unlock_irqrestore(&ctx->dev->irqlock, flags); + + mtk_vcodec_enc_clock_on(&ctx->dev->pm); + ret = ctx->enc_if->encode(ctx->drv_handle, opt, frm_buf, + bs_buf, result); + mtk_vcodec_enc_clock_off(&ctx->dev->pm); + + spin_lock_irqsave(&ctx->dev->irqlock, flags); + ctx->dev->curr_ctx = NULL; + spin_unlock_irqrestore(&ctx->dev->irqlock, flags); + + mtk_venc_unlock(ctx); + return ret; +} + +int venc_if_deinit(struct mtk_vcodec_ctx *ctx) +{ + int ret = 0; + + if (ctx->drv_handle == 0) + return 0; + + mtk_venc_lock(ctx); + mtk_vcodec_enc_clock_on(&ctx->dev->pm); + ret = ctx->enc_if->deinit(ctx->drv_handle); + mtk_vcodec_enc_clock_off(&ctx->dev->pm); + mtk_venc_unlock(ctx); + + ctx->drv_handle = 0; + + return ret; +} diff --git a/drivers/media/platform/mtk-vcodec/venc_drv_if.h b/drivers/media/platform/mtk-vcodec/venc_drv_if.h new file mode 100644 index 000000000000..a6e7d32e55cb --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/venc_drv_if.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2016 MediaTek Inc. + * Author: Daniel Hsiao + * Jungchang Tsao + * Tiffany Lin + * + * This program is free software; you can redistribute it and/or + * modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _VENC_DRV_IF_H_ +#define _VENC_DRV_IF_H_ + +#include "mtk_vcodec_drv.h" +#include "mtk_vcodec_util.h" + +/* + * enum venc_yuv_fmt - The type of input yuv format + * (VPU related: If you change the order, you must also update the VPU codes.) + * @VENC_YUV_FORMAT_I420: I420 YUV format + * @VENC_YUV_FORMAT_YV12: YV12 YUV format + * @VENC_YUV_FORMAT_NV12: NV12 YUV format + * @VENC_YUV_FORMAT_NV21: NV21 YUV format + */ +enum venc_yuv_fmt { + VENC_YUV_FORMAT_I420 = 3, + VENC_YUV_FORMAT_YV12 = 5, + VENC_YUV_FORMAT_NV12 = 6, + VENC_YUV_FORMAT_NV21 = 7, +}; + +/* + * enum venc_start_opt - encode frame option used in venc_if_encode() + * @VENC_START_OPT_ENCODE_SEQUENCE_HEADER: encode SPS/PPS for H264 + * @VENC_START_OPT_ENCODE_FRAME: encode normal frame + */ +enum venc_start_opt { + VENC_START_OPT_ENCODE_SEQUENCE_HEADER, + VENC_START_OPT_ENCODE_FRAME, +}; + +/* + * enum venc_set_param_type - The type of set parameter used in + * venc_if_set_param() + * (VPU related: If you change the order, you must also update the VPU codes.) + * @VENC_SET_PARAM_ENC: set encoder parameters + * @VENC_SET_PARAM_FORCE_INTRA: force an intra frame + * @VENC_SET_PARAM_ADJUST_BITRATE: adjust bitrate (in bps) + * @VENC_SET_PARAM_ADJUST_FRAMERATE: set frame rate + * @VENC_SET_PARAM_GOP_SIZE: set IDR interval + * @VENC_SET_PARAM_INTRA_PERIOD: set I frame interval + * @VENC_SET_PARAM_SKIP_FRAME: set H264 skip one frame + * @VENC_SET_PARAM_PREPEND_HEADER: set H264 prepend SPS/PPS before IDR + * @VENC_SET_PARAM_TS_MODE: set VP8 temporal scalability mode + */ +enum venc_set_param_type { + VENC_SET_PARAM_ENC, + VENC_SET_PARAM_FORCE_INTRA, + VENC_SET_PARAM_ADJUST_BITRATE, + VENC_SET_PARAM_ADJUST_FRAMERATE, + VENC_SET_PARAM_GOP_SIZE, + VENC_SET_PARAM_INTRA_PERIOD, + VENC_SET_PARAM_SKIP_FRAME, + VENC_SET_PARAM_PREPEND_HEADER, + VENC_SET_PARAM_TS_MODE, +}; + +/* + * struct venc_enc_prm - encoder settings for VENC_SET_PARAM_ENC used in + * venc_if_set_param() + * @input_fourcc: input yuv format + * @h264_profile: V4L2 defined H.264 profile + * @h264_level: V4L2 defined H.264 level + * @width: image width + * @height: image height + * @buf_width: buffer width + * @buf_height: buffer height + * @frm_rate: frame rate in fps + * @intra_period: intra frame period + * @bitrate: target bitrate in bps + * @gop_size: group of picture size + */ +struct venc_enc_param { + enum venc_yuv_fmt input_yuv_fmt; + unsigned int h264_profile; + unsigned int h264_level; + unsigned int width; + unsigned int height; + unsigned int buf_width; + unsigned int buf_height; + unsigned int frm_rate; + unsigned int intra_period; + unsigned int bitrate; + unsigned int gop_size; +}; + +/* + * struct venc_frm_buf - frame buffer information used in venc_if_encode() + * @fb_addr: plane frame buffer addresses + */ +struct venc_frm_buf { + struct mtk_vcodec_mem fb_addr[MTK_VCODEC_MAX_PLANES]; +}; + +/* + * struct venc_done_result - This is return information used in venc_if_encode() + * @bs_size: output bitstream size + * @is_key_frm: output is key frame or not + */ +struct venc_done_result { + unsigned int bs_size; + bool is_key_frm; +}; + +/* + * venc_if_init - Create the driver handle + * @ctx: device context + * @fourcc: encoder input format + * Return: 0 if creating handle successfully, otherwise it is failed. + */ +int venc_if_init(struct mtk_vcodec_ctx *ctx, unsigned int fourcc); + +/* + * venc_if_deinit - Release the driver handle + * @ctx: device context + * Return: 0 if releasing handle successfully, otherwise it is failed. + */ +int venc_if_deinit(struct mtk_vcodec_ctx *ctx); + +/* + * venc_if_set_param - Set parameter to driver + * @ctx: device context + * @type: parameter type + * @in: input parameter + * Return: 0 if setting param successfully, otherwise it is failed. + */ +int venc_if_set_param(struct mtk_vcodec_ctx *ctx, + enum venc_set_param_type type, + struct venc_enc_param *in); + +/* + * venc_if_encode - Encode one frame + * @ctx: device context + * @opt: encode frame option + * @frm_buf: input frame buffer information + * @bs_buf: output bitstream buffer infomraiton + * @result: encode result + * Return: 0 if encoding frame successfully, otherwise it is failed. + */ +int venc_if_encode(struct mtk_vcodec_ctx *ctx, + enum venc_start_opt opt, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + struct venc_done_result *result); + +#endif /* _VENC_DRV_IF_H_ */ diff --git a/drivers/media/platform/mtk-vcodec/venc_ipi_msg.h b/drivers/media/platform/mtk-vcodec/venc_ipi_msg.h new file mode 100644 index 000000000000..4c869cb6fbf7 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/venc_ipi_msg.h @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2016 MediaTek Inc. + * Author: Jungchang Tsao + * Daniel Hsiao + * Tiffany Lin + * + * This program is free software; you can redistribute it and/or + * modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _VENC_IPI_MSG_H_ +#define _VENC_IPI_MSG_H_ + +#define AP_IPIMSG_VENC_BASE 0xC000 +#define VPU_IPIMSG_VENC_BASE 0xD000 + +/** + * enum venc_ipi_msg_id - message id between AP and VPU + * (ipi stands for inter-processor interrupt) + * @AP_IPIMSG_ENC_XXX: AP to VPU cmd message id + * @VPU_IPIMSG_ENC_XXX_DONE: VPU ack AP cmd message id + */ +enum venc_ipi_msg_id { + AP_IPIMSG_ENC_INIT = AP_IPIMSG_VENC_BASE, + AP_IPIMSG_ENC_SET_PARAM, + AP_IPIMSG_ENC_ENCODE, + AP_IPIMSG_ENC_DEINIT, + + VPU_IPIMSG_ENC_INIT_DONE = VPU_IPIMSG_VENC_BASE, + VPU_IPIMSG_ENC_SET_PARAM_DONE, + VPU_IPIMSG_ENC_ENCODE_DONE, + VPU_IPIMSG_ENC_DEINIT_DONE, +}; + +/** + * struct venc_ap_ipi_msg_init - AP to VPU init cmd structure + * @msg_id: message id (AP_IPIMSG_XXX_ENC_INIT) + * @reserved: reserved for future use. vpu is running in 32bit. Without + * this reserved field, if kernel run in 64bit. this struct size + * will be different between kernel and vpu + * @venc_inst: AP encoder instance + * (struct venc_vp8_inst/venc_h264_inst *) + */ +struct venc_ap_ipi_msg_init { + uint32_t msg_id; + uint32_t reserved; + uint64_t venc_inst; +}; + +/** + * struct venc_ap_ipi_msg_set_param - AP to VPU set_param cmd structure + * @msg_id: message id (AP_IPIMSG_XXX_ENC_SET_PARAM) + * @vpu_inst_addr: VPU encoder instance addr + * (struct venc_vp8_vsi/venc_h264_vsi *) + * @param_id: parameter id (venc_set_param_type) + * @data_item: number of items in the data array + * @data[8]: data array to store the set parameters + */ +struct venc_ap_ipi_msg_set_param { + uint32_t msg_id; + uint32_t vpu_inst_addr; + uint32_t param_id; + uint32_t data_item; + uint32_t data[8]; +}; + +/** + * struct venc_ap_ipi_msg_enc - AP to VPU enc cmd structure + * @msg_id: message id (AP_IPIMSG_XXX_ENC_ENCODE) + * @vpu_inst_addr: VPU encoder instance addr + * (struct venc_vp8_vsi/venc_h264_vsi *) + * @bs_mode: bitstream mode for h264 + * (H264_BS_MODE_SPS/H264_BS_MODE_PPS/H264_BS_MODE_FRAME) + * @input_addr: pointer to input image buffer plane + * @bs_addr: pointer to output bit stream buffer + * @bs_size: bit stream buffer size + */ +struct venc_ap_ipi_msg_enc { + uint32_t msg_id; + uint32_t vpu_inst_addr; + uint32_t bs_mode; + uint32_t input_addr[3]; + uint32_t bs_addr; + uint32_t bs_size; +}; + +/** + * struct venc_ap_ipi_msg_deinit - AP to VPU deinit cmd structure + * @msg_id: message id (AP_IPIMSG_XXX_ENC_DEINIT) + * @vpu_inst_addr: VPU encoder instance addr + * (struct venc_vp8_vsi/venc_h264_vsi *) + */ +struct venc_ap_ipi_msg_deinit { + uint32_t msg_id; + uint32_t vpu_inst_addr; +}; + +/** + * enum venc_ipi_msg_status - VPU ack AP cmd status + */ +enum venc_ipi_msg_status { + VENC_IPI_MSG_STATUS_OK, + VENC_IPI_MSG_STATUS_FAIL, +}; + +/** + * struct venc_vpu_ipi_msg_common - VPU ack AP cmd common structure + * @msg_id: message id (VPU_IPIMSG_XXX_DONE) + * @status: cmd status (venc_ipi_msg_status) + * @venc_inst: AP encoder instance (struct venc_vp8_inst/venc_h264_inst *) + */ +struct venc_vpu_ipi_msg_common { + uint32_t msg_id; + uint32_t status; + uint64_t venc_inst; +}; + +/** + * struct venc_vpu_ipi_msg_init - VPU ack AP init cmd structure + * @msg_id: message id (VPU_IPIMSG_XXX_ENC_SET_PARAM_DONE) + * @status: cmd status (venc_ipi_msg_status) + * @venc_inst: AP encoder instance (struct venc_vp8_inst/venc_h264_inst *) + * @vpu_inst_addr: VPU encoder instance addr + * (struct venc_vp8_vsi/venc_h264_vsi *) + * @reserved: reserved for future use. vpu is running in 32bit. Without + * this reserved field, if kernel run in 64bit. this struct size + * will be different between kernel and vpu + */ +struct venc_vpu_ipi_msg_init { + uint32_t msg_id; + uint32_t status; + uint64_t venc_inst; + uint32_t vpu_inst_addr; + uint32_t reserved; +}; + +/** + * struct venc_vpu_ipi_msg_set_param - VPU ack AP set_param cmd structure + * @msg_id: message id (VPU_IPIMSG_XXX_ENC_SET_PARAM_DONE) + * @status: cmd status (venc_ipi_msg_status) + * @venc_inst: AP encoder instance (struct venc_vp8_inst/venc_h264_inst *) + * @param_id: parameter id (venc_set_param_type) + * @data_item: number of items in the data array + * @data[6]: data array to store the return result + */ +struct venc_vpu_ipi_msg_set_param { + uint32_t msg_id; + uint32_t status; + uint64_t venc_inst; + uint32_t param_id; + uint32_t data_item; + uint32_t data[6]; +}; + +/** + * enum venc_ipi_msg_enc_state - Type of encode state + * VEN_IPI_MSG_ENC_STATE_FRAME: one frame being encoded + * VEN_IPI_MSG_ENC_STATE_PART: bit stream buffer full + * VEN_IPI_MSG_ENC_STATE_SKIP: encoded skip frame + * VEN_IPI_MSG_ENC_STATE_ERROR: encounter error + */ +enum venc_ipi_msg_enc_state { + VEN_IPI_MSG_ENC_STATE_FRAME, + VEN_IPI_MSG_ENC_STATE_PART, + VEN_IPI_MSG_ENC_STATE_SKIP, + VEN_IPI_MSG_ENC_STATE_ERROR, +}; + +/** + * struct venc_vpu_ipi_msg_enc - VPU ack AP enc cmd structure + * @msg_id: message id (VPU_IPIMSG_XXX_ENC_ENCODE_DONE) + * @status: cmd status (venc_ipi_msg_status) + * @venc_inst: AP encoder instance (struct venc_vp8_inst/venc_h264_inst *) + * @state: encode state (venc_ipi_msg_enc_state) + * @is_key_frm: whether the encoded frame is key frame + * @bs_size: encoded bitstream size + * @reserved: reserved for future use. vpu is running in 32bit. Without + * this reserved field, if kernel run in 64bit. this struct size + * will be different between kernel and vpu + */ +struct venc_vpu_ipi_msg_enc { + uint32_t msg_id; + uint32_t status; + uint64_t venc_inst; + uint32_t state; + uint32_t is_key_frm; + uint32_t bs_size; + uint32_t reserved; +}; + +/** + * struct venc_vpu_ipi_msg_deinit - VPU ack AP deinit cmd structure + * @msg_id: message id (VPU_IPIMSG_XXX_ENC_DEINIT_DONE) + * @status: cmd status (venc_ipi_msg_status) + * @venc_inst: AP encoder instance (struct venc_vp8_inst/venc_h264_inst *) + */ +struct venc_vpu_ipi_msg_deinit { + uint32_t msg_id; + uint32_t status; + uint64_t venc_inst; +}; + +#endif /* _VENC_IPI_MSG_H_ */ diff --git a/drivers/media/platform/mtk-vcodec/venc_vpu_if.c b/drivers/media/platform/mtk-vcodec/venc_vpu_if.c new file mode 100644 index 000000000000..a01c7599b510 --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/venc_vpu_if.c @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2016 MediaTek Inc. + * Author: PoChun Lin + * + * This program is free software; you can redistribute it and/or + * modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "mtk_vpu.h" +#include "venc_ipi_msg.h" +#include "venc_vpu_if.h" + +static void handle_enc_init_msg(struct venc_vpu_inst *vpu, void *data) +{ + struct venc_vpu_ipi_msg_init *msg = data; + + vpu->inst_addr = msg->vpu_inst_addr; + vpu->vsi = vpu_mapping_dm_addr(vpu->dev, msg->vpu_inst_addr); +} + +static void handle_enc_encode_msg(struct venc_vpu_inst *vpu, void *data) +{ + struct venc_vpu_ipi_msg_enc *msg = data; + + vpu->state = msg->state; + vpu->bs_size = msg->bs_size; + vpu->is_key_frm = msg->is_key_frm; +} + +static void vpu_enc_ipi_handler(void *data, unsigned int len, void *priv) +{ + struct venc_vpu_ipi_msg_common *msg = data; + struct venc_vpu_inst *vpu = + (struct venc_vpu_inst *)(unsigned long)msg->venc_inst; + + mtk_vcodec_debug(vpu, "msg_id %x inst %p status %d", + msg->msg_id, vpu, msg->status); + + switch (msg->msg_id) { + case VPU_IPIMSG_ENC_INIT_DONE: + handle_enc_init_msg(vpu, data); + break; + case VPU_IPIMSG_ENC_SET_PARAM_DONE: + break; + case VPU_IPIMSG_ENC_ENCODE_DONE: + handle_enc_encode_msg(vpu, data); + break; + case VPU_IPIMSG_ENC_DEINIT_DONE: + break; + default: + mtk_vcodec_err(vpu, "unknown msg id %x", msg->msg_id); + break; + } + + vpu->signaled = 1; + vpu->failure = (msg->status != VENC_IPI_MSG_STATUS_OK); + + mtk_vcodec_debug_leave(vpu); +} + +static int vpu_enc_send_msg(struct venc_vpu_inst *vpu, void *msg, + int len) +{ + int status; + + mtk_vcodec_debug_enter(vpu); + + if (!vpu->dev) { + mtk_vcodec_err(vpu, "inst dev is NULL"); + return -EINVAL; + } + + status = vpu_ipi_send(vpu->dev, vpu->id, msg, len); + if (status) { + uint32_t msg_id = *(uint32_t *)msg; + + mtk_vcodec_err(vpu, "vpu_ipi_send msg_id %x len %d fail %d", + msg_id, len, status); + return -EINVAL; + } + if (vpu->failure) + return -EINVAL; + + mtk_vcodec_debug_leave(vpu); + + return 0; +} + +int vpu_enc_init(struct venc_vpu_inst *vpu) +{ + int status; + struct venc_ap_ipi_msg_init out; + + mtk_vcodec_debug_enter(vpu); + + init_waitqueue_head(&vpu->wq_hd); + vpu->signaled = 0; + vpu->failure = 0; + + status = vpu_ipi_register(vpu->dev, vpu->id, vpu_enc_ipi_handler, + NULL, NULL); + if (status) { + mtk_vcodec_err(vpu, "vpu_ipi_register fail %d", status); + return -EINVAL; + } + + memset(&out, 0, sizeof(out)); + out.msg_id = AP_IPIMSG_ENC_INIT; + out.venc_inst = (unsigned long)vpu; + if (vpu_enc_send_msg(vpu, &out, sizeof(out))) { + mtk_vcodec_err(vpu, "AP_IPIMSG_ENC_INIT fail"); + return -EINVAL; + } + + mtk_vcodec_debug_leave(vpu); + + return 0; +} + +int vpu_enc_set_param(struct venc_vpu_inst *vpu, + enum venc_set_param_type id, + struct venc_enc_param *enc_param) +{ + struct venc_ap_ipi_msg_set_param out; + + mtk_vcodec_debug(vpu, "id %d ->", id); + + memset(&out, 0, sizeof(out)); + out.msg_id = AP_IPIMSG_ENC_SET_PARAM; + out.vpu_inst_addr = vpu->inst_addr; + out.param_id = id; + switch (id) { + case VENC_SET_PARAM_ENC: + out.data_item = 0; + break; + case VENC_SET_PARAM_FORCE_INTRA: + out.data_item = 0; + break; + case VENC_SET_PARAM_ADJUST_BITRATE: + out.data_item = 1; + out.data[0] = enc_param->bitrate; + break; + case VENC_SET_PARAM_ADJUST_FRAMERATE: + out.data_item = 1; + out.data[0] = enc_param->frm_rate; + break; + case VENC_SET_PARAM_GOP_SIZE: + out.data_item = 1; + out.data[0] = enc_param->gop_size; + break; + case VENC_SET_PARAM_INTRA_PERIOD: + out.data_item = 1; + out.data[0] = enc_param->intra_period; + break; + case VENC_SET_PARAM_SKIP_FRAME: + out.data_item = 0; + break; + default: + mtk_vcodec_err(vpu, "id %d not supported", id); + return -EINVAL; + } + if (vpu_enc_send_msg(vpu, &out, sizeof(out))) { + mtk_vcodec_err(vpu, + "AP_IPIMSG_ENC_SET_PARAM %d fail", id); + return -EINVAL; + } + + mtk_vcodec_debug(vpu, "id %d <-", id); + + return 0; +} + +int vpu_enc_encode(struct venc_vpu_inst *vpu, unsigned int bs_mode, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + unsigned int *bs_size) +{ + struct venc_ap_ipi_msg_enc out; + + mtk_vcodec_debug(vpu, "bs_mode %d ->", bs_mode); + + memset(&out, 0, sizeof(out)); + out.msg_id = AP_IPIMSG_ENC_ENCODE; + out.vpu_inst_addr = vpu->inst_addr; + out.bs_mode = bs_mode; + if (frm_buf) { + if ((frm_buf->fb_addr[0].dma_addr % 16 == 0) && + (frm_buf->fb_addr[1].dma_addr % 16 == 0) && + (frm_buf->fb_addr[2].dma_addr % 16 == 0)) { + out.input_addr[0] = frm_buf->fb_addr[0].dma_addr; + out.input_addr[1] = frm_buf->fb_addr[1].dma_addr; + out.input_addr[2] = frm_buf->fb_addr[2].dma_addr; + } else { + mtk_vcodec_err(vpu, "dma_addr not align to 16"); + return -EINVAL; + } + } + if (bs_buf) { + out.bs_addr = bs_buf->dma_addr; + out.bs_size = bs_buf->size; + } + if (vpu_enc_send_msg(vpu, &out, sizeof(out))) { + mtk_vcodec_err(vpu, "AP_IPIMSG_ENC_ENCODE %d fail", + bs_mode); + return -EINVAL; + } + + mtk_vcodec_debug(vpu, "bs_mode %d state %d size %d key_frm %d <-", + bs_mode, vpu->state, vpu->bs_size, vpu->is_key_frm); + + return 0; +} + +int vpu_enc_deinit(struct venc_vpu_inst *vpu) +{ + struct venc_ap_ipi_msg_deinit out; + + mtk_vcodec_debug_enter(vpu); + + memset(&out, 0, sizeof(out)); + out.msg_id = AP_IPIMSG_ENC_DEINIT; + out.vpu_inst_addr = vpu->inst_addr; + if (vpu_enc_send_msg(vpu, &out, sizeof(out))) { + mtk_vcodec_err(vpu, "AP_IPIMSG_ENC_DEINIT fail"); + return -EINVAL; + } + + mtk_vcodec_debug_leave(vpu); + + return 0; +} diff --git a/drivers/media/platform/mtk-vcodec/venc_vpu_if.h b/drivers/media/platform/mtk-vcodec/venc_vpu_if.h new file mode 100644 index 000000000000..215d1e01362e --- /dev/null +++ b/drivers/media/platform/mtk-vcodec/venc_vpu_if.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016 MediaTek Inc. + * Author: PoChun Lin + * + * This program is free software; you can redistribute it and/or + * modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _VENC_VPU_IF_H_ +#define _VENC_VPU_IF_H_ + +#include "mtk_vpu.h" +#include "venc_drv_if.h" + +/* + * struct venc_vpu_inst - encoder VPU driver instance + * @wq_hd: wait queue used for vpu cmd trigger then wait vpu interrupt done + * @signaled: flag used for checking vpu interrupt done + * @failure: flag to show vpu cmd succeeds or not + * @state: enum venc_ipi_msg_enc_state + * @bs_size: bitstream size for skip frame case usage + * @is_key_frm: key frame flag + * @inst_addr: VPU instance addr + * @vsi: driver structure allocated by VPU side and shared to AP side for + * control and info share + * @id: the id of inter-processor interrupt + * @ctx: context for v4l2 layer integration + * @dev: device for v4l2 layer integration + */ +struct venc_vpu_inst { + wait_queue_head_t wq_hd; + int signaled; + int failure; + int state; + int bs_size; + int is_key_frm; + unsigned int inst_addr; + void *vsi; + enum ipi_id id; + struct mtk_vcodec_ctx *ctx; + struct platform_device *dev; +}; + +int vpu_enc_init(struct venc_vpu_inst *vpu); +int vpu_enc_set_param(struct venc_vpu_inst *vpu, + enum venc_set_param_type id, + struct venc_enc_param *param); +int vpu_enc_encode(struct venc_vpu_inst *vpu, unsigned int bs_mode, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + unsigned int *bs_size); +int vpu_enc_deinit(struct venc_vpu_inst *vpu); + +#endif diff --git a/drivers/media/platform/mtk-vpu/Makefile b/drivers/media/platform/mtk-vpu/Makefile new file mode 100644 index 000000000000..58cc1b4bc9f2 --- /dev/null +++ b/drivers/media/platform/mtk-vpu/Makefile @@ -0,0 +1,3 @@ +mtk-vpu-y += mtk_vpu.o + +obj-$(CONFIG_VIDEO_MEDIATEK_VPU) += mtk-vpu.o diff --git a/drivers/media/platform/mtk-vpu/mtk_vpu.c b/drivers/media/platform/mtk-vpu/mtk_vpu.c new file mode 100644 index 000000000000..c9bf58c97878 --- /dev/null +++ b/drivers/media/platform/mtk-vpu/mtk_vpu.c @@ -0,0 +1,946 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: Andrew-CT Chen +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mtk_vpu.h" + +/** + * VPU (video processor unit) is a tiny processor controlling video hardware + * related to video codec, scaling and color format converting. + * VPU interfaces with other blocks by share memory and interrupt. + **/ + +#define INIT_TIMEOUT_MS 2000U +#define IPI_TIMEOUT_MS 2000U +#define VPU_FW_VER_LEN 16 + +/* maximum program/data TCM (Tightly-Coupled Memory) size */ +#define VPU_PTCM_SIZE (96 * SZ_1K) +#define VPU_DTCM_SIZE (32 * SZ_1K) +/* the offset to get data tcm address */ +#define VPU_DTCM_OFFSET 0x18000UL +/* daynamic allocated maximum extended memory size */ +#define VPU_EXT_P_SIZE SZ_1M +#define VPU_EXT_D_SIZE SZ_4M +/* maximum binary firmware size */ +#define VPU_P_FW_SIZE (VPU_PTCM_SIZE + VPU_EXT_P_SIZE) +#define VPU_D_FW_SIZE (VPU_DTCM_SIZE + VPU_EXT_D_SIZE) +/* the size of share buffer between Host and VPU */ +#define SHARE_BUF_SIZE 48 + +/* binary firmware name */ +#define VPU_P_FW "vpu_p.bin" +#define VPU_D_FW "vpu_d.bin" + +#define VPU_RESET 0x0 +#define VPU_TCM_CFG 0x0008 +#define VPU_PMEM_EXT0_ADDR 0x000C +#define VPU_PMEM_EXT1_ADDR 0x0010 +#define VPU_TO_HOST 0x001C +#define VPU_DMEM_EXT0_ADDR 0x0014 +#define VPU_DMEM_EXT1_ADDR 0x0018 +#define HOST_TO_VPU 0x0024 +#define VPU_PC_REG 0x0060 +#define VPU_WDT_REG 0x0084 + +/* vpu inter-processor communication interrupt */ +#define VPU_IPC_INT BIT(8) + +/** + * enum vpu_fw_type - VPU firmware type + * + * @P_FW: program firmware + * @D_FW: data firmware + * + */ +enum vpu_fw_type { + P_FW, + D_FW, +}; + +/** + * struct vpu_mem - VPU extended program/data memory information + * + * @va: the kernel virtual memory address of VPU extended memory + * @pa: the physical memory address of VPU extended memory + * + */ +struct vpu_mem { + void *va; + dma_addr_t pa; +}; + +/** + * struct vpu_regs - VPU TCM and configuration registers + * + * @tcm: the register for VPU Tightly-Coupled Memory + * @cfg: the register for VPU configuration + * @irq: the irq number for VPU interrupt + */ +struct vpu_regs { + void __iomem *tcm; + void __iomem *cfg; + int irq; +}; + +/** + * struct vpu_wdt_handler - VPU watchdog reset handler + * + * @reset_func: reset handler + * @priv: private data + */ +struct vpu_wdt_handler { + void (*reset_func)(void *); + void *priv; +}; + +/** + * struct vpu_wdt - VPU watchdog workqueue + * + * @handler: VPU watchdog reset handler + * @ws: workstruct for VPU watchdog + * @wq: workqueue for VPU watchdog + */ +struct vpu_wdt { + struct vpu_wdt_handler handler[VPU_RST_MAX]; + struct work_struct ws; + struct workqueue_struct *wq; +}; + +/** + * struct vpu_run - VPU initialization status + * + * @signaled: the signal of vpu initialization completed + * @fw_ver: VPU firmware version + * @enc_capability: encoder capability which is not used for now and + * the value is reserved for future use + * @wq: wait queue for VPU initialization status + */ +struct vpu_run { + u32 signaled; + char fw_ver[VPU_FW_VER_LEN]; + unsigned int enc_capability; + wait_queue_head_t wq; +}; + +/** + * struct vpu_ipi_desc - VPU IPI descriptor + * + * @handler: IPI handler + * @name: the name of IPI handler + * @priv: the private data of IPI handler + */ +struct vpu_ipi_desc { + ipi_handler_t handler; + const char *name; + void *priv; +}; + +/** + * struct share_obj - DTCM (Data Tightly-Coupled Memory) buffer shared with + * AP and VPU + * + * @id: IPI id + * @len: share buffer length + * @share_buf: share buffer data + */ +struct share_obj { + s32 id; + u32 len; + unsigned char share_buf[SHARE_BUF_SIZE]; +}; + +/** + * struct mtk_vpu - vpu driver data + * @extmem: VPU extended memory information + * @reg: VPU TCM and configuration registers + * @run: VPU initialization status + * @ipi_desc: VPU IPI descriptor + * @recv_buf: VPU DTCM share buffer for receiving. The + * receive buffer is only accessed in interrupt context. + * @send_buf: VPU DTCM share buffer for sending + * @dev: VPU struct device + * @clk: VPU clock on/off + * @fw_loaded: indicate VPU firmware loaded + * @enable_4GB: VPU 4GB mode on/off + * @vpu_mutex: protect mtk_vpu (except recv_buf) and ensure only + * one client to use VPU service at a time. For example, + * suppose a client is using VPU to decode VP8. + * If the other client wants to encode VP8, + * it has to wait until VP8 decode completes. + * @wdt_refcnt WDT reference count to make sure the watchdog can be + * disabled if no other client is using VPU service + * @ack_wq: The wait queue for each codec and mdp. When sleeping + * processes wake up, they will check the condition + * "ipi_id_ack" to run the corresponding action or + * go back to sleep. + * @ipi_id_ack: The ACKs for registered IPI function sending + * interrupt to VPU + * + */ +struct mtk_vpu { + struct vpu_mem extmem[2]; + struct vpu_regs reg; + struct vpu_run run; + struct vpu_wdt wdt; + struct vpu_ipi_desc ipi_desc[IPI_MAX]; + struct share_obj *recv_buf; + struct share_obj *send_buf; + struct device *dev; + struct clk *clk; + bool fw_loaded; + bool enable_4GB; + struct mutex vpu_mutex; /* for protecting vpu data data structure */ + u32 wdt_refcnt; + wait_queue_head_t ack_wq; + bool ipi_id_ack[IPI_MAX]; +}; + +static inline void vpu_cfg_writel(struct mtk_vpu *vpu, u32 val, u32 offset) +{ + writel(val, vpu->reg.cfg + offset); +} + +static inline u32 vpu_cfg_readl(struct mtk_vpu *vpu, u32 offset) +{ + return readl(vpu->reg.cfg + offset); +} + +static inline bool vpu_running(struct mtk_vpu *vpu) +{ + return vpu_cfg_readl(vpu, VPU_RESET) & BIT(0); +} + +static void vpu_clock_disable(struct mtk_vpu *vpu) +{ + /* Disable VPU watchdog */ + mutex_lock(&vpu->vpu_mutex); + if (!--vpu->wdt_refcnt) + vpu_cfg_writel(vpu, + vpu_cfg_readl(vpu, VPU_WDT_REG) & ~(1L << 31), + VPU_WDT_REG); + mutex_unlock(&vpu->vpu_mutex); + + clk_disable(vpu->clk); +} + +static int vpu_clock_enable(struct mtk_vpu *vpu) +{ + int ret; + + ret = clk_enable(vpu->clk); + if (ret) + return ret; + /* Enable VPU watchdog */ + mutex_lock(&vpu->vpu_mutex); + if (!vpu->wdt_refcnt++) + vpu_cfg_writel(vpu, + vpu_cfg_readl(vpu, VPU_WDT_REG) | (1L << 31), + VPU_WDT_REG); + mutex_unlock(&vpu->vpu_mutex); + + return ret; +} + +int vpu_ipi_register(struct platform_device *pdev, + enum ipi_id id, ipi_handler_t handler, + const char *name, void *priv) +{ + struct mtk_vpu *vpu = platform_get_drvdata(pdev); + struct vpu_ipi_desc *ipi_desc; + + if (!vpu) { + dev_err(&pdev->dev, "vpu device in not ready\n"); + return -EPROBE_DEFER; + } + + if (id >= 0 && id < IPI_MAX && handler) { + ipi_desc = vpu->ipi_desc; + ipi_desc[id].name = name; + ipi_desc[id].handler = handler; + ipi_desc[id].priv = priv; + return 0; + } + + dev_err(&pdev->dev, "register vpu ipi id %d with invalid arguments\n", + id); + return -EINVAL; +} +EXPORT_SYMBOL_GPL(vpu_ipi_register); + +int vpu_ipi_send(struct platform_device *pdev, + enum ipi_id id, void *buf, + unsigned int len) +{ + struct mtk_vpu *vpu = platform_get_drvdata(pdev); + struct share_obj *send_obj = vpu->send_buf; + unsigned long timeout; + int ret = 0; + + if (id <= IPI_VPU_INIT || id >= IPI_MAX || + len > sizeof(send_obj->share_buf) || !buf) { + dev_err(vpu->dev, "failed to send ipi message\n"); + return -EINVAL; + } + + ret = vpu_clock_enable(vpu); + if (ret) { + dev_err(vpu->dev, "failed to enable vpu clock\n"); + return ret; + } + if (!vpu_running(vpu)) { + dev_err(vpu->dev, "vpu_ipi_send: VPU is not running\n"); + ret = -EINVAL; + goto clock_disable; + } + + mutex_lock(&vpu->vpu_mutex); + + /* Wait until VPU receives the last command */ + timeout = jiffies + msecs_to_jiffies(IPI_TIMEOUT_MS); + do { + if (time_after(jiffies, timeout)) { + dev_err(vpu->dev, "vpu_ipi_send: IPI timeout!\n"); + ret = -EIO; + goto mut_unlock; + } + } while (vpu_cfg_readl(vpu, HOST_TO_VPU)); + + memcpy((void *)send_obj->share_buf, buf, len); + send_obj->len = len; + send_obj->id = id; + + vpu->ipi_id_ack[id] = false; + /* send the command to VPU */ + vpu_cfg_writel(vpu, 0x1, HOST_TO_VPU); + + mutex_unlock(&vpu->vpu_mutex); + + /* wait for VPU's ACK */ + timeout = msecs_to_jiffies(IPI_TIMEOUT_MS); + ret = wait_event_timeout(vpu->ack_wq, vpu->ipi_id_ack[id], timeout); + vpu->ipi_id_ack[id] = false; + if (ret == 0) { + dev_err(vpu->dev, "vpu ipi %d ack time out !", id); + ret = -EIO; + goto clock_disable; + } + vpu_clock_disable(vpu); + + return 0; + +mut_unlock: + mutex_unlock(&vpu->vpu_mutex); +clock_disable: + vpu_clock_disable(vpu); + + return ret; +} +EXPORT_SYMBOL_GPL(vpu_ipi_send); + +static void vpu_wdt_reset_func(struct work_struct *ws) +{ + struct vpu_wdt *wdt = container_of(ws, struct vpu_wdt, ws); + struct mtk_vpu *vpu = container_of(wdt, struct mtk_vpu, wdt); + struct vpu_wdt_handler *handler = wdt->handler; + int index, ret; + + dev_info(vpu->dev, "vpu reset\n"); + ret = vpu_clock_enable(vpu); + if (ret) { + dev_err(vpu->dev, "[VPU] wdt enables clock failed %d\n", ret); + return; + } + mutex_lock(&vpu->vpu_mutex); + vpu_cfg_writel(vpu, 0x0, VPU_RESET); + vpu->fw_loaded = false; + mutex_unlock(&vpu->vpu_mutex); + vpu_clock_disable(vpu); + + for (index = 0; index < VPU_RST_MAX; index++) { + if (handler[index].reset_func) { + handler[index].reset_func(handler[index].priv); + dev_dbg(vpu->dev, "wdt handler func %d\n", index); + } + } +} + +int vpu_wdt_reg_handler(struct platform_device *pdev, + void wdt_reset(void *), + void *priv, enum rst_id id) +{ + struct mtk_vpu *vpu = platform_get_drvdata(pdev); + struct vpu_wdt_handler *handler; + + if (!vpu) { + dev_err(&pdev->dev, "vpu device in not ready\n"); + return -EPROBE_DEFER; + } + + handler = vpu->wdt.handler; + + if (id >= 0 && id < VPU_RST_MAX && wdt_reset) { + dev_dbg(vpu->dev, "wdt register id %d\n", id); + mutex_lock(&vpu->vpu_mutex); + handler[id].reset_func = wdt_reset; + handler[id].priv = priv; + mutex_unlock(&vpu->vpu_mutex); + return 0; + } + + dev_err(vpu->dev, "register vpu wdt handler failed\n"); + return -EINVAL; +} +EXPORT_SYMBOL_GPL(vpu_wdt_reg_handler); + +unsigned int vpu_get_venc_hw_capa(struct platform_device *pdev) +{ + struct mtk_vpu *vpu = platform_get_drvdata(pdev); + + return vpu->run.enc_capability; +} +EXPORT_SYMBOL_GPL(vpu_get_venc_hw_capa); + +void *vpu_mapping_dm_addr(struct platform_device *pdev, + u32 dtcm_dmem_addr) +{ + struct mtk_vpu *vpu = platform_get_drvdata(pdev); + + if (!dtcm_dmem_addr || + (dtcm_dmem_addr > (VPU_DTCM_SIZE + VPU_EXT_D_SIZE))) { + dev_err(vpu->dev, "invalid virtual data memory address\n"); + return ERR_PTR(-EINVAL); + } + + if (dtcm_dmem_addr < VPU_DTCM_SIZE) + return (__force void *)(dtcm_dmem_addr + vpu->reg.tcm + + VPU_DTCM_OFFSET); + + return vpu->extmem[D_FW].va + (dtcm_dmem_addr - VPU_DTCM_SIZE); +} +EXPORT_SYMBOL_GPL(vpu_mapping_dm_addr); + +struct platform_device *vpu_get_plat_device(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *vpu_node; + struct platform_device *vpu_pdev; + + vpu_node = of_parse_phandle(dev->of_node, "mediatek,vpu", 0); + if (!vpu_node) { + dev_err(dev, "can't get vpu node\n"); + return NULL; + } + + vpu_pdev = of_find_device_by_node(vpu_node); + if (WARN_ON(!vpu_pdev)) { + dev_err(dev, "vpu pdev failed\n"); + of_node_put(vpu_node); + return NULL; + } + + return vpu_pdev; +} +EXPORT_SYMBOL_GPL(vpu_get_plat_device); + +/* load vpu program/data memory */ +static int load_requested_vpu(struct mtk_vpu *vpu, + const struct firmware *vpu_fw, + u8 fw_type) +{ + size_t tcm_size = fw_type ? VPU_DTCM_SIZE : VPU_PTCM_SIZE; + size_t fw_size = fw_type ? VPU_D_FW_SIZE : VPU_P_FW_SIZE; + char *fw_name = fw_type ? VPU_D_FW : VPU_P_FW; + size_t dl_size = 0; + size_t extra_fw_size = 0; + void *dest; + int ret; + + ret = request_firmware(&vpu_fw, fw_name, vpu->dev); + if (ret < 0) { + dev_err(vpu->dev, "Failed to load %s, %d\n", fw_name, ret); + return ret; + } + dl_size = vpu_fw->size; + if (dl_size > fw_size) { + dev_err(vpu->dev, "fw %s size %zu is abnormal\n", fw_name, + dl_size); + release_firmware(vpu_fw); + return -EFBIG; + } + dev_dbg(vpu->dev, "Downloaded fw %s size: %zu.\n", + fw_name, + dl_size); + /* reset VPU */ + vpu_cfg_writel(vpu, 0x0, VPU_RESET); + + /* handle extended firmware size */ + if (dl_size > tcm_size) { + dev_dbg(vpu->dev, "fw size %zu > limited fw size %zu\n", + dl_size, tcm_size); + extra_fw_size = dl_size - tcm_size; + dev_dbg(vpu->dev, "extra_fw_size %zu\n", extra_fw_size); + dl_size = tcm_size; + } + dest = (__force void *)vpu->reg.tcm; + if (fw_type == D_FW) + dest += VPU_DTCM_OFFSET; + memcpy(dest, vpu_fw->data, dl_size); + /* download to extended memory if need */ + if (extra_fw_size > 0) { + dest = vpu->extmem[fw_type].va; + dev_dbg(vpu->dev, "download extended memory type %x\n", + fw_type); + memcpy(dest, vpu_fw->data + tcm_size, extra_fw_size); + } + + release_firmware(vpu_fw); + + return 0; +} + +int vpu_load_firmware(struct platform_device *pdev) +{ + struct mtk_vpu *vpu = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + struct vpu_run *run = &vpu->run; + const struct firmware *vpu_fw = NULL; + int ret; + + if (!pdev) { + dev_err(dev, "VPU platform device is invalid\n"); + return -EINVAL; + } + + mutex_lock(&vpu->vpu_mutex); + if (vpu->fw_loaded) { + mutex_unlock(&vpu->vpu_mutex); + return 0; + } + mutex_unlock(&vpu->vpu_mutex); + + ret = vpu_clock_enable(vpu); + if (ret) { + dev_err(dev, "enable clock failed %d\n", ret); + return ret; + } + + mutex_lock(&vpu->vpu_mutex); + + run->signaled = false; + dev_dbg(vpu->dev, "firmware request\n"); + /* Downloading program firmware to device*/ + ret = load_requested_vpu(vpu, vpu_fw, P_FW); + if (ret < 0) { + dev_err(dev, "Failed to request %s, %d\n", VPU_P_FW, ret); + goto OUT_LOAD_FW; + } + + /* Downloading data firmware to device */ + ret = load_requested_vpu(vpu, vpu_fw, D_FW); + if (ret < 0) { + dev_err(dev, "Failed to request %s, %d\n", VPU_D_FW, ret); + goto OUT_LOAD_FW; + } + + vpu->fw_loaded = true; + /* boot up vpu */ + vpu_cfg_writel(vpu, 0x1, VPU_RESET); + + ret = wait_event_interruptible_timeout(run->wq, + run->signaled, + msecs_to_jiffies(INIT_TIMEOUT_MS) + ); + if (ret == 0) { + ret = -ETIME; + dev_err(dev, "wait vpu initialization timout!\n"); + goto OUT_LOAD_FW; + } else if (-ERESTARTSYS == ret) { + dev_err(dev, "wait vpu interrupted by a signal!\n"); + goto OUT_LOAD_FW; + } + + ret = 0; + dev_info(dev, "vpu is ready. Fw version %s\n", run->fw_ver); + +OUT_LOAD_FW: + mutex_unlock(&vpu->vpu_mutex); + vpu_clock_disable(vpu); + + return ret; +} +EXPORT_SYMBOL_GPL(vpu_load_firmware); + +static void vpu_init_ipi_handler(void *data, unsigned int len, void *priv) +{ + struct mtk_vpu *vpu = (struct mtk_vpu *)priv; + struct vpu_run *run = (struct vpu_run *)data; + + vpu->run.signaled = run->signaled; + strncpy(vpu->run.fw_ver, run->fw_ver, VPU_FW_VER_LEN); + vpu->run.enc_capability = run->enc_capability; + wake_up_interruptible(&vpu->run.wq); +} + +#ifdef CONFIG_DEBUG_FS +static ssize_t vpu_debug_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[256]; + unsigned int len; + unsigned int running, pc, vpu_to_host, host_to_vpu, wdt; + int ret; + struct device *dev = file->private_data; + struct mtk_vpu *vpu = dev_get_drvdata(dev); + + ret = vpu_clock_enable(vpu); + if (ret) { + dev_err(vpu->dev, "[VPU] enable clock failed %d\n", ret); + return 0; + } + + /* vpu register status */ + running = vpu_running(vpu); + pc = vpu_cfg_readl(vpu, VPU_PC_REG); + wdt = vpu_cfg_readl(vpu, VPU_WDT_REG); + host_to_vpu = vpu_cfg_readl(vpu, HOST_TO_VPU); + vpu_to_host = vpu_cfg_readl(vpu, VPU_TO_HOST); + vpu_clock_disable(vpu); + + if (running) { + len = snprintf(buf, sizeof(buf), "VPU is running\n\n" + "FW Version: %s\n" + "PC: 0x%x\n" + "WDT: 0x%x\n" + "Host to VPU: 0x%x\n" + "VPU to Host: 0x%x\n", + vpu->run.fw_ver, pc, wdt, + host_to_vpu, vpu_to_host); + } else { + len = snprintf(buf, sizeof(buf), "VPU not running\n"); + } + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static const struct file_operations vpu_debug_fops = { + .open = simple_open, + .read = vpu_debug_read, +}; +#endif /* CONFIG_DEBUG_FS */ + +static void vpu_free_ext_mem(struct mtk_vpu *vpu, u8 fw_type) +{ + struct device *dev = vpu->dev; + size_t fw_ext_size = fw_type ? VPU_EXT_D_SIZE : VPU_EXT_P_SIZE; + + dma_free_coherent(dev, fw_ext_size, vpu->extmem[fw_type].va, + vpu->extmem[fw_type].pa); +} + +static int vpu_alloc_ext_mem(struct mtk_vpu *vpu, u32 fw_type) +{ + struct device *dev = vpu->dev; + size_t fw_ext_size = fw_type ? VPU_EXT_D_SIZE : VPU_EXT_P_SIZE; + u32 vpu_ext_mem0 = fw_type ? VPU_DMEM_EXT0_ADDR : VPU_PMEM_EXT0_ADDR; + u32 vpu_ext_mem1 = fw_type ? VPU_DMEM_EXT1_ADDR : VPU_PMEM_EXT1_ADDR; + u32 offset_4gb = vpu->enable_4GB ? 0x40000000 : 0; + + vpu->extmem[fw_type].va = dma_alloc_coherent(dev, + fw_ext_size, + &vpu->extmem[fw_type].pa, + GFP_KERNEL); + if (!vpu->extmem[fw_type].va) { + dev_err(dev, "Failed to allocate the extended program memory\n"); + return PTR_ERR(vpu->extmem[fw_type].va); + } + + /* Disable extend0. Enable extend1 */ + vpu_cfg_writel(vpu, 0x1, vpu_ext_mem0); + vpu_cfg_writel(vpu, (vpu->extmem[fw_type].pa & 0xFFFFF000) + offset_4gb, + vpu_ext_mem1); + + dev_info(dev, "%s extend memory phy=0x%llx virt=0x%p\n", + fw_type ? "Data" : "Program", + (unsigned long long)vpu->extmem[fw_type].pa, + vpu->extmem[fw_type].va); + + return 0; +} + +static void vpu_ipi_handler(struct mtk_vpu *vpu) +{ + struct share_obj *rcv_obj = vpu->recv_buf; + struct vpu_ipi_desc *ipi_desc = vpu->ipi_desc; + + if (rcv_obj->id < IPI_MAX && ipi_desc[rcv_obj->id].handler) { + ipi_desc[rcv_obj->id].handler(rcv_obj->share_buf, + rcv_obj->len, + ipi_desc[rcv_obj->id].priv); + if (rcv_obj->id > IPI_VPU_INIT) { + vpu->ipi_id_ack[rcv_obj->id] = true; + wake_up(&vpu->ack_wq); + } + } else { + dev_err(vpu->dev, "No such ipi id = %d\n", rcv_obj->id); + } +} + +static int vpu_ipi_init(struct mtk_vpu *vpu) +{ + /* Disable VPU to host interrupt */ + vpu_cfg_writel(vpu, 0x0, VPU_TO_HOST); + + /* shared buffer initialization */ + vpu->recv_buf = (__force struct share_obj *)(vpu->reg.tcm + + VPU_DTCM_OFFSET); + vpu->send_buf = vpu->recv_buf + 1; + memset(vpu->recv_buf, 0, sizeof(struct share_obj)); + memset(vpu->send_buf, 0, sizeof(struct share_obj)); + + return 0; +} + +static irqreturn_t vpu_irq_handler(int irq, void *priv) +{ + struct mtk_vpu *vpu = priv; + u32 vpu_to_host; + int ret; + + /* + * Clock should have been enabled already. + * Enable again in case vpu_ipi_send times out + * and has disabled the clock. + */ + ret = clk_enable(vpu->clk); + if (ret) { + dev_err(vpu->dev, "[VPU] enable clock failed %d\n", ret); + return IRQ_NONE; + } + vpu_to_host = vpu_cfg_readl(vpu, VPU_TO_HOST); + if (vpu_to_host & VPU_IPC_INT) { + vpu_ipi_handler(vpu); + } else { + dev_err(vpu->dev, "vpu watchdog timeout! 0x%x", vpu_to_host); + queue_work(vpu->wdt.wq, &vpu->wdt.ws); + } + + /* VPU won't send another interrupt until we set VPU_TO_HOST to 0. */ + vpu_cfg_writel(vpu, 0x0, VPU_TO_HOST); + clk_disable(vpu->clk); + + return IRQ_HANDLED; +} + +#ifdef CONFIG_DEBUG_FS +static struct dentry *vpu_debugfs; +#endif +static int mtk_vpu_probe(struct platform_device *pdev) +{ + struct mtk_vpu *vpu; + struct device *dev; + struct resource *res; + int ret = 0; + + dev_dbg(&pdev->dev, "initialization\n"); + + dev = &pdev->dev; + vpu = devm_kzalloc(dev, sizeof(*vpu), GFP_KERNEL); + if (!vpu) + return -ENOMEM; + + vpu->dev = &pdev->dev; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcm"); + vpu->reg.tcm = devm_ioremap_resource(dev, res); + if (IS_ERR((__force void *)vpu->reg.tcm)) + return PTR_ERR((__force void *)vpu->reg.tcm); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg_reg"); + vpu->reg.cfg = devm_ioremap_resource(dev, res); + if (IS_ERR((__force void *)vpu->reg.cfg)) + return PTR_ERR((__force void *)vpu->reg.cfg); + + /* Get VPU clock */ + vpu->clk = devm_clk_get(dev, "main"); + if (IS_ERR(vpu->clk)) { + dev_err(dev, "get vpu clock failed\n"); + return PTR_ERR(vpu->clk); + } + + platform_set_drvdata(pdev, vpu); + + ret = clk_prepare(vpu->clk); + if (ret) { + dev_err(dev, "prepare vpu clock failed\n"); + return ret; + } + + /* VPU watchdog */ + vpu->wdt.wq = create_singlethread_workqueue("vpu_wdt"); + if (!vpu->wdt.wq) { + dev_err(dev, "initialize wdt workqueue failed\n"); + return -ENOMEM; + } + INIT_WORK(&vpu->wdt.ws, vpu_wdt_reset_func); + mutex_init(&vpu->vpu_mutex); + + ret = vpu_clock_enable(vpu); + if (ret) { + dev_err(dev, "enable vpu clock failed\n"); + goto workqueue_destroy; + } + + dev_dbg(dev, "vpu ipi init\n"); + ret = vpu_ipi_init(vpu); + if (ret) { + dev_err(dev, "Failed to init ipi\n"); + goto disable_vpu_clk; + } + + /* register vpu initialization IPI */ + ret = vpu_ipi_register(pdev, IPI_VPU_INIT, vpu_init_ipi_handler, + "vpu_init", vpu); + if (ret) { + dev_err(dev, "Failed to register IPI_VPU_INIT\n"); + goto vpu_mutex_destroy; + } + +#ifdef CONFIG_DEBUG_FS + vpu_debugfs = debugfs_create_file("mtk_vpu", S_IRUGO, NULL, (void *)dev, + &vpu_debug_fops); + if (!vpu_debugfs) { + ret = -ENOMEM; + goto cleanup_ipi; + } +#endif + + /* Set PTCM to 96K and DTCM to 32K */ + vpu_cfg_writel(vpu, 0x2, VPU_TCM_CFG); + + vpu->enable_4GB = !!(totalram_pages > (SZ_2G >> PAGE_SHIFT)); + dev_info(dev, "4GB mode %u\n", vpu->enable_4GB); + + if (vpu->enable_4GB) { + ret = of_reserved_mem_device_init(dev); + if (ret) + dev_info(dev, "init reserved memory failed\n"); + /* continue to use dynamic allocation if failed */ + } + + ret = vpu_alloc_ext_mem(vpu, D_FW); + if (ret) { + dev_err(dev, "Allocate DM failed\n"); + goto remove_debugfs; + } + + ret = vpu_alloc_ext_mem(vpu, P_FW); + if (ret) { + dev_err(dev, "Allocate PM failed\n"); + goto free_d_mem; + } + + init_waitqueue_head(&vpu->run.wq); + init_waitqueue_head(&vpu->ack_wq); + + res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!res) { + dev_err(dev, "get IRQ resource failed.\n"); + ret = -ENXIO; + goto free_p_mem; + } + vpu->reg.irq = platform_get_irq(pdev, 0); + ret = devm_request_irq(dev, vpu->reg.irq, vpu_irq_handler, 0, + pdev->name, vpu); + if (ret) { + dev_err(dev, "failed to request irq\n"); + goto free_p_mem; + } + + vpu_clock_disable(vpu); + dev_dbg(dev, "initialization completed\n"); + + return 0; + +free_p_mem: + vpu_free_ext_mem(vpu, P_FW); +free_d_mem: + vpu_free_ext_mem(vpu, D_FW); +remove_debugfs: + of_reserved_mem_device_release(dev); +#ifdef CONFIG_DEBUG_FS + debugfs_remove(vpu_debugfs); +cleanup_ipi: +#endif + memset(vpu->ipi_desc, 0, sizeof(struct vpu_ipi_desc) * IPI_MAX); +vpu_mutex_destroy: + mutex_destroy(&vpu->vpu_mutex); +disable_vpu_clk: + vpu_clock_disable(vpu); +workqueue_destroy: + destroy_workqueue(vpu->wdt.wq); + + return ret; +} + +static const struct of_device_id mtk_vpu_match[] = { + { + .compatible = "mediatek,mt8173-vpu", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, mtk_vpu_match); + +static int mtk_vpu_remove(struct platform_device *pdev) +{ + struct mtk_vpu *vpu = platform_get_drvdata(pdev); + +#ifdef CONFIG_DEBUG_FS + debugfs_remove(vpu_debugfs); +#endif + if (vpu->wdt.wq) { + flush_workqueue(vpu->wdt.wq); + destroy_workqueue(vpu->wdt.wq); + } + vpu_free_ext_mem(vpu, P_FW); + vpu_free_ext_mem(vpu, D_FW); + mutex_destroy(&vpu->vpu_mutex); + clk_unprepare(vpu->clk); + + return 0; +} + +static struct platform_driver mtk_vpu_driver = { + .probe = mtk_vpu_probe, + .remove = mtk_vpu_remove, + .driver = { + .name = "mtk_vpu", + .of_match_table = mtk_vpu_match, + }, +}; + +module_platform_driver(mtk_vpu_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Mediatek Video Prosessor Unit driver"); diff --git a/drivers/media/platform/mtk-vpu/mtk_vpu.h b/drivers/media/platform/mtk-vpu/mtk_vpu.h new file mode 100644 index 000000000000..5ab37f04bdfd --- /dev/null +++ b/drivers/media/platform/mtk-vpu/mtk_vpu.h @@ -0,0 +1,162 @@ +/* +* Copyright (c) 2016 MediaTek Inc. +* Author: Andrew-CT Chen +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License version 2 as +* published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +*/ + +#ifndef _MTK_VPU_H +#define _MTK_VPU_H + +#include + +/** + * VPU (video processor unit) is a tiny processor controlling video hardware + * related to video codec, scaling and color format converting. + * VPU interfaces with other blocks by share memory and interrupt. + **/ + +typedef void (*ipi_handler_t) (void *data, + unsigned int len, + void *priv); + +/** + * enum ipi_id - the id of inter-processor interrupt + * + * @IPI_VPU_INIT: The interrupt from vpu is to notfiy kernel + VPU initialization completed. + IPI_VPU_INIT is sent from VPU when firmware is + loaded. AP doesn't need to send IPI_VPU_INIT + command to VPU. + For other IPI below, AP should send the request + to VPU to trigger the interrupt. + * @IPI_VENC_H264: The interrupt from vpu is to notify kernel to + handle H264 video encoder job, and vice versa. + * @IPI_VENC_VP8: The interrupt fro vpu is to notify kernel to + handle VP8 video encoder job,, and vice versa. + * @IPI_MAX: The maximum IPI number + */ + +enum ipi_id { + IPI_VPU_INIT = 0, + IPI_VENC_H264, + IPI_VENC_VP8, + IPI_MAX, +}; + +/** + * enum rst_id - reset id to register reset function for VPU watchdog timeout + * + * @VPU_RST_ENC: encoder reset id + * @VPU_RST_MAX: maximum reset id + */ +enum rst_id { + VPU_RST_ENC, + VPU_RST_MAX, +}; + +/** + * vpu_ipi_register - register an ipi function + * + * @pdev: VPU platform device + * @id: IPI ID + * @handler: IPI handler + * @name: IPI name + * @priv: private data for IPI handler + * + * Register an ipi function to receive ipi interrupt from VPU. + * + * Return: Return 0 if ipi registers successfully, otherwise it is failed. + */ +int vpu_ipi_register(struct platform_device *pdev, enum ipi_id id, + ipi_handler_t handler, const char *name, void *priv); + +/** + * vpu_ipi_send - send data from AP to vpu. + * + * @pdev: VPU platform device + * @id: IPI ID + * @buf: the data buffer + * @len: the data buffer length + * + * This function is thread-safe. When this function returns, + * VPU has received the data and starts the processing. + * When the processing completes, IPI handler registered + * by vpu_ipi_register will be called in interrupt context. + * + * Return: Return 0 if sending data successfully, otherwise it is failed. + **/ +int vpu_ipi_send(struct platform_device *pdev, + enum ipi_id id, void *buf, + unsigned int len); + +/** + * vpu_get_plat_device - get VPU's platform device + * + * @pdev: the platform device of the module requesting VPU platform + * device for using VPU API. + * + * Return: Return NULL if it is failed. + * otherwise it is VPU's platform device + **/ +struct platform_device *vpu_get_plat_device(struct platform_device *pdev); + +/** + * vpu_wdt_reg_handler - register a VPU watchdog handler + * + * @pdev: VPU platform device + * @vpu_wdt_reset_func: the callback reset function + * @private_data: the private data for reset function + * @rst_id: reset id + * + * Register a handler performing own tasks when vpu reset by watchdog + * + * Return: Return 0 if the handler is added successfully, + * otherwise it is failed. + * + **/ +int vpu_wdt_reg_handler(struct platform_device *pdev, + void vpu_wdt_reset_func(void *), + void *priv, enum rst_id id); +/** + * vpu_get_venc_hw_capa - get video encoder hardware capability + * + * @pdev: VPU platform device + * + * Return: video encoder hardware capability + **/ +unsigned int vpu_get_venc_hw_capa(struct platform_device *pdev); + +/** + * vpu_load_firmware - download VPU firmware and boot it + * + * @pdev: VPU platform device + * + * Return: Return 0 if downloading firmware successfully, + * otherwise it is failed + **/ +int vpu_load_firmware(struct platform_device *pdev); + +/** + * vpu_mapping_dm_addr - Mapping DTCM/DMEM to kernel virtual address + * + * @pdev: VPU platform device + * @dmem_addr: VPU's data memory address + * + * Mapping the VPU's DTCM (Data Tightly-Coupled Memory) / + * DMEM (Data Extended Memory) memory address to + * kernel virtual address. + * + * Return: Return ERR_PTR(-EINVAL) if mapping failed, + * otherwise the mapped kernel virtual address + **/ +void *vpu_mapping_dm_addr(struct platform_device *pdev, + u32 dtcm_dmem_addr); +#endif /* _MTK_VPU_H */ diff --git a/drivers/media/platform/mx2_emmaprp.c b/drivers/media/platform/mx2_emmaprp.c index 3c4012d42d69..c639406fe72e 100644 --- a/drivers/media/platform/mx2_emmaprp.c +++ b/drivers/media/platform/mx2_emmaprp.c @@ -211,7 +211,6 @@ struct emmaprp_dev { struct clk *clk_emma_ahb, *clk_emma_ipg; struct v4l2_m2m_dev *m2m_dev; - struct vb2_alloc_ctx *alloc_ctx; }; struct emmaprp_ctx { @@ -690,7 +689,7 @@ static const struct v4l2_ioctl_ops emmaprp_ioctl_ops = { */ static int emmaprp_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes, - unsigned int sizes[], void *alloc_ctxs[]) + unsigned int sizes[], struct device *alloc_devs[]) { struct emmaprp_ctx *ctx = vb2_get_drv_priv(vq); struct emmaprp_q_data *q_data; @@ -710,8 +709,6 @@ static int emmaprp_queue_setup(struct vb2_queue *vq, *nbuffers = count; sizes[0] = size; - alloc_ctxs[0] = ctx->dev->alloc_ctx; - dprintk(ctx->dev, "get %d buffer(s) of size %d each.\n", count, size); return 0; @@ -765,6 +762,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq, src_vq->ops = &emmaprp_qops; src_vq->mem_ops = &vb2_dma_contig_memops; src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + src_vq->dev = ctx->dev->v4l2_dev.dev; ret = vb2_queue_init(src_vq); if (ret) @@ -777,6 +775,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->ops = &emmaprp_qops; dst_vq->mem_ops = &vb2_dma_contig_memops; dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + dst_vq->dev = ctx->dev->v4l2_dev.dev; return vb2_queue_init(dst_vq); } @@ -948,18 +947,11 @@ static int emmaprp_probe(struct platform_device *pdev) if (ret) goto rel_vdev; - pcdev->alloc_ctx = vb2_dma_contig_init_ctx(&pdev->dev); - if (IS_ERR(pcdev->alloc_ctx)) { - v4l2_err(&pcdev->v4l2_dev, "Failed to alloc vb2 context\n"); - ret = PTR_ERR(pcdev->alloc_ctx); - goto rel_vdev; - } - pcdev->m2m_dev = v4l2_m2m_init(&m2m_ops); if (IS_ERR(pcdev->m2m_dev)) { v4l2_err(&pcdev->v4l2_dev, "Failed to init mem2mem device\n"); ret = PTR_ERR(pcdev->m2m_dev); - goto rel_ctx; + goto rel_vdev; } ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0); @@ -973,8 +965,6 @@ static int emmaprp_probe(struct platform_device *pdev) rel_m2m: v4l2_m2m_release(pcdev->m2m_dev); -rel_ctx: - vb2_dma_contig_cleanup_ctx(pcdev->alloc_ctx); rel_vdev: video_device_release(vfd); unreg_dev: @@ -993,7 +983,6 @@ static int emmaprp_remove(struct platform_device *pdev) video_unregister_device(pcdev->vfd); v4l2_m2m_release(pcdev->m2m_dev); - vb2_dma_contig_cleanup_ctx(pcdev->alloc_ctx); v4l2_device_unregister(&pcdev->v4l2_dev); mutex_destroy(&pcdev->dev_mutex); diff --git a/drivers/media/platform/omap/omap_vout.c b/drivers/media/platform/omap/omap_vout.c index 70c28d19ea04..4afc999c0780 100644 --- a/drivers/media/platform/omap/omap_vout.c +++ b/drivers/media/platform/omap/omap_vout.c @@ -1318,71 +1318,16 @@ s_crop_err: return ret; } -static int vidioc_queryctrl(struct file *file, void *fh, - struct v4l2_queryctrl *ctrl) +static int omap_vout_s_ctrl(struct v4l2_ctrl *ctrl) { + struct omap_vout_device *vout = + container_of(ctrl->handler, struct omap_vout_device, ctrl_handler); int ret = 0; switch (ctrl->id) { - case V4L2_CID_ROTATE: - ret = v4l2_ctrl_query_fill(ctrl, 0, 270, 90, 0); - break; - case V4L2_CID_BG_COLOR: - ret = v4l2_ctrl_query_fill(ctrl, 0, 0xFFFFFF, 1, 0); - break; - case V4L2_CID_VFLIP: - ret = v4l2_ctrl_query_fill(ctrl, 0, 1, 1, 0); - break; - default: - ctrl->name[0] = '\0'; - ret = -EINVAL; - } - return ret; -} - -static int vidioc_g_ctrl(struct file *file, void *fh, struct v4l2_control *ctrl) -{ - int ret = 0; - struct omap_vout_device *vout = fh; - - switch (ctrl->id) { - case V4L2_CID_ROTATE: - ctrl->value = vout->control[0].value; - break; - case V4L2_CID_BG_COLOR: - { - struct omap_overlay_manager_info info; - struct omap_overlay *ovl; - - ovl = vout->vid_info.overlays[0]; - if (!ovl->manager || !ovl->manager->get_manager_info) { - ret = -EINVAL; - break; - } - - ovl->manager->get_manager_info(ovl->manager, &info); - ctrl->value = info.default_color; - break; - } - case V4L2_CID_VFLIP: - ctrl->value = vout->control[2].value; - break; - default: - ret = -EINVAL; - } - return ret; -} - -static int vidioc_s_ctrl(struct file *file, void *fh, struct v4l2_control *a) -{ - int ret = 0; - struct omap_vout_device *vout = fh; - - switch (a->id) { - case V4L2_CID_ROTATE: - { + case V4L2_CID_ROTATE: { struct omapvideo_info *ovid; - int rotation = a->value; + int rotation = ctrl->val; ovid = &vout->vid_info; @@ -1405,15 +1350,13 @@ static int vidioc_s_ctrl(struct file *file, void *fh, struct v4l2_control *a) ret = -EINVAL; break; } - - vout->control[0].value = rotation; mutex_unlock(&vout->lock); break; } case V4L2_CID_BG_COLOR: { struct omap_overlay *ovl; - unsigned int color = a->value; + unsigned int color = ctrl->val; struct omap_overlay_manager_info info; ovl = vout->vid_info.overlays[0]; @@ -1432,15 +1375,13 @@ static int vidioc_s_ctrl(struct file *file, void *fh, struct v4l2_control *a) ret = -EINVAL; break; } - - vout->control[1].value = color; mutex_unlock(&vout->lock); break; } case V4L2_CID_VFLIP: { struct omapvideo_info *ovid; - unsigned int mirror = a->value; + unsigned int mirror = ctrl->val; ovid = &vout->vid_info; @@ -1457,16 +1398,19 @@ static int vidioc_s_ctrl(struct file *file, void *fh, struct v4l2_control *a) break; } vout->mirror = mirror; - vout->control[2].value = mirror; mutex_unlock(&vout->lock); break; } default: - ret = -EINVAL; + return -EINVAL; } return ret; } +static const struct v4l2_ctrl_ops omap_vout_ctrl_ops = { + .s_ctrl = omap_vout_s_ctrl, +}; + static int vidioc_reqbufs(struct file *file, void *fh, struct v4l2_requestbuffers *req) { @@ -1831,11 +1775,8 @@ static const struct v4l2_ioctl_ops vout_ioctl_ops = { .vidioc_g_fmt_vid_out = vidioc_g_fmt_vid_out, .vidioc_try_fmt_vid_out = vidioc_try_fmt_vid_out, .vidioc_s_fmt_vid_out = vidioc_s_fmt_vid_out, - .vidioc_queryctrl = vidioc_queryctrl, - .vidioc_g_ctrl = vidioc_g_ctrl, .vidioc_s_fbuf = vidioc_s_fbuf, .vidioc_g_fbuf = vidioc_g_fbuf, - .vidioc_s_ctrl = vidioc_s_ctrl, .vidioc_try_fmt_vid_out_overlay = vidioc_try_fmt_vid_overlay, .vidioc_s_fmt_vid_out_overlay = vidioc_s_fmt_vid_overlay, .vidioc_g_fmt_vid_out_overlay = vidioc_g_fmt_vid_overlay, @@ -1865,9 +1806,9 @@ static int __init omap_vout_setup_video_data(struct omap_vout_device *vout) { struct video_device *vfd; struct v4l2_pix_format *pix; - struct v4l2_control *control; struct omap_overlay *ovl = vout->vid_info.overlays[0]; struct omap_dss_device *display = ovl->get_device(ovl); + struct v4l2_ctrl_handler *hdl; /* set the default pix */ pix = &vout->pix; @@ -1896,29 +1837,32 @@ static int __init omap_vout_setup_video_data(struct omap_vout_device *vout) omap_vout_new_format(pix, &vout->fbuf, &vout->crop, &vout->win); - /*Initialize the control variables for - rotation, flipping and background color. */ - control = vout->control; - control[0].id = V4L2_CID_ROTATE; - control[0].value = 0; + hdl = &vout->ctrl_handler; + v4l2_ctrl_handler_init(hdl, 3); + v4l2_ctrl_new_std(hdl, &omap_vout_ctrl_ops, + V4L2_CID_ROTATE, 0, 270, 90, 0); + v4l2_ctrl_new_std(hdl, &omap_vout_ctrl_ops, + V4L2_CID_BG_COLOR, 0, 0xffffff, 1, 0); + v4l2_ctrl_new_std(hdl, &omap_vout_ctrl_ops, + V4L2_CID_VFLIP, 0, 1, 1, 0); + if (hdl->error) + return hdl->error; + vout->rotation = 0; vout->mirror = false; - vout->control[2].id = V4L2_CID_HFLIP; - vout->control[2].value = 0; if (vout->vid_info.rotation_type == VOUT_ROT_VRFB) vout->vrfb_bpp = 2; - control[1].id = V4L2_CID_BG_COLOR; - control[1].value = 0; - /* initialize the video_device struct */ vfd = vout->vfd = video_device_alloc(); if (!vfd) { printk(KERN_ERR VOUT_NAME ": could not allocate" " video device struct\n"); + v4l2_ctrl_handler_free(hdl); return -ENOMEM; } + vfd->ctrl_handler = hdl; vfd->release = video_device_release; vfd->ioctl_ops = &vout_ioctl_ops; @@ -2092,6 +2036,7 @@ static void omap_vout_cleanup_device(struct omap_vout_device *vout) video_unregister_device(vfd); } } + v4l2_ctrl_handler_free(&vout->ctrl_handler); if (ovid->rotation_type == VOUT_ROT_VRFB) { omap_vout_release_vrfb(vout); /* Free the VRFB buffer if allocated diff --git a/drivers/media/platform/omap/omap_voutdef.h b/drivers/media/platform/omap/omap_voutdef.h index 9ccfe1f475a4..49de1475e473 100644 --- a/drivers/media/platform/omap/omap_voutdef.h +++ b/drivers/media/platform/omap/omap_voutdef.h @@ -11,6 +11,7 @@ #ifndef OMAP_VOUTDEF_H #define OMAP_VOUTDEF_H +#include #include