From c1b38e08d1fbc7d345e952e9760bf3f9b44d0f42 Mon Sep 17 00:00:00 2001 From: Axel Berndt Date: Tue, 21 Dec 2021 14:20:55 +0100 Subject: [PATCH] v0.1.7 - Little edit in method `mpmToolbox.gui.score.ScoreDisplayPanel.mouseMoved()` that changes the mouse cursor to hand symbol when moved to a clickable overlay element. - Fixed potential division-by-zero bug in `mpmToolbox.gui.syncPlayer.SyncPlayer.PlaybackRunnable.run()`. - Reorganized some classes, i.e., classes `Score`, `ScoreNote`, `ScorePage` moved from package `mpmToolbox.gui.score` to package `mpmToolbox.projectData.score`. - Added "Hide Overlay" button to the score widget (classes `mpmToolbox.gui.score.ScoreDocumentData` and `ScoreDisplayPanel`) that allows to show the score without the overlays. - Addition to the spectrogram context menu to switch between normalized and non-normalized display. - New method `getPart()` in classes `mpmToolbox.gui.msmTree.MsmTreeNode` and `mpmToolbox.gui.mpmTree.MpmTreeNode` to retrieve the MSM `part` element that the node belongs to. - New package `mpmToolbox.supplementary.avlTree` that implements the AVL Tree data structure. - New package `mpmToolbox.projectData.alignment` with several new classes that serve to associate measurements in audio recordings with MSM data, display them as piano roll and interact with it. - Alignment data is stored in `mpr` project files. - Added class `mpmToolbox.gui.audio.PianoRollPanel` which is also the basis for the classes `WaveformPanel` and `SpectrogramPanel` in the same package. - Several optimizations when editing `Performance` names, adding and removing `Performance` or `Audio` objects from and to the project in order to reduce update traffic between the widgets and the re-rendering of performances for overlay display in the audio widget. - Added button "Align Frequencies with MIDI Pitches" to the spectrogram specs. These set the min. and max frequency of the spectrogram. Use these to align it vertically with the piano roll. - Added combobox and sub-class `PartChooserItem` to class `mpmToolbox.gui.audio.AudioDocumentData` to choose the musical part or select all parts to be displayed by the piano roll overlay. --- history.md | 9 +- .../gui/audio/AudioDocumentData.java | 177 ++++++++++++-- src/mpmToolbox/gui/audio/PianoRollPanel.java | 179 +++++++++++--- .../gui/audio/SpectrogramPanel.java | 62 +++-- src/mpmToolbox/gui/audio/WaveformPanel.java | 58 +++-- src/mpmToolbox/gui/msmTree/MsmTree.java | 30 +++ .../gui/score/ScoreDisplayPanel.java | 9 +- src/mpmToolbox/gui/syncPlayer/SyncPlayer.java | 15 +- src/mpmToolbox/projectData/Audio.java | 16 +- .../projectData/alignment/Alignment.java | 144 ++++++++++- .../projectData/alignment/Note.java | 50 +++- .../projectData/alignment/Part.java | 224 ++++++++++++++++-- .../projectData/alignment/PianoRoll.java | 19 +- 13 files changed, 850 insertions(+), 142 deletions(-) diff --git a/history.md b/history.md index 5cd2ab0..5aeb9a7 100644 --- a/history.md +++ b/history.md @@ -2,16 +2,19 @@ #### v0.1.7 +- Little edit in method `mpmToolbox.gui.score.ScoreDisplayPanel.mouseMoved()` that changes the mouse cursor to hand symbol when moved to a clickable overlay element. - Fixed potential division-by-zero bug in `mpmToolbox.gui.syncPlayer.SyncPlayer.PlaybackRunnable.run()`. - Reorganized some classes, i.e., classes `Score`, `ScoreNote`, `ScorePage` moved from package `mpmToolbox.gui.score` to package `mpmToolbox.projectData.score`. - Added "Hide Overlay" button to the score widget (classes `mpmToolbox.gui.score.ScoreDocumentData` and `ScoreDisplayPanel`) that allows to show the score without the overlays. - Addition to the spectrogram context menu to switch between normalized and non-normalized display. - New method `getPart()` in classes `mpmToolbox.gui.msmTree.MsmTreeNode` and `mpmToolbox.gui.mpmTree.MpmTreeNode` to retrieve the MSM `part` element that the node belongs to. -- New package `mpmToolbox.supplementary.avlTree` that implements AVL Tree data structure. -- New package `mpmToolbox.projectData.alignment` with several new classes that serve to associate measurements in audio recordings with MSM data. +- New package `mpmToolbox.supplementary.avlTree` that implements the AVL Tree data structure. +- New package `mpmToolbox.projectData.alignment` with several new classes that serve to associate measurements in audio recordings with MSM data, display them as piano roll and interact with it. +- Alignment data is stored in `mpr` project files. - Added class `mpmToolbox.gui.audio.PianoRollPanel` which is also the basis for the classes `WaveformPanel` and `SpectrogramPanel` in the same package. - Several optimizations when editing `Performance` names, adding and removing `Performance` or `Audio` objects from and to the project in order to reduce update traffic between the widgets and the re-rendering of performances for overlay display in the audio widget. -- Added buttons "Min. MIDI Pitch" and "Max. MIDI Pitch" to the spectrogram specs. These set the min. and max frequency of the spectrogram. Use these to align it with the piano roll. +- Added button "Align Frequencies with MIDI Pitches" to the spectrogram specs. These set the min. and max frequency of the spectrogram. Use these to align it vertically with the piano roll. +- Added combobox and sub-class `PartChooserItem` to class `mpmToolbox.gui.audio.AudioDocumentData` to choose the musical part or select all parts to be displayed by the piano roll overlay. #### v0.1.6 diff --git a/src/mpmToolbox/gui/audio/AudioDocumentData.java b/src/mpmToolbox/gui/audio/AudioDocumentData.java index 5717662..fac52a5 100644 --- a/src/mpmToolbox/gui/audio/AudioDocumentData.java +++ b/src/mpmToolbox/gui/audio/AudioDocumentData.java @@ -4,14 +4,21 @@ import com.alee.api.data.Orientation; import com.alee.extended.split.WebMultiSplitPane; import com.alee.extended.tab.DocumentData; +import com.alee.laf.button.WebButton; +import com.alee.laf.combobox.WebComboBox; import com.alee.laf.panel.WebPanel; +import meico.mei.Helper; import meico.mpm.elements.Performance; +import meico.supplementary.KeyValue; import mpmToolbox.gui.ProjectPane; +import mpmToolbox.gui.Settings; import mpmToolbox.projectData.Audio; import mpmToolbox.projectData.alignment.Alignment; import mpmToolbox.supplementary.Tools; +import nu.xom.Element; import java.awt.*; +import java.awt.event.ItemEvent; import java.awt.event.MouseEvent; import java.awt.event.MouseWheelEvent; @@ -25,7 +32,7 @@ public class AudioDocumentData extends DocumentData { private final WaveformPanel waveform; private final SpectrogramPanel spectrogram; - private final PianoRollPanel pianoRoll; +// private final PianoRollPanel pianoRoll; private int channelNumber = -1; // index of the waveform/channel to be rendered to image; -1 means all channels private int leftmostSample = -1; // index of the first sample to be rendered to image @@ -33,6 +40,10 @@ public class AudioDocumentData extends DocumentData { private Alignment alignment; // this is the alignment with the piano roll overlay used in the sub-panels + private final WebComboBox partChooser = new WebComboBox(); // with this combobox the user can select whether all musical part or only on individual part should be displayed in the piano roll overlay +// private final WebSwitch globalLocalSwitch = new WebSwitch(); + private final WebButton resetAlignment = new WebButton("Reset Alignment"); + /** * constructor * @param parent @@ -41,18 +52,107 @@ public AudioDocumentData(@NotNull ProjectPane parent) { super("Audio", "Audio", null); this.parent = parent; + this.makePartChooser(); +// this.makeGlobalLocalSwitch(); + this.makeResetButton(); + this.waveform = new WaveformPanel(this); this.spectrogram = new SpectrogramPanel(this); - this.pianoRoll = new PianoRollPanel(this); +// this.pianoRoll = new PianoRollPanel(this); this.setComponent(this.audioPanel); this.setClosable(false); + this.updateAudio(false); this.updateAlignment(false); + this.updateAudioTools(); this.draw(); } + /** + * helper method for the constructor; it creates the contents of the part chooser + */ + private void makePartChooser() { + this.partChooser.setPadding(Settings.paddingInDialogs); + this.partChooser.setToolTip("Select the part to be displayed in the piano roll overlay."); + + this.partChooser.addItem(new PartChooserItem("All parts", null)); + + for (Element partElt : this.getParent().getMsm().getParts()) { + int number = Integer.parseInt(Helper.getAttributeValue("number", partElt)); + String name = "Part " + number + " " + Helper.getAttributeValue("name", partElt); + this.partChooser.addItem(new PartChooserItem(name, number)); + } + + this.partChooser.addItemListener(itemEvent -> { + if (itemEvent.getStateChange() == ItemEvent.SELECTED) { + this.repaintAllComponents(); + } + }); + } + +// /** +// * helper method for the constructor; it creates the switch between local and global editing mode +// */ +// private void makeGlobalLocalSwitch() { +// WebLabel global = new WebLabel("Global"); +// global.setPadding(Settings.paddingInDialogs); +// +// WebLabel local = new WebLabel("Part"); +// local.setPadding(Settings.paddingInDialogs); +// +// this.globalLocalSwitch.setSwitchComponents(global, local); +// +// WebLabel gripperLabel = new WebLabel("Edit"); +// gripperLabel.setPadding(Settings.paddingInDialogs); +// gripperLabel.setHorizontalAlignment(WebLabel.CENTER); +// this.globalLocalSwitch.getGripper().add(gripperLabel); +// +// this.globalLocalSwitch.setSelected(true); +// } + + /** + * define the button to reset the alignment + */ + private void makeResetButton() { + this.resetAlignment.setPadding(Settings.paddingInDialogs); + this.resetAlignment.addActionListener(actionEvent -> { + this.alignment.reset(); + + // in case of audio alignment being reset, scale the initial alignment to the milliseconds length of the audio; so all notes are visible and in a good starting position + if (this.getParent().getSyncPlayer().isAudioAlignmentSelected()) { + Audio audio = this.getParent().getSyncPlayer().getSelectedAudio(); + double milliseconds = ((double) audio.getNumberOfSamples() / audio.getFrameRate()) * 1000.0; + this.alignment.scaleOverallTiming(milliseconds); + } + + this.alignment.recomputePianoRoll(); + this.repaintAllComponents(); + }); + } + + /** + * this enables or disables the part chooser according to whether there is a performance selected in the syncPlayer + */ + public void updateAudioTools() { + this.partChooser.setEnabled(this.alignment != null); + this.resetAlignment.setEnabled(this.alignment != null); + } + + /** + * get the number of the selected part + * @return the number or null if all parts are selected + */ + protected Integer getPianoRollPartNumber() { + if (this.partChooser.getSelectedItem() == null) + return null; + + return ((PartChooserItem) this.partChooser.getSelectedItem()).getValue(); + } + + + /** * this draws the content of the audio analysis frame */ @@ -62,11 +162,18 @@ private void draw() { splitPane.setContinuousLayout(true); // when the divider is moved the content is continuously redrawn splitPane.add(this.waveform); splitPane.add(this.spectrogram); - splitPane.add(this.pianoRoll); +// splitPane.add(this.pianoRoll); // TODO: this pane will be used to display and edit timing curves GridBagLayout gridBagLayout = (GridBagLayout) this.audioPanel.getLayout(); Tools.addComponentToGridBagLayout(this.audioPanel, gridBagLayout, splitPane, 0, 0, 1, 1, 1.0, 1.0, 0, 0, GridBagConstraints.BOTH, GridBagConstraints.CENTER); -// Tools.addComponentToGridBagLayout(this.audioPanel, gridBagLayout, new WebLabel("Buttons go here"), 0, 1, 1, 1, 1.0, 0.0, 0, 0, GridBagConstraints.NONE, GridBagConstraints.SOUTH); + + // the panel with the buttons + WebPanel buttonPanel = new WebPanel(new GridBagLayout()); + GridBagLayout buttonLayout = (GridBagLayout) buttonPanel.getLayout(); + Tools.addComponentToGridBagLayout(buttonPanel, buttonLayout, this.partChooser, 0, 1, 1, 1, 1.0, 0.0, 0, 0, GridBagConstraints.NONE, GridBagConstraints.CENTER); +// Tools.addComponentToGridBagLayout(buttonPanel, buttonLayout, this.globalLocalSwitch, 1, 1, 1, 1, 1.0, 0.0, 0, 0, GridBagConstraints.NONE, GridBagConstraints.CENTER); + Tools.addComponentToGridBagLayout(buttonPanel, buttonLayout, this.resetAlignment, 2, 1, 1, 1, 1.0, 0.0, 0, 0, GridBagConstraints.NONE, GridBagConstraints.CENTER); + Tools.addComponentToGridBagLayout(this.audioPanel, gridBagLayout, buttonPanel, 0, 1, 1, 1, 1.0, 0.0, 0, 0, GridBagConstraints.NONE, GridBagConstraints.CENTER); } /** @@ -85,13 +192,13 @@ protected SpectrogramPanel getSpectrogramPanel() { return this.spectrogram; } - /** - * a getter for the piano roll panel - * @return - */ - protected PianoRollPanel getPianoRollPanel() { - return this.pianoRoll; - } +// /** +// * a getter for the piano roll panel +// * @return +// */ +// protected PianoRollPanel getPianoRollPanel() { +// return this.pianoRoll; +// } /** * The sequence at which the child components update their visualizations is important. @@ -100,7 +207,7 @@ protected PianoRollPanel getPianoRollPanel() { protected void repaintAllComponents() { this.waveform.repaint(); this.spectrogram.repaint(); - this.pianoRoll.repaint(); +// this.pianoRoll.repaint(); } /** @@ -110,7 +217,7 @@ protected void repaintAllComponents() { protected void communicateMouseEventToAllComponents(MouseEvent e) { this.waveform.setMousePosition(e); this.spectrogram.setMousePosition(e); - this.pianoRoll.setMousePosition(e); +// this.pianoRoll.setMousePosition(e); } /** @@ -133,7 +240,7 @@ public void updateAlignment(boolean doRepaint) { } if (doRepaint) - this.repaintAllComponents(); // repaint of all components + this.repaintAllComponents(); } /** @@ -160,7 +267,7 @@ public void updateAudio(boolean doRepaint) { this.spectrogram.setAudio(); if (doRepaint) - this.repaintAllComponents(); // repaint of all components + this.repaintAllComponents(); } /** @@ -247,7 +354,7 @@ protected void setRightmostSample(int rightmostSample) { * this shifts the visualisations left or right by the specified offset * @param sampleOffset offset in samples */ - protected void scroll(double sampleOffset) { + private void scroll(double sampleOffset) { if (this.parent.getAudio() == null) return; @@ -261,7 +368,7 @@ protected void scroll(double sampleOffset) { this.setRightmostSample((int) (this.rightmostSample + sampleOffset)); this.spectrogram.updateScroll(); - this.repaintAllComponents(); // triggers repaint for all components + this.repaintAllComponents(); } /** @@ -269,14 +376,14 @@ protected void scroll(double sampleOffset) { * @param pivotSample * @param zoomFactor */ - protected void zoom(int pivotSample, double zoomFactor) { + private void zoom(int pivotSample, double zoomFactor) { if (zoomFactor == 0.0) return; if (zoomFactor < 0.0) { // zoom in int leftmostSample = pivotSample - (int) ((pivotSample - this.leftmostSample) * zoomFactor); int rightmostSample = (int) ((this.rightmostSample - pivotSample) * zoomFactor) + pivotSample; - if ((rightmostSample - leftmostSample) > 1) { // make sure there are at least two samples to be drawn, if we zoom too far in, left==right, we cannot zoom out again + if ((rightmostSample - leftmostSample) > 1) { // make sure there are at least two samples to be drawn, if we zoom too far in, left==right, we cannot zoom out again this.setLeftmostSample(leftmostSample); this.setRightmostSample(rightmostSample); } @@ -292,14 +399,14 @@ else if (zoomFactor > 0.0) { // zoom out this.spectrogram.updateZoom(); - this.repaintAllComponents(); // triggers repaint for all components + this.repaintAllComponents(); } /** * process a mouse drag event; to be invoked by sub-panels WaveformPanel, SpectrogramPanel * @param e */ - protected void mouseDragged(MouseEvent e) { + protected void scroll(MouseEvent e) { if (this.getWaveformPanel().mousePosition == null) { this.getWaveformPanel().setMousePosition(e); return; @@ -311,7 +418,6 @@ protected void mouseDragged(MouseEvent e) { this.communicateMouseEventToAllComponents(e); this.scroll(sampleOffset); - } /** @@ -322,10 +428,35 @@ protected void mouseWheelMoved(MouseWheelEvent e){ if ((this.getAudio() == null) || (e.getWheelRotation() == 0)) return; - int pivotSample = this.getWaveformPanel().getSampleIndex(e.getPoint()); + int pivotSample = this.getWaveformPanel().getSampleIndex(e.getPoint().getX()); double zoomFactor = (e.getWheelRotation() < 0) ? 0.9 : 1.1; this.communicateMouseEventToAllComponents(e); this.zoom(pivotSample, zoomFactor); } + + /** + * This class represents an item in the part chooser combobox of the audio analysis widget. + * @author Axel Berndt + */ + private static class PartChooserItem extends KeyValue { + /** + * This constructor creates a part chooser item with the specified name key and part number. + * @param string + */ + private PartChooserItem(String string, Integer partNumber) { + super(string, partNumber); + } + + /** + * All combobox items require this method. The override here makes sure that the string being returned + * is the performance's name instead of some Java Object ID. + * @return + */ + @Override + public String toString() { + return this.getKey(); + } + } + } diff --git a/src/mpmToolbox/gui/audio/PianoRollPanel.java b/src/mpmToolbox/gui/audio/PianoRollPanel.java index bdff00b..e5d29df 100644 --- a/src/mpmToolbox/gui/audio/PianoRollPanel.java +++ b/src/mpmToolbox/gui/audio/PianoRollPanel.java @@ -1,12 +1,14 @@ package mpmToolbox.gui.audio; import com.alee.laf.label.WebLabel; -import com.alee.laf.menu.WebMenu; +import com.alee.laf.menu.WebCheckBoxMenuItem; import com.alee.laf.menu.WebMenuItem; +import com.alee.laf.menu.WebPopupMenu; import com.alee.laf.panel.WebPanel; -import meico.mei.Helper; +import mpmToolbox.gui.msmTree.MsmTree; +import mpmToolbox.gui.msmTree.MsmTreeNode; +import mpmToolbox.projectData.alignment.Note; import mpmToolbox.projectData.alignment.PianoRoll; -import nu.xom.Element; import java.awt.*; import java.awt.event.*; @@ -19,8 +21,9 @@ public class PianoRollPanel extends WebPanel implements ComponentListener, MouseListener, MouseMotionListener, MouseWheelListener { protected final AudioDocumentData parent; protected final WebLabel noData; - protected Point mousePosition = null; // this is to keep track of the mouse position and draw a cursor on the panel - protected boolean mouseInThisPanel = false; // this is set true when the mouse enters this panel and false if the mouse exits + protected Point mousePosition = null; // this is to keep track of the mouse position and draw a cursor on the panel + protected boolean mouseInThisPanel = false; // this is set true when the mouse enters this panel and false if the mouse exits + protected final NoteDrag dragGesture = new NoteDrag(); // this is set true when a track gesture is started, so that even iv the mouse moves over other notes or free space, only the initial note is dragged /** * constructor @@ -65,51 +68,68 @@ protected void drawPianoRoll(Graphics2D g2d) { if (this.parent.getAlignment() == null) return; - // TODO: draw piano roll overlay; this is just test code, yet + // draw piano roll overlay double fromMilliseconds = ((double) this.parent.getLeftmostSample() / this.parent.getAudio().getFrameRate()) * 1000.0; double toMilliseconds = ((double) this.parent.getRightmostSample() / this.parent.getAudio().getFrameRate()) * 1000.0; - PianoRoll pianoRoll = this.parent.getAlignment().getPianoRoll(fromMilliseconds, toMilliseconds, this.getWidth(), 128); -// PianoRoll pianoRoll = this.parent.getAudio().getAlignment().getPianoRoll(fromMilliseconds, toMilliseconds, this.getWidth(), 128); -// PianoRoll pianoRoll = (new Alignment(this.parent.parent.getSyncPlayer().getSelectedPerformance().perform(this.parent.parent.getMsm()), null)).getPianoRoll(fromMilliseconds, toMilliseconds, this.getWidth(), 128); - g2d.drawImage(pianoRoll, 0, this.getHeight(), this.getWidth(), -this.getHeight(), this); + PianoRoll pianoRoll = this.retrievePianoRoll(fromMilliseconds, toMilliseconds, this.getWidth(), 128); + g2d.drawImage(pianoRoll, 0, this.getHeight(), this.getWidth(), -this.getHeight(), this); } /** - * create context menu submenu of the part to be displayed in the piano roll overlay + * retrieve the piano roll from the parent + * @param fromMilliseconds + * @param toMilliseconds + * @param width + * @param height * @return */ - protected WebMenu getPianoRollTools() { - WebMenu pianoRollTools = new WebMenu("Piano Roll"); + protected PianoRoll retrievePianoRoll(double fromMilliseconds, double toMilliseconds, int width, int height) { + Integer partNumber = this.parent.getPianoRollPartNumber(); + if (partNumber != null) { // draw only the selected musical part + return this.parent.getAlignment().getPart(partNumber).getPianoRoll(fromMilliseconds, toMilliseconds, this.getWidth(), 128); + } - if (this.parent.getAlignment() == null) { - pianoRollTools.setEnabled(false); - } else { - pianoRollTools.setEnabled(true); + // draw all musical parts + return this.parent.getAlignment().getPianoRoll(fromMilliseconds, toMilliseconds, width, height); +// return this.parent.getAudio().getAlignment().getPianoRoll(fromMilliseconds, toMilliseconds, width, height); +// return (new Alignment(this.parent.parent.getSyncPlayer().getSelectedPerformance().perform(this.parent.parent.getMsm()), null)).getPianoRoll(fromMilliseconds, toMilliseconds, width, height); + } - WebMenu choosePart = new WebMenu("Choose Part"); - choosePart.setToolTipText("Select the part displayed in the piano roll overlay."); + /** + * retrieve the note reference behind a certain pixel position in the current piano roll image + * @param x horizontal pixel position in the piano roll image + * @param y vertical pixel position in the piano roll image + * @return the Note object or null + */ + protected Note getNoteAt(int x, int y) { + if (this.parent.getAlignment() == null) + return null; - WebMenuItem perfAllParts = new WebMenuItem("All Parts"); - choosePart.add(perfAllParts); - perfAllParts.addActionListener(actionEvent -> { - // TODO ... - }); + Integer partNumber = this.parent.getPianoRollPartNumber(); + PianoRoll pianoRoll = (partNumber != null) ? this.parent.getAlignment().getPart(partNumber).getPianoRoll() : this.parent.getAlignment().getPianoRoll(); + if (pianoRoll == null) + return null; - for (Element partElt : this.parent.getParent().getMsm().getParts()) { - int number = Integer.parseInt(Helper.getAttributeValue("number", partElt)); - String name = "Part " + number + " " + Helper.getAttributeValue("name", partElt); - WebMenuItem partItem = new WebMenuItem(name); - choosePart.add(partItem); - partItem.addActionListener(actionEvent -> { - // TODO ... - }); - } - - pianoRollTools.add(choosePart); - } + return pianoRoll.getNoteAt(x, y); + } - return pianoRollTools; + /** + * retrieve the note reference behind a certain relative position in the current piano roll image + * @param x relative horizontal position in the piano roll image (should be in [0, 1]) + * @param y relative vertical position in the piano roll image (should be in [0, 1]) + * @return the Note object or null + */ + protected Note getNoteAt(double x, double y) { + if (this.parent.getAlignment() == null) + return null; + + Integer partNumber = this.parent.getPianoRollPartNumber(); + PianoRoll pianoRoll = (partNumber != null) ? this.parent.getAlignment().getPart(partNumber).getPianoRoll() : this.parent.getAlignment().getPianoRoll(); + if (pianoRoll == null) + return null; + + return pianoRoll.getNoteAt((int) (pianoRoll.getWidth() * x), (int) (pianoRoll.getHeight() * (-y + 1.0))); } /** @@ -122,6 +142,42 @@ protected void setAudio() { this.remove(this.noData); } + /** + * compute which sample the mouse cursor is pointing at + * @param x horizontal pixel position in the panel + * @return + */ + protected int getSampleIndex(double x) { + double relativePosition = x / this.getWidth(); + return (int) Math.round((relativePosition * (this.parent.getRightmostSample() - this.parent.getLeftmostSample())) + this.parent.getLeftmostSample()); + } + + /** + * create a context menu for the position of the mouse click; + * further entries to the menu may be added by inheritances + * @param e + * @return + */ + protected WebPopupMenu getContextMenu(MouseEvent e) { + WebPopupMenu menu = new WebPopupMenu(); + + // make "note fixed" entry + Note note = this.getNoteAt(e.getPoint().getX() / this.getWidth(), e.getPoint().getY() / this.getHeight()); + if (note != null) { + WebCheckBoxMenuItem setFixed = new WebCheckBoxMenuItem("Note fixed", note.isFixed()); + setFixed.addActionListener(actionEvent -> { + note.setFixed(!note.isFixed()); + this.parent.getAlignment().updateTiming(); + this.parent.getAlignment().recomputePianoRoll(); + this.parent.repaintAllComponents(); + }); + setFixed.setToolTipText("Pins the note at its position."); + menu.add(setFixed); + } + + return menu; + } + /** * set the mouse position * @param e @@ -180,6 +236,9 @@ public void mousePressed(MouseEvent e) { */ @Override public void mouseReleased(MouseEvent e) { + this.dragGesture.lockedOnNote = false; // open the drag gesture lock + this.dragGesture.note = null; // forget the note we might have been dragging + this.mouseMoved(e); // do the same as if the mouse was just moved } /** @@ -213,6 +272,9 @@ public void mouseExited(MouseEvent e) { public void mouseMoved(MouseEvent e) { this.parent.communicateMouseEventToAllComponents(e); this.parent.repaintAllComponents(); + + Note note = this.getNoteAt(this.mousePosition.getX() / this.getWidth(), this.mousePosition.getY() / this.getHeight()); + this.setCursor((note == null) ? Cursor.getDefaultCursor() : new Cursor(Cursor.HAND_CURSOR)); } /** @@ -221,7 +283,17 @@ public void mouseMoved(MouseEvent e) { */ @Override public void mouseClicked(MouseEvent e) { + Note note = this.getNoteAt(this.mousePosition.getX() / this.getWidth(), this.mousePosition.getY() / this.getHeight()); + if (note == null) + return; + + MsmTree msmTree = this.parent.getParent().getMsmTree(); // a handle to the msm tree + MsmTreeNode msmTreeNode = this.parent.getParent().getMsmTree().findNode(note.getId(), true); + if (msmTreeNode == null) // if nothing has been selected + return; // done + msmTree.setSelectedNode(msmTreeNode); // select the node in the msm tree + msmTree.scrollPathToVisible(msmTreeNode.getTreePath()); // scroll the tree so the node is visible } /** @@ -230,7 +302,33 @@ public void mouseClicked(MouseEvent e) { */ @Override public void mouseDragged(MouseEvent e) { - this.parent.mouseDragged(e); + if (this.mousePosition == null) + return; + + if (!this.dragGesture.lockedOnNote) { // if no drag gesture currently running, we start a new one + this.dragGesture.lockedOnNote = true; // lock the drag gesture on ... + this.dragGesture.note = this.getNoteAt(this.mousePosition.getX() / this.getWidth(), this.mousePosition.getY() / this.getHeight()); // the note that the mouse is currently at or null + } + + if (this.dragGesture.note == null) { // if we perform a drag event with no note, it should be interpreted as scrolling + this.parent.scroll(e); + return; + } + + // we perform a drag event on a note + this.setCursor(new Cursor(Cursor.W_RESIZE_CURSOR)); + + double pixelOffset = e.getX() - this.mousePosition.getX(); + int samplesInFrame = this.parent.getRightmostSample() - this.parent.getLeftmostSample(); + double sampleOffset = (pixelOffset * samplesInFrame) / this.getWidth(); + double millisecOffset = (sampleOffset * 1000.0) / this.parent.getAudio().getFrameRate(); + + this.parent.getAlignment().reposition(this.dragGesture.note, this.dragGesture.note.getMillisecondsDate() + millisecOffset); // move the note and do the timing transform + this.parent.getAlignment().updateTiming(); + this.parent.getAlignment().recomputePianoRoll(); + + this.parent.communicateMouseEventToAllComponents(e); + this.parent.repaintAllComponents(); } /** @@ -241,4 +339,9 @@ public void mouseDragged(MouseEvent e) { public void mouseWheelMoved(MouseWheelEvent e) { this.parent.mouseWheelMoved(e); } + + protected static class NoteDrag { + protected boolean lockedOnNote = false; + protected Note note = null; + } } diff --git a/src/mpmToolbox/gui/audio/SpectrogramPanel.java b/src/mpmToolbox/gui/audio/SpectrogramPanel.java index 7b2d415..5fc2dbf 100644 --- a/src/mpmToolbox/gui/audio/SpectrogramPanel.java +++ b/src/mpmToolbox/gui/audio/SpectrogramPanel.java @@ -11,6 +11,7 @@ import com.tagtraum.jipes.math.WindowFunction; import mpmToolbox.gui.Settings; import mpmToolbox.projectData.Audio; +import mpmToolbox.projectData.alignment.Note; import mpmToolbox.supplementary.Tools; import javax.swing.*; @@ -128,6 +129,42 @@ public void componentResized(ComponentEvent e) { super.componentResized(e); } + /** + * on mouse enter event + * @param e + */ + @Override + public void mouseEntered(MouseEvent e) { + if (this.noData.isShowing() || this.spectrogramSpecs.isShowing()) + return; + + super.mouseEntered(e); + } + + /** + * on mouse exit event + * @param e + */ + @Override + public void mouseExited(MouseEvent e) { + if (this.noData.isShowing() || this.spectrogramSpecs.isShowing()) + return; + + super.mouseExited(e); + } + + /** + * on mouse exit event + * @param e + */ + @Override + public void mouseMoved(MouseEvent e) { + if (this.noData.isShowing() || this.spectrogramSpecs.isShowing()) + return; + + super.mouseMoved(e); + } + /** * on mouse click event * @@ -141,15 +178,15 @@ public void mouseClicked(MouseEvent e) { switch (e.getButton()) { case MouseEvent.BUTTON1: // left click - // TODO: select a note, place a marker ... + super.mouseClicked(e); // select a note break; case MouseEvent.BUTTON3: // right click = context menu - WebPopupMenu menu = new WebPopupMenu(); + WebPopupMenu menu = this.getContextMenu(e); // play from here WebMenuItem playFromHere = new WebMenuItem("Play from here"); playFromHere.addActionListener(actionEvent -> { - this.parent.getParent().getSyncPlayer().triggerPlayback(this.parent.getWaveformPanel().getSampleIndex(e.getPoint())); + this.parent.getParent().getSyncPlayer().triggerPlayback(this.parent.getWaveformPanel().getSampleIndex(e.getPoint().getX())); }); menu.add(playFromHere); @@ -169,10 +206,6 @@ public void mouseClicked(MouseEvent e) { }); menu.add(normalize); - // choose overlay - menu.add(this.getPianoRollTools()); - - menu.show(this, e.getX() - 25, e.getY()); break; } @@ -258,12 +291,6 @@ public SpectrogramSpecs(SpectrogramPanel parent) { maxFreqUnit.setPadding(Settings.paddingInDialogs); Tools.addComponentToGridBagLayout(this, (GridBagLayout) this.getLayout(), maxFreqUnit, 3, 4, 1, 1, 0.1, 1.0, 0, 0, GridBagConstraints.BOTH, GridBagConstraints.LINE_START); - WebButton maxMidi = new WebButton("Max. MIDI Pitch"); - maxMidi.addActionListener(actionEvent -> { - maxFreq.setValue(12543.8539514160); - }); - Tools.addComponentToGridBagLayout(this, (GridBagLayout) this.getLayout(), maxMidi, 4, 4, 1, 1, 0.1, 1.0, 0, 0, GridBagConstraints.BOTH, GridBagConstraints.LINE_START); - // min frequency WebLabel minFreqLabel = new WebLabel("Min. Frequency:", WebLabel.RIGHT); minFreqLabel.setPadding(Settings.paddingInDialogs); @@ -277,11 +304,14 @@ public SpectrogramSpecs(SpectrogramPanel parent) { minFreqUnit.setPadding(Settings.paddingInDialogs); Tools.addComponentToGridBagLayout(this, (GridBagLayout) this.getLayout(), minFreqUnit, 3, 5, 1, 1, 0.1, 1.0, 0, 0, GridBagConstraints.BOTH, GridBagConstraints.LINE_START); - WebButton minMidi = new WebButton("Min. MIDI Pitch"); - minMidi.addActionListener(actionEvent -> { + WebButton alignFreq2Midi = new WebButton("

Align Frequencies

with MIDI Pitches

"); + alignFreq2Midi.setPadding(Settings.paddingInDialogs); + alignFreq2Midi.setToolTip("Sets min. and max. frequency of the CQT so that the piano roll pitches align with it."); + alignFreq2Midi.addActionListener(actionEvent -> { + maxFreq.setValue(12543.8539514160); minFreq.setValue(8.1757989156); }); - Tools.addComponentToGridBagLayout(this, (GridBagLayout) this.getLayout(), minMidi, 4, 5, 1, 1, 0.1, 1.0, 0, 0, GridBagConstraints.BOTH, GridBagConstraints.LINE_START); + Tools.addComponentToGridBagLayout(this, (GridBagLayout) this.getLayout(), alignFreq2Midi, 4, 4, 1, 2, 0.1, 1.0, 0, 0, GridBagConstraints.BOTH, GridBagConstraints.LINE_START); // bins per semitone WebLabel binsLabel = new WebLabel("Bins per Semitone:", WebLabel.RIGHT); diff --git a/src/mpmToolbox/gui/audio/WaveformPanel.java b/src/mpmToolbox/gui/audio/WaveformPanel.java index dca0fc7..b3294fc 100644 --- a/src/mpmToolbox/gui/audio/WaveformPanel.java +++ b/src/mpmToolbox/gui/audio/WaveformPanel.java @@ -1,5 +1,6 @@ package mpmToolbox.gui.audio; +import com.alee.laf.menu.WebCheckBoxMenuItem; import com.alee.laf.menu.WebMenu; import com.alee.laf.menu.WebMenuItem; import com.alee.laf.menu.WebPopupMenu; @@ -34,10 +35,10 @@ protected void paintComponent(Graphics g) { if (waveformImage == null) return; - Graphics2D g2 = (Graphics2D)g; // make g a Graphics2D object so we can use its extended drawing features + Graphics2D g2 = (Graphics2D)g; // make g a Graphics2D object, so we can use its extended drawing features g2.drawImage(waveformImage, 0, 0, this); // draw the waveform - this.drawPianoRoll(g2); // TODO this is only for testing, yet + this.drawPianoRoll(g2); // draw the mouse cursor if (this.mousePosition != null) { @@ -46,7 +47,7 @@ protected void paintComponent(Graphics g) { // g2.drawLine(0, this.mousePosition.y, this.getWidth(), this.mousePosition.y); // print info text - int sampleIndex = this.getSampleIndex(this.mousePosition); + int sampleIndex = this.getSampleIndex(this.mousePosition.getX()); double millisec = Tools.round(((double) sampleIndex / this.parent.getAudio().getFrameRate()) * 1000.0, 2); g2.setColor(Color.LIGHT_GRAY); g2.drawString("Sample No.: " + sampleIndex, 2, Settings.getDefaultFontSize()); @@ -55,13 +56,39 @@ protected void paintComponent(Graphics g) { } /** - * compute which sample the mouse cursor is pointing at - * @param mousePosition - * @return + * on mouse enter event + * @param e */ - protected int getSampleIndex(Point mousePosition) { - double relativePosition = mousePosition.getX() / this.getWidth(); - return (int) Math.round((relativePosition * (this.parent.getRightmostSample() - this.parent.getLeftmostSample())) + this.parent.getLeftmostSample()); + @Override + public void mouseEntered(MouseEvent e) { + if (this.parent.getAudio() == null) + return; + + super.mouseEntered(e); + } + + /** + * on mouse exit event + * @param e + */ + @Override + public void mouseExited(MouseEvent e) { + if (this.parent.getAudio() == null) + return; + + super.mouseExited(e); + } + + /** + * on mouse exit event + * @param e + */ + @Override + public void mouseMoved(MouseEvent e) { + if (this.parent.getAudio() == null) + return; + + super.mouseMoved(e); } /** @@ -70,20 +97,20 @@ protected int getSampleIndex(Point mousePosition) { */ @Override public void mouseClicked(MouseEvent e) { - if (this.parent.getAudio() == null) // if there is no audio data - return; // nothing to do here + if (this.parent.getAudio() == null) // if there is no audio data + return; // nothing to do here switch (e.getButton()) { case MouseEvent.BUTTON1: // left click - // TODO: select a note, place a marker ... + super.mouseClicked(e); // select a note break; case MouseEvent.BUTTON3: // right click = context menu - WebPopupMenu menu = new WebPopupMenu(); + WebPopupMenu menu = this.getContextMenu(e); // play from here WebMenuItem playFromHere = new WebMenuItem("Play from here"); playFromHere.addActionListener(actionEvent -> { - this.parent.getParent().getSyncPlayer().triggerPlayback(this.getSampleIndex(e.getPoint())); + this.parent.getParent().getSyncPlayer().triggerPlayback(this.getSampleIndex(e.getPoint().getX())); }); menu.add(playFromHere); @@ -106,9 +133,6 @@ public void mouseClicked(MouseEvent e) { } menu.add(chooseChannel); - // choose overlay - menu.add(this.getPianoRollTools()); - menu.show(this, e.getX() - 25, e.getY()); break; } diff --git a/src/mpmToolbox/gui/msmTree/MsmTree.java b/src/mpmToolbox/gui/msmTree/MsmTree.java index a411a68..ee6a4a2 100644 --- a/src/mpmToolbox/gui/msmTree/MsmTree.java +++ b/src/mpmToolbox/gui/msmTree/MsmTree.java @@ -10,6 +10,7 @@ import com.alee.managers.style.StyleId; import mpmToolbox.gui.ProjectPane; import mpmToolbox.gui.score.ScoreDisplayPanel; +import nu.xom.Attribute; import nu.xom.Element; import nu.xom.Node; @@ -139,6 +140,35 @@ public MsmTreeNode findNode(Node userObject, boolean depthFirstStrategy) { return null; } + /** + * find the first MsmTreeNode with the specified ID + * @param id + * @param depthFirstStrategy true for depth first search, false for breath first search + * @return + */ + public MsmTreeNode findNode(String id, boolean depthFirstStrategy) { + if (id == null) + return null; + + Enumeration e = depthFirstStrategy ? this.getRootNode().depthFirstEnumeration() : this.getRootNode().breadthFirstEnumeration(); + + while (e.hasMoreElements()) { + MsmTreeNode treeNode = e.nextElement(); + if (!(treeNode.getUserObject() instanceof Element)) + continue; + Element xml = (Element) treeNode.getUserObject(); + + Attribute idAtt = xml.getAttribute("id", "http://www.w3.org/XML/1998/namespace"); + if (idAtt == null) + continue; + + if (idAtt.getValue().equals(id)) + return treeNode; + } + + return null; + } + /** * The TreeSelectionListener is connected to the MSM tree and fires when something is selected there. * So the score display can highlight notes if possible. diff --git a/src/mpmToolbox/gui/score/ScoreDisplayPanel.java b/src/mpmToolbox/gui/score/ScoreDisplayPanel.java index 502aa18..51a2dd3 100644 --- a/src/mpmToolbox/gui/score/ScoreDisplayPanel.java +++ b/src/mpmToolbox/gui/score/ScoreDisplayPanel.java @@ -886,8 +886,11 @@ public void mouseDragged(MouseEvent mouseEvent) { */ @Override public void mouseMoved(MouseEvent mouseEvent) { - if (mouseEvent.isControlDown()) // if CTRL is pressed we are in pan & zoom mode and there is nothing to be done + if (mouseEvent.isControlDown()) { // if CTRL is pressed we are in pan & zoom mode and there is nothing to be done + Element selectedElement = this.getOverlayElementAt(mouseEvent); // get the overlay element at the mouse position + this.setCursor((selectedElement == null) ? Cursor.getDefaultCursor() : new Cursor(Cursor.HAND_CURSOR)); // change mouse cursor to hand if there is an overlay element return; + } switch (this.parent.currentInteractionMode) { case markNotes: @@ -903,6 +906,8 @@ public void mouseMoved(MouseEvent mouseEvent) { break; case panAndZoom: + Element selectedElement = this.getOverlayElementAt(mouseEvent); // get the overlay element at the mouse position + this.setCursor((selectedElement == null) ? Cursor.getDefaultCursor() : new Cursor(Cursor.HAND_CURSOR)); // change mouse cursor to hand if there is an overlay element default: break; } @@ -989,8 +994,6 @@ public void keyTyped(KeyEvent keyEvent) { */ @Override public void keyPressed(KeyEvent keyEvent) { -// if (keyEvent.isControlDown()) -// this.parent.markNotesButton.setSelected(!this.parent.markNotesButton.isSelected()); } /** diff --git a/src/mpmToolbox/gui/syncPlayer/SyncPlayer.java b/src/mpmToolbox/gui/syncPlayer/SyncPlayer.java index ce6c913..3239a8c 100644 --- a/src/mpmToolbox/gui/syncPlayer/SyncPlayer.java +++ b/src/mpmToolbox/gui/syncPlayer/SyncPlayer.java @@ -96,6 +96,7 @@ private void makeGui() { if (this.parent.getAudioFrame() != null) { this.parent.getAudioFrame().updateAlignment(true); } + this.parent.getAudioFrame().updateAudioTools(); } }); Tools.addComponentToGridBagLayout(this, (GridBagLayout) this.getLayout(), this.performanceChooser, 0, 0, 1, 1, 1.0, 1.0, 0, 0, GridBagConstraints.BOTH, GridBagConstraints.LINE_START); @@ -134,14 +135,14 @@ private void makeGui() { if (itemEvent.getStateChange() == ItemEvent.SELECTED) { // System.out.println(itemEvent.toString()); Audio audio = ((AudioChooserItem) itemEvent.getItem()).getValue(); - if (audio == null) { // audio is unselected - if (this.performanceChooser.getSelectedItem() == this.alignmentPerformance) // if alignment performance option was selected - this.performanceChooser.setSelectedIndex(0); // jump to the first option - this.performanceChooser.removeItem(this.alignmentPerformance); // remove the alignment performance option from the performance chooser + if (audio == null) { // audio is unselected + if (this.performanceChooser.getSelectedItem() == this.alignmentPerformance) // if alignment performance option was selected + this.performanceChooser.setSelectedIndex(0); // jump to the first option + this.performanceChooser.removeItem(this.alignmentPerformance); // remove the alignment performance option from the performance chooser } else { - this.updatePerformanceList(); // update the performance chooser list to add/delete the alignment performance option + this.updatePerformanceList(); // update the performance chooser list to add/delete the alignment performance option } - this.parent.getAudioFrame().updateAudio(true); // communicate the selection to the audio analysis frame as this should also display it + this.parent.getAudioFrame().updateAudio(true); // communicate the selection to the audio analysis frame as this should also display it } }); Tools.addComponentToGridBagLayout(this, (GridBagLayout) this.getLayout(), this.audioChooser, 0, 1, 1, 1, 1.0, 1.0, 0, 0, GridBagConstraints.BOTH, GridBagConstraints.LINE_START); @@ -615,7 +616,7 @@ public void run() { if (this.midiIsLonger) { relativePlaybackPosition = getMidiPlayer().getRelativePosition(); } else { - relativePlaybackPosition = (getAudioPlayer().getMicrosecondLength() >= this.microsecAudioOffset) ? 1.0 : (double) (getAudioPlayer().getMicrosecondPosition() - this.microsecAudioOffset) / (double) (getAudioPlayer().getMicrosecondLength() - this.microsecAudioOffset); + relativePlaybackPosition = (getAudioPlayer().getMicrosecondLength() <= this.microsecAudioOffset) ? 1.0 : (double) (getAudioPlayer().getMicrosecondPosition() - this.microsecAudioOffset) / (double) (getAudioPlayer().getMicrosecondLength() - this.microsecAudioOffset); } if ((playbackSlider.getValue() == sliderMax) || (!getAudioPlayer().isPlaying() && !getMidiPlayer().isPlaying())) { diff --git a/src/mpmToolbox/projectData/Audio.java b/src/mpmToolbox/projectData/Audio.java index 79169ae..b8eb82a 100644 --- a/src/mpmToolbox/projectData/Audio.java +++ b/src/mpmToolbox/projectData/Audio.java @@ -23,7 +23,7 @@ public class Audio extends meico.audio.Audio { protected final ArrayList waveforms; // contains the waveform data for each audio channel as doubles in [-1.0, 1.0] private WaveformImage waveformImage = null; // the waveform image of this audio data private SpectrogramImage spectrogramImage = null; // the visualization of the above spectrogram - private final Alignment alignment; + private Alignment alignment; // audio to MSM alignment /** * constructor; use this one to load and decode MP3 files @@ -35,8 +35,7 @@ public Audio(File file, Msm msm) throws IOException, UnsupportedAudioFileExcepti super(file); this.waveforms = convertByteArray2DoubleArray(this.getAudio(), this.getFormat()); - this.alignment = new Alignment(msm, null); - this.alignment.scaleTiming(((double) this.getNumberOfSamples() / this.getFrameRate()) * 1000.0); // scale the initial alignment to the milliseconds length of the audio; so all notes are visible and in a good starting position + this.initAlignment(msm); } /** @@ -55,7 +54,7 @@ public Audio(Element projectAudioData, String projectBasePath, Msm msm) throws I Element alignmentData = projectAudioData.getFirstChildElement("alignment"); this.alignment = new Alignment(msm, alignmentData); if (alignmentData == null) // if we had no alignment data from the project file, an initial alignment was generated with a default tempo that will potentially not fit the audio length - this.alignment.scaleTiming(((double) this.getNumberOfSamples() / this.getFrameRate()) * 1000.0); // scale the initial alignment to the milliseconds length of the audio; so all notes are visible and in a good starting position + this.alignment.scaleOverallTiming(((double) this.getNumberOfSamples() / this.getFrameRate()) * 1000.0); // scale the initial alignment to the milliseconds length of the audio; so all notes are visible and in a good starting position } /** @@ -78,6 +77,15 @@ public Alignment getAlignment() { return this.alignment; } + /** + * initialize or reset the alignment data for this Audio object + * @param msm the Msm instance to be aligned with this Audio object + */ + public void initAlignment(Msm msm) { + this.alignment = new Alignment(msm, null); + this.alignment.scaleOverallTiming(((double) this.getNumberOfSamples() / this.getFrameRate()) * 1000.0); // scale the initial alignment to the milliseconds length of the audio; so all notes are visible and in a good starting position + } + /** * a getter for the waveform data * @return an ArrayList where each element is the waveform of one channel diff --git a/src/mpmToolbox/projectData/alignment/Alignment.java b/src/mpmToolbox/projectData/alignment/Alignment.java index 572502d..c1d25b4 100644 --- a/src/mpmToolbox/projectData/alignment/Alignment.java +++ b/src/mpmToolbox/projectData/alignment/Alignment.java @@ -18,6 +18,7 @@ public class Alignment { private final ArrayList parts = new ArrayList<>(); private PianoRoll pianoRoll = null; private final Msm msm; + private ArrayList timingTransformation = new ArrayList<>(); // each element provides the following values {startDate, endDate, toStartDate, toEndDate}, all in milliseconds /** * constructor @@ -144,12 +145,121 @@ public Element toXml() { return out; } + /** + * Use this method when notes are un-fixed or fixed notes are silently repositioned. + * It recomputes the milliseconds timing of all notes. + */ + public void updateTiming() { + // create an ordered list of all fixed notes + ArrayList fixedNotes = new ArrayList<>(); + for (Part part : this.getParts()) { + ArrayList f = part.getAllFixedNotes(false); // get the part's list of fixed notes in the order of the current timing + + if (fixedNotes.isEmpty()) { // if the fixedNotes list is empty + fixedNotes.addAll(f); // we can simply add the part's list, it is already ordered + continue; // go on with the next part + } + + int i = 0; + for (Note n : f) { + boolean added = false; + for (; i < fixedNotes.size(); ++i) { + if (fixedNotes.get(i).getMillisecondsDate() > n.getMillisecondsDate()) { + fixedNotes.add(i, n); + added = true; + i++; + break; + } + } + if (!added) + fixedNotes.add(n); + } + } + + if (fixedNotes.isEmpty()) // if no fixed notes + return; // we are done + + this.timingTransformation.clear(); // we compute the timing transformation data anew + Note beginner = null; // the section to be scaled begins with this note's millisecondsDate and initialMillisecondsDate + Note stopper = null; // the section is scaled into [beginner.millisecondsDate; this note's millisecondsDate) + for (int i=0; i < fixedNotes.size(); ++i) { + Note n = fixedNotes.get(i); + + if (stopper == null) // if we seek a stopper + stopper = n; // we take the first note we find + + // check if this is a beginner note, i.e. a note that was not shifted before another fixed note + boolean isEnder = true; + for (int j=i+1; j < fixedNotes.size(); ++j) { // check if another fixed note follows in the sequence that was initially before the current note + if (fixedNotes.get(j).getInitialMillisecondsDate() < n.getInitialMillisecondsDate()) { // if we found one + isEnder = false; // set the flag false + break; + } + } + + if (isEnder) { // if we found the next beginner = "ender" of the previous section + if (beginner != null) // usually we have a beginner + this.timingTransformation.add(new double[]{beginner.getInitialMillisecondsDate(), n.getInitialMillisecondsDate(), beginner.getMillisecondsDate(), stopper.getMillisecondsDate()}); + else // but right at the beginning we have no beginner but have to handle the notes before the first fixed note + this.timingTransformation.add(new double[]{0.0, n.getInitialMillisecondsDate(), Math.max(0.0, stopper.getMillisecondsDate() - n.getInitialMillisecondsDate()), stopper.getMillisecondsDate()}); + + beginner = n; + stopper = null; + } + } + + // after the above loop there is the last beginner left for which we have to add an entry in the timingTransformation list + assert beginner != null; + this.timingTransformation.add(new double[]{beginner.getInitialMillisecondsDate(), Double.MAX_VALUE, beginner.getMillisecondsDate(), Double.MAX_VALUE}); + + this.renderTiming(); // compute the new milliseconds timing + } + + /** + * apply the current timing transformation to all parts, so their note get new millisecondsDates and millisecondsDateEnds + */ + private void renderTiming() { + for (Part part : this.getParts()) // apply the timing transform to each part + part.transformTiming(this.timingTransformation); + } + + /** + * Sets a new milliseconds date and end date of the note and places it in the note sequence accordingly. + * The note is made fixed. The non-fixed notes are not repositioned by this method! This has to be done subsequently. + * @param note the note to be moved + * @param toMilliseconds the new onset position of the note + */ + public void reposition(@NotNull Note note, double toMilliseconds) { + note.setFixed(true); // pin the note to its position, so it will not be affected by timing interpolations when another note is dragged + + if (note.getMillisecondsDate() == toMilliseconds) // the note does not move + return; // done + + // we do not allow shifting the note to negative timing + if (toMilliseconds < 0.0) + toMilliseconds = 0.0; + + for (Part part : this.getParts()) + if (part.contains(note)) + part.reposition(note, toMilliseconds); + } + + /** + * resets the values of each note in each part to their initial values; + * the invoking application should also run recomputePianoRoll() to update the piano roll image + */ + public void reset() { + for (Part part : this.getParts()) { + part.reset(); + } + } + /** * Scale the complete music to the specified length, i.e. the last milliseconds.date.end. * This transformation is also applied to fixed notes! - * @param milliseconds + * @param milliseconds the milliseconds length to which the notes are scaled or null if no scaling is set */ - public void scaleTiming(double milliseconds) { + public void scaleOverallTiming(double milliseconds) { Note lastNoteSounding = null; for (Part part : this.parts) { @@ -165,7 +275,7 @@ public void scaleTiming(double milliseconds) { double factor = milliseconds / lastNoteSounding.getMillisecondsDateEnd(); for (Part part : this.parts) { - part.scaleTiming(factor); + part.scaleOverallTiming(factor); } } @@ -191,6 +301,34 @@ public PianoRoll getPianoRoll(double fromMilliseconds, double toMilliseconds, in return this.pianoRoll; } + /** + * get the latest PianoRoll instance without recomputing it + * @return the piano roll or null + */ + public PianoRoll getPianoRoll() { + return this.pianoRoll; + } + + /** + * recomputes the piano roll image with the same metrics as the current one + * @return + */ + public PianoRoll recomputePianoRoll() { + if (this.pianoRoll == null) + return null; + + for (Part p : this.parts) + p.recomputePianoRoll(); + + double fromMilliseconds = this.pianoRoll.getFromMilliseconds(); + double toMilliseconds = this.pianoRoll.getToMilliseconds(); + int imgWidth = this.pianoRoll.getWidth(); + int imgHeight = this.pianoRoll.getHeight(); + + this.pianoRoll = null; + return this.getPianoRoll(fromMilliseconds, toMilliseconds, imgWidth, imgHeight); + } + /** * put all alignment data (milliseconds.date, milliseconds.date.end, velocity) into a clone Msm object * @return diff --git a/src/mpmToolbox/projectData/alignment/Note.java b/src/mpmToolbox/projectData/alignment/Note.java index d2b9de0..fc40e14 100644 --- a/src/mpmToolbox/projectData/alignment/Note.java +++ b/src/mpmToolbox/projectData/alignment/Note.java @@ -10,12 +10,17 @@ * @author Axel Berndt */ public class Note { - private final Element xml; // a reference to the original MSM element - private double millisecondsDate; - private double millisecondsDateEnd; + private final Element xml; // a reference to the original MSM element + + private final double initialMillisecondsDate; // the initial date of the note will remain unaltered throughout any other transformations + private final double initialMillisecondsDateEnd; // the initial end date of the note will remain unaltered throughout any other transformations + + private double millisecondsDate; // the date of the note, subject to alteration + private double millisecondsDateEnd; // the end date of the note, subject to alteration + private double velocity; private double pitch; - private boolean fixed = false; // signals that the values of this note should not be scaled when another note is edited; this note is fixed + private boolean fixed = false; // signals that the values of this note should not be scaled when another note is edited; this note is fixed /** * constructor @@ -31,7 +36,8 @@ protected Note(Element xml) throws InvalidDataException, NumberFormatException { if (millisDate == null) throw new InvalidDataException("Invalid MSM element " + xml.toXML() + "; missing attribute date."); } - this.millisecondsDate = Double.parseDouble(millisDate.getValue()); + this.initialMillisecondsDate = Double.parseDouble(millisDate.getValue()); + this.millisecondsDate = this.initialMillisecondsDate; // parse the note's offset date Attribute millisEnd = Helper.getAttribute("milliseconds.date.end", xml); @@ -39,11 +45,12 @@ protected Note(Element xml) throws InvalidDataException, NumberFormatException { millisEnd = Helper.getAttribute("duration", xml); if (millisEnd == null) throw new InvalidDataException("Invalid MSM element " + xml.toXML() + "; missing attribute duration."); - this.millisecondsDateEnd = this.millisecondsDate + Double.parseDouble(millisEnd.getValue()); + this.initialMillisecondsDateEnd = this.millisecondsDate + Double.parseDouble(millisEnd.getValue()); } else { - this.millisecondsDateEnd = Double.parseDouble(millisEnd.getValue()); + this.initialMillisecondsDateEnd = Double.parseDouble(millisEnd.getValue()); } + this.millisecondsDateEnd = this.initialMillisecondsDateEnd; // parse the note's MIDI pitch Attribute ptch = Helper.getAttribute("midi.pitch", xml); @@ -89,14 +96,39 @@ protected void syncWith(Element alignmentData) { this.setFixed(Boolean.parseBoolean(a.getValue())); } + /** + * undo all changes and set the note's initial values + */ + public void reset() { + this.millisecondsDate = this.initialMillisecondsDate; + this.millisecondsDateEnd = this.initialMillisecondsDateEnd; + this.fixed = false; + } + /** * get the xml:id of the note * @return */ - protected String getId() { + public String getId() { return this.xml.getAttributeValue("id", "http://www.w3.org/XML/1998/namespace"); } + /** + * read the initial milliseconds date of the note + * @return + */ + public double getInitialMillisecondsDate() { + return this.initialMillisecondsDate; + } + + /** + * read the initial milliseconds end date of the note + * @return + */ + public double getInitialMillisecondsDateEnd() { + return this.initialMillisecondsDateEnd; + } + /** * read the milliseconds date of the note * @return @@ -181,7 +213,7 @@ public void setFixed(boolean fixed) { * access the original MSM element * @return */ - protected Element getXml() { + public Element getXml() { return this.xml; } diff --git a/src/mpmToolbox/projectData/alignment/Part.java b/src/mpmToolbox/projectData/alignment/Part.java index e2c9b36..aa4212b 100644 --- a/src/mpmToolbox/projectData/alignment/Part.java +++ b/src/mpmToolbox/projectData/alignment/Part.java @@ -1,7 +1,9 @@ package mpmToolbox.projectData.alignment; +import com.alee.api.annotations.NotNull; import com.sun.media.sound.InvalidDataException; import meico.mei.Helper; +import meico.supplementary.KeyValue; import nu.xom.Attribute; import nu.xom.Element; @@ -14,10 +16,11 @@ * @author Axel Berndt */ public class Part { - private final Element xml; // a reference to the original MSM part element + private final Element xml; // a reference to the original MSM part element private final int number; private final HashMap notes = new HashMap<>(); - private final ArrayList noteSequence = new ArrayList<>(); // the notes in sequential order + private final ArrayList noteSequence = new ArrayList<>(); // the notes in sequential order of their current date + private final ArrayList initialSequence = new ArrayList<>(); // the notes in sequential order of their initial date private PianoRoll pianoRoll = null; /** @@ -73,19 +76,43 @@ protected void syncWith(Element alignmentData) { public Note add(Note note) { Note out = this.notes.put(note.getId(), note); // this will add the id-note pair to the hashmap and overwrite any other note behind the same id; out will hold that previous note or null - if (out != null) // if there was a previous note that we replaced with the above line - this.noteSequence.remove(out); // remove it also from the sequence + if (out != null) { // if there was a previous note that we replaced with the above line + this.noteSequence.remove(out); // remove it also from the sequences + this.initialSequence.remove(out); + } + this.addToInitialSequence(note); // add note to initial sequence + this.addToSequence(note); // add the note at the right position to the sequence + + return out; + } - // add the note at the right position to the sequence - int i = this.noteSequence.size()-1; + /** + * add the note to the note sequence + * @param note + */ + private void addToSequence(Note note) { + // add note to noteSequence + int i = this.noteSequence.size() - 1; for (; i >= 0; --i) { double date = this.noteSequence.get(i).getMillisecondsDate(); if (date <= note.getMillisecondsDate()) break; } - this.noteSequence.add(i+1, note); // insert note also to the sequence + this.noteSequence.add(i + 1, note); // insert note also to the sequence + } - return out; + /** + * add the note to the initial sequence + * @param note + */ + private void addToInitialSequence(Note note) { + int i = this.initialSequence.size() - 1; + for (; i >= 0; --i) { + double date = this.initialSequence.get(i).getInitialMillisecondsDate(); + if (date <= note.getInitialMillisecondsDate()) + break; + } + this.initialSequence.add(i + 1, note); // insert note also to the sequence } /** @@ -95,8 +122,10 @@ public Note add(Note note) { */ public Note remove(String id) { Note out = this.notes.remove(id); - if (out != null) + if (out != null) { this.noteSequence.remove(out); + this.initialSequence.remove(out); + } return out; } @@ -134,6 +163,15 @@ public Note getNote(String id) { return this.notes.get(id); } + /** + * checks if the part contains the given note + * @param note + * @return + */ + public boolean contains(Note note) { + return this.notes.containsValue(note); + } + /** * find the first note at or after milliseconds and return its index in noteSequence * @param milliseconds @@ -160,7 +198,48 @@ else if (this.noteSequence.get(mid + 1).getMillisecondsDate() >= milliseconds) mid = (first + last) / 2; } return -1; + } + + /** + * retrieve the fixed notes at and around the specified milliseconds date + * @param milliseconds + * @return array of 3 notes {before, at, after}; may contain null values! + */ + public Note[] getFixedNoteBeforeAtAfter(double milliseconds) { + Note before = null; + Note at = null; + Note after = null; + + for (Note note : this.noteSequence) { + if (!note.isFixed()) + continue; + + if (note.getMillisecondsDate() > milliseconds) { + after = note; + break; + } + if (note.getMillisecondsDate() == milliseconds) + at = note; + else // if (note.getMillisecondsDate() < milliseconds) + before = note; + } + + return new Note[] {before, at, after}; + } + + /** + * collect all fixed notes from this part's note sequence and return them in timely order + * @param initialOrder get the fixed notes in their initial or current order + * @return + */ + public ArrayList getAllFixedNotes(boolean initialOrder) { + ArrayList fixed = new ArrayList<>(); + for (Note note : ((initialOrder) ? this.initialSequence : this.noteSequence)) { + if (note.isFixed()) + fixed.add(note); + } + return fixed; } /** @@ -183,15 +262,109 @@ public Note getLastNoteSounding() { return out; } + /** + * resets the values of each note to their initial value + * the invoking application should also run recomputePianoRoll() to update the piano roll image + */ + protected void reset() { + this.noteSequence.clear(); + + for (Note note : this.initialSequence) { + note.reset(); + this.noteSequence.add(note); + } + } + /** * Scales all notes' milliseconds dates by the specified factor. * This transformation is also applied to fixed notes! * @param factor */ - public void scaleTiming(double factor) { - for (Note note : this.noteSequence) { - note.setMillisecondsDate(note.getMillisecondsDate() * factor); - note.setMillisecondsDateEnd(note.getMillisecondsDateEnd() * factor); + protected void scaleOverallTiming(double factor) { + for (Note note : this.initialSequence) { + note.setMillisecondsDate(note.getInitialMillisecondsDate() * factor); + note.setMillisecondsDateEnd(note.getInitialMillisecondsDateEnd() * factor); + } + } + + /** + * Sets a new milliseconds date and end date of the note and places it in the note sequence accordingly. + * The non-fixed notes are not repositioned by this method! this has to be done subsequently. + * @param note + * @param toMilliseconds + */ + protected void reposition(@NotNull Note note, double toMilliseconds) { + note.setMillisecondsDateEnd(note.getInitialMillisecondsDateEnd() - note.getInitialMillisecondsDate() + toMilliseconds); + note.setMillisecondsDate(toMilliseconds); + this.noteSequence.remove(note); + this.addToSequence(note); + } + + /** + * places all non-fixed notes according to the positions of the fixed notes + * @param timingTransformation the transformation data; each element provides the following values {startDate, endDate, toStartDate, toEndDate}, all in milliseconds + */ + protected void transformTiming(ArrayList timingTransformation) { + for (double[] segment : timingTransformation) + this.transformTiming(segment[0], segment[1], segment[2], segment[3]); + + this.noteSequence.clear(); + for (Note note : this.initialSequence) + this.addToSequence(note); + } + + /** + * helper method for transformTiming(fixedNotes); + * transforms all note onsets in [startDate; endDate) to [toStartDate; toEndDate) + * and offsets in (startDate; endDate] to (toStartDate; toEndDate] + * @param startDate + * @param endDate + * @param toStartDate + * @param toEndDate + */ + private void transformTiming(double startDate, double endDate, double toStartDate, double toEndDate) { + boolean shiftDontScale = (endDate == startDate) || (toStartDate >= toEndDate); + double scaleFactor = shiftDontScale ? 1.0 : (toEndDate - toStartDate) / (endDate - startDate); // if the section has 0 length, we do not scale anything + + ArrayList fixed = new ArrayList<>(); + int i = 0; + for (; i < this.initialSequence.size(); ++i) { + Note note = this.initialSequence.get(i); + + if (note.isFixed()) { // if the note is fixed + fixed.add(note); // store for further treatment + continue; + } + + if (note.getInitialMillisecondsDate() >= endDate) // we have to check only the notes that start before endDate + break; + + // transform the note's onset + double iDate = note.getInitialMillisecondsDate(); + if (iDate >= startDate) + note.setMillisecondsDate(shiftDontScale ? toStartDate : ((iDate - startDate) * scaleFactor) + toStartDate); + + // transform the note's offset + iDate = note.getInitialMillisecondsDateEnd(); + if ((iDate > startDate) && (iDate <= endDate)) + note.setMillisecondsDateEnd(shiftDontScale ? endDate - startDate + toStartDate : ((iDate - startDate) * scaleFactor) + toStartDate); + } + + // now treat the fixed notes + for (; i < this.initialSequence.size(); ++i) { + Note note = this.initialSequence.get(i); + if (note.isFixed()) + fixed.add(note); // store for further treatment + } + for (Note f : fixed) { + if ((f.getMillisecondsDate() >= toEndDate) // if it is behind the target end date + || (f.getInitialMillisecondsDate() > startDate))// or it was behind the initial start date + continue; // we leave it unaltered + + // change the milliseconds offset date of the fixed note; its onset date is fixed + double iDate = f.getInitialMillisecondsDateEnd(); + if ((iDate > startDate) && (iDate <= endDate)) + f.setMillisecondsDateEnd(shiftDontScale ? endDate - startDate + toStartDate : ((iDate - startDate) * scaleFactor) + toStartDate); } } @@ -203,7 +376,7 @@ public void scaleTiming(double factor) { * @param imgHeight should correspond with the highest pitch class to be displayed, because one row of pixels is one diatonic pitch class, starting with MIDI's pitch 0; for MIDI-compliance use 128 * @return */ - protected PianoRoll getPianoRoll(double fromMilliseconds, double toMilliseconds, int imgWidth, int imgHeight) { + public PianoRoll getPianoRoll(double fromMilliseconds, double toMilliseconds, int imgWidth, int imgHeight) { if (fromMilliseconds == toMilliseconds) return null; @@ -245,7 +418,30 @@ protected PianoRoll getPianoRoll(double fromMilliseconds, double toMilliseconds, return this.pianoRoll; } + /** + * get the latest PianoRoll instance without recomputing it + * @return the piano roll or null + */ + public PianoRoll getPianoRoll() { + return this.pianoRoll; + } + + /** + * recomputes the piano roll image with the same metrics as the current one + * @return + */ + public PianoRoll recomputePianoRoll() { + if (this.pianoRoll == null) + return null; + double fromMilliseconds = this.pianoRoll.getFromMilliseconds(); + double toMilliseconds = this.pianoRoll.getToMilliseconds(); + int imgWidth = this.pianoRoll.getWidth(); + int imgHeight = this.pianoRoll.getHeight(); + + this.pianoRoll = null; + return this.getPianoRoll(fromMilliseconds, toMilliseconds, imgWidth, imgHeight); + } /** * access the original MSM element diff --git a/src/mpmToolbox/projectData/alignment/PianoRoll.java b/src/mpmToolbox/projectData/alignment/PianoRoll.java index e43a569..c5a7838 100644 --- a/src/mpmToolbox/projectData/alignment/PianoRoll.java +++ b/src/mpmToolbox/projectData/alignment/PianoRoll.java @@ -62,7 +62,15 @@ private boolean set(int x, int y, Note note) { if ((x < 0) || (x >= this.getWidth()) || (y < 0) || (y >= this.getHeight())) return false; - Color color = (note == null) ? new Color(0, 0, 0, 0) : Settings.scoreNoteColor; // if note is null, make the pixel completely transparent, otherwise set the note color + // if note is null, make the pixel completely transparent, otherwise set the performance color for fixed notes and note color for non-fixed notes + Color color; + if (note == null) + color = new Color(0, 0, 0, 0); + else if (note.isFixed()) + color = Settings.scorePerformanceColorHighlighted; + else + color = Settings.scoreNoteColor; + this.setRGB(x, y, color.getRGB()); // set the note color this.noteReferences[x][y] = note; // set reference return true; @@ -92,12 +100,13 @@ private void add(int x, int y, @NotNull Note note) { rgba[2] = (this.getRGB(x, y)) & 0xFF; rgba[3] = (this.getRGB(x, y) >> 24) & 0xff; + Color color = (note.isFixed()) ? Settings.scorePerformanceColorHighlighted : Settings.scoreNoteColor; // add the color of a note so that it becomes brighter, but avoid numbers greater than 255 - int r = Math.min(255, Math.round((((255f - rgba[0]) * Settings.scoreNoteColor.getRed()) / 255) + rgba[0])); - int g = Math.min(255, Math.round((((255f - rgba[1]) * Settings.scoreNoteColor.getGreen()) / 255) + rgba[1])); - int b = Math.min(255, Math.round((((255f - rgba[2]) * Settings.scoreNoteColor.getBlue()) / 255) + rgba[2])); - int a = Math.min(255, Math.round((((255f - rgba[3]) * Settings.scoreNoteColor.getAlpha()) / 255) + rgba[3])); + int r = Math.min(255, Math.round((((255f - rgba[0]) * color.getRed()) / 255) + rgba[0])); + int g = Math.min(255, Math.round((((255f - rgba[1]) * color.getGreen()) / 255) + rgba[1])); + int b = Math.min(255, Math.round((((255f - rgba[2]) * color.getBlue()) / 255) + rgba[2])); + int a = Math.min(255, Math.round((((255f - rgba[3]) * color.getAlpha()) / 255) + rgba[3])); // int a = Settings.scoreNoteColor.getAlpha(); // for a constant transparency this.setRGB(x, y, new Color(r, g, b, a).getRGB()); // add the color